From 2d61ac475e64c967005d95ad24d7f91ae148a441 Mon Sep 17 00:00:00 2001 From: LearnPrompt Date: Wed, 2 Sep 2026 14:13:25 +0900 Subject: [PATCH 1/9] =?UTF-8?q?feat(carl-file-organizer):=20=E8=BF=81?= =?UTF-8?q?=E5=85=A5=20carl-skills=EF=BC=8C=E4=B8=89=E8=89=B2/=E6=9D=82?= =?UTF-8?q?=E4=B9=B1=E5=BA=A6/notes=20=E5=9F=BA=E5=BA=A7=E3=80=81Windows?= =?UTF-8?q?=20=E5=85=BC=E5=AE=B9=E3=80=81=E4=B8=89=E8=89=B2=E6=8A=A5?= =?UTF-8?q?=E5=91=8A=E6=A8=A1=E6=9D=BF=E4=B8=8E=20build=5Freport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 45 + skills/ops/carl-file-organizer/README.md | 182 +++ skills/ops/carl-file-organizer/SKILL.md | 90 ++ .../carl-file-organizer/agents/openai.yaml | 3 + .../assets/report_template.html | 899 +++++++++++ .../carl-file-organizer/docs/plan-schema.md | 295 ++++ .../carl-file-organizer/docs/review-page.md | 104 ++ .../docs/storage-schema.md | 258 +++ .../carl-file-organizer/references/macos.md | 118 ++ .../references/partitions.md | 82 + .../carl-file-organizer/references/tiers.md | 117 ++ .../carl-file-organizer/references/windows.md | 97 ++ .../scripts/build_report.py | 1319 +++++++++++++++ .../scripts/carl_file_organizer/__init__.py | 3 + .../scripts/carl_file_organizer/__main__.py | 5 + .../scripts/carl_file_organizer/classifier.py | 298 ++++ .../scripts/carl_file_organizer/cli.py | 238 +++ .../scripts/carl_file_organizer/config.py | 544 +++++++ .../scripts/carl_file_organizer/executor.py | 1337 +++++++++++++++ .../scripts/carl_file_organizer/grouping.py | 327 ++++ .../scripts/carl_file_organizer/guard.py | 737 +++++++++ .../scripts/carl_file_organizer/i18n.py | 355 ++++ .../scripts/carl_file_organizer/ids.py | 24 + .../scripts/carl_file_organizer/manifest.py | 369 +++++ .../scripts/carl_file_organizer/notes.py | 283 ++++ .../scripts/carl_file_organizer/opener.py | 82 + .../scripts/carl_file_organizer/paths.py | 413 +++++ .../scripts/carl_file_organizer/planner.py | 1435 +++++++++++++++++ .../carl_file_organizer/profiles/__init__.py | 10 + .../carl_file_organizer/profiles/simple.json | 500 ++++++ .../carl_file_organizer/profiles/tiered.json | 501 ++++++ .../scripts/carl_file_organizer/report.py | 1266 +++++++++++++++ .../scripts/carl_file_organizer/scanner.py | 320 ++++ .../scripts/carl_file_organizer/server.py | 424 +++++ .../scripts/carl_file_organizer/status.py | 241 +++ .../scripts/carl_file_organizer/tags.py | 302 ++++ .../scripts/carl_file_organizer/trash.py | 270 ++++ .../scripts/carl_file_organizer/undo.py | 444 +++++ .../scripts/demo_fixture.py | 270 ++++ .../carl-file-organizer/scripts/hygiene.sh | 89 + .../carl-file-organizer/scripts/organize.py | 36 + .../scripts/storage_scan.py | 1090 +++++++++++++ tests/cfo_helpers.py | 85 + tests/cfo_path.py | 24 + .../carl_file_organizer/plan-v2-sample.json | 1230 ++++++++++++++ .../storage-analysis-sample.json | 245 +++ tests/test_cfo_carl_file_organizer.py | 140 ++ tests/test_cfo_classifier.py | 140 ++ tests/test_cfo_colors.py | 303 ++++ tests/test_cfo_config.py | 438 +++++ tests/test_cfo_executor.py | 949 +++++++++++ tests/test_cfo_fixture_contract.py | 281 ++++ tests/test_cfo_grouping.py | 201 +++ tests/test_cfo_guard.py | 273 ++++ tests/test_cfo_manifest.py | 234 +++ tests/test_cfo_notes.py | 287 ++++ tests/test_cfo_opener.py | 108 ++ tests/test_cfo_planner.py | 648 ++++++++ tests/test_cfo_portability.py | 182 +++ tests/test_cfo_report.py | 379 +++++ tests/test_cfo_report_template.py | 316 ++++ tests/test_cfo_server.py | 309 ++++ tests/test_cfo_status.py | 171 ++ tests/test_cfo_tags.py | 142 ++ tests/test_cfo_trash.py | 157 ++ tests/test_cfo_undo.py | 299 ++++ 66 files changed, 23363 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 skills/ops/carl-file-organizer/README.md create mode 100644 skills/ops/carl-file-organizer/SKILL.md create mode 100644 skills/ops/carl-file-organizer/agents/openai.yaml create mode 100644 skills/ops/carl-file-organizer/assets/report_template.html create mode 100644 skills/ops/carl-file-organizer/docs/plan-schema.md create mode 100644 skills/ops/carl-file-organizer/docs/review-page.md create mode 100644 skills/ops/carl-file-organizer/docs/storage-schema.md create mode 100644 skills/ops/carl-file-organizer/references/macos.md create mode 100644 skills/ops/carl-file-organizer/references/partitions.md create mode 100644 skills/ops/carl-file-organizer/references/tiers.md create mode 100644 skills/ops/carl-file-organizer/references/windows.md create mode 100644 skills/ops/carl-file-organizer/scripts/build_report.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py create mode 100644 skills/ops/carl-file-organizer/scripts/demo_fixture.py create mode 100755 skills/ops/carl-file-organizer/scripts/hygiene.sh create mode 100755 skills/ops/carl-file-organizer/scripts/organize.py create mode 100644 skills/ops/carl-file-organizer/scripts/storage_scan.py create mode 100644 tests/cfo_helpers.py create mode 100644 tests/cfo_path.py create mode 100644 tests/fixtures/carl_file_organizer/plan-v2-sample.json create mode 100644 tests/fixtures/carl_file_organizer/storage-analysis-sample.json create mode 100644 tests/test_cfo_carl_file_organizer.py create mode 100644 tests/test_cfo_classifier.py create mode 100644 tests/test_cfo_colors.py create mode 100644 tests/test_cfo_config.py create mode 100644 tests/test_cfo_executor.py create mode 100644 tests/test_cfo_fixture_contract.py create mode 100644 tests/test_cfo_grouping.py create mode 100644 tests/test_cfo_guard.py create mode 100644 tests/test_cfo_manifest.py create mode 100644 tests/test_cfo_notes.py create mode 100644 tests/test_cfo_opener.py create mode 100644 tests/test_cfo_planner.py create mode 100644 tests/test_cfo_portability.py create mode 100644 tests/test_cfo_report.py create mode 100644 tests/test_cfo_report_template.py create mode 100644 tests/test_cfo_server.py create mode 100644 tests/test_cfo_status.py create mode 100644 tests/test_cfo_tags.py create mode 100644 tests/test_cfo_trash.py create mode 100644 tests/test_cfo_undo.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9f00d04 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.12"] + exclude: + # macos-latest is arm64, and setup-python only resolves 3.9 from + # 3.9.13 onward there, sometimes failing to find a build at all. + # The ubuntu 3.9 leg already guards the syntax floor. + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Everything is stdlib on purpose, so there is nothing to install and the + # 3.9 leg is a real floor rather than a formality. + - name: Compile check + run: python -m compileall -q skills tests + - name: Run tests + run: python -m unittest discover -s tests -v + + hygiene: + name: hygiene + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: no private paths, borrowed vocabulary or private product names + run: bash skills/ops/carl-file-organizer/scripts/hygiene.sh diff --git a/skills/ops/carl-file-organizer/README.md b/skills/ops/carl-file-organizer/README.md new file mode 100644 index 0000000..d56184c --- /dev/null +++ b/skills/ops/carl-file-organizer/README.md @@ -0,0 +1,182 @@ +**中文** · [English](./README.en.md) + +# 文件整理 · Carl File Organizer + +下载目录整理器。零依赖 Python 命令行工具,先只读扫描出一份方案,你在网页上看过、勾选过,它才动手。默认只进废纸篓,永久删除要你显式打开开关。 + +```bash +uvx --from git+https://github.com/LearnPrompt/carl-file-organizer carl-file-organizer plan ~/Downloads +open ~/Downloads/00_下载目录管理/report.html +``` + +打开的页面里逐项看方案,勾上你同意的,导出批准文件,再跑一遍 apply,细节往下看。 + +## 安装 + +三种装法,挑一种。 + +```bash +# 一次性跑,不留常驻命令 +uvx --from git+https://github.com/LearnPrompt/carl-file-organizer carl-file-organizer plan ~/Downloads + +# 装成常驻命令,以后直接 carl-file-organizer +uv tool install git+https://github.com/LearnPrompt/carl-file-organizer +# 或者 +pipx install git+https://github.com/LearnPrompt/carl-file-organizer + +# 自己克隆 +git clone https://github.com/LearnPrompt/carl-file-organizer +cd carl-file-organizer +python3 -m carl_file_organizer plan ~/Downloads +``` + +钉住某个版本就在仓库地址后面加 `@v0.2.0`。`uvx` 首次运行会现场拉代码构建,之后按 commit 缓存,不会每次都重新拉。 + +只需要 Python 3.9 及以上,零第三方依赖。macOS 上功能齐全,包括进废纸篓和 Finder 标签。Linux 上 `plan` 和 `apply` 照常跑,废纸篓退化成系统的 `gio trash`,查不到 `gio` 时废纸篓选项直接不出现在页面上,不会悄悄改成永久删除。 + +## 三步用起来 + +第一步,只读扫描: + +```bash +carl-file-organizer plan ~/Downloads --lang zh +``` + +不动任何文件。产物落在 `~/Downloads/00_下载目录管理/`,一份 `plan.json` 给程序和 Agent 读,一份 `report.html` 给你看。首次跑还会在 `~/Downloads/.carl-file-organizer.json` 写一行最小配置,记住你选的语言和规则集,下次不用再传 `--lang`。 + +第二步,打开 `report.html`,逐项看它想怎么归类,想留哪份重复文件,哪些进废纸篓。两条路都行:双击直接打开,勾完点导出,浏览器会存下一份 `carl-file-organizer-approved.json`,默认就落在 `~/Downloads`;或者跑 `carl-file-organizer review ~/Downloads/00_下载目录管理/plan.json --serve`,本地起一个只绑 127.0.0.1 的服务,页面上的按钮直接落地,不用手动导出再执行。 + +![审核页(中文)](./docs/images/report-zh.png) + +第三步,执行: + +```bash +carl-file-organizer apply ~/Downloads/carl-file-organizer-approved.json +``` + +传进去的就是刚才导出的那份文件,放在哪都行,这里按浏览器的默认下载位置写。执行完它会把这份批准文件收进 `~/Downloads/00_下载目录管理/approved-<时间戳>.json` 留档,顶层不会多出一个散件。 + +第一次在终端里真正让它把文件送进废纸篓,macOS 会弹一个允许终端控制 Finder 的系统授权,点允许就行,这是系统自己的机制,不是这个工具要你交出什么权限。 + +后悔了: + +```bash +carl-file-organizer undo ~/Downloads/00_下载目录管理/audit.jsonl +``` + +按执行记录逆序把移动过的文件原路移回。复查完想清掉页面留下的 Finder 标签,跑 `carl-file-organizer clear-tags ~/Downloads`。 + +## 它不会做什么 + +不读文件内容,敏感命名只看名字就把它隔离进 60_敏感信息,不会因为怕漏判就打开文件看一眼。不删除,默认进废纸篓,永久删除要显式加 `--allow-permanent-delete` 并且只对你已经批准过的 id 生效,命令行不加这个开关时整批永久删除请求直接被拒绝。不覆盖同名文件,目标位置已经有东西时转进重复待确认区,两份都留着。不出你指定的这一个目录,源和目标必须落在同一个根目录之内,`/`、家目录本身、`~/Library` 一律拒绝。不碰 `.git`、隐藏文件和符号链接段落里的路径。不联网,`review --serve` 只绑在 127.0.0.1 上,还带一次性 token,关掉终端就失效。每次 apply 都往 `audit.jsonl` 里写一行,`undo` 照着它回滚。 + +动手之前它还会查一遍这个文件是不是正被别的东西用着:有没有进程正打开它,有没有被 launchd、cron 或者 shell 配置文件引用,有没有你自己项目目录下的符号链接指向它,是不是一个 git worktree 或者虚拟环境。查到了就先停手,把原因写进方案里让你自己决定,不会替你硬闯。这套检查覆盖不到某个 App 自己数据库里存的绝对路径,比如某个编辑器的最近项目列表,这类情况靠静置期、Finder 标签和 undo 兜底,不是靠猜,这一点要老实说清楚。 + +## 为什么会有它 + +戒指老爷爷那套是说出你卡在哪,老爷爷点人。这里换了个场景:文件先站好队,老爷爷点谁走谁走。它不替你拿主意,它只是先把八成能用规则说清楚的判断做完——扩展名、静置了多久、是不是压缩包配对、是不是重复副本、是不是能重新生成的构建产物,这些代码就能跑。剩下两成看名字看上下文才能判断的,程序单独开一栏叫待判断,连蒙的动作都不做,留给你自己看,或者交给 Agent 按文件名判断后再回来找你确认。 + +这套流程真在一个跑了一个多月的下载目录上走过好几轮,移动了大几十项,没删过一个东西,每一项都留着清单和 Finder 标签,事后复查一遍复查数归零才算完。 + +## 分区 + +| 编号 | 中文名 | 英文名 | 放什么 | +|---|---|---|---| +| 00 | 收件箱 | Inbox | 新到的、静置期没到的、待判断的、按 Finder 标签分组的、重复待确认的 | +| 10 | 工作区 | Workspace | 项目代码、脚本和工具导出、数据库与查询文件 | +| 20 | 知识库 | Library | 文档资料、图片素材、视频素材、音频素材、网页资料,再往下一层按格式分 | +| 60 | 敏感信息 | Sensitive | 按文件名判断出的敏感文件,待你自己转移,程序不读内容 | +| 90 | 归档 | Archive | 静置超过 7 天的压缩包和安装包 | + +目录深度封顶三层,`--lang zh|en` 选中英文名字。已经存在的编号目录被当成边界,程序不会再往里面深扫。 + +## 它靠什么判断 + +| 机制 | 规则 | 你能调什么 | +|---|---|---| +| 静置期 | 文档类默认 48 小时,压缩包和安装包默认 168 小时,没到期的留在收件箱不强行分类 | `.carl-file-organizer.json` 里的 `aging` 段 | +| 敏感命名隔离 | 精确文件名、去掉后缀的精确短词、敏感后缀、长关键词四层判断,命中就进 60_敏感信息待转移,全程不打开文件 | `sensitive.exact_names`、`name_patterns` | +| Finder 标签 | 本轮被动过的文件和目录打上「文件移动」标签,方便肉眼核对,`clear-tags` 统一清掉 | 无,标记失败只提醒不中断 | +| 压缩包配对 | `a.zip` 和解压出来的 `a/` 同时存在时归成一组,页面上单选留谁,默认留目录,选之前两份都不动 | 组内单选 | +| 重复副本 | 先比对文件大小,加 `--hash-duplicates` 才进一步用 sha256 比内容;名字像副本但没比对过内容的,默认两份都保留 | `--hash-duplicates` | +| 可再生产物 | `node_modules`、`.next`、`dist`、`build`、`__pycache__` 这类只给废纸篓选项,附一句怎么重新生成 | `regenerable` 名单 | + +## 命令 + +| 命令 | 做什么 | 会不会动文件 | +|---|---|---| +| `carl-file-organizer plan [目录]` | 只读扫描,生成 plan.json 和 report.html | 否 | +| `carl-file-organizer review --serve` | 本地起服务打开审核页,按钮直接处置已勾选项 | 是,仅限批准过的 | +| `carl-file-organizer apply ` | 执行批准过的动作 | 是 | +| `carl-file-organizer undo ` | 按记录逆序移回原处 | 是 | +| `carl-file-organizer status [目录]` | 看最后一次整理的时间和现在是不是又乱了 | 否 | +| `carl-file-organizer clear-tags [目录]` | 清掉本工具打过的 Finder 标签 | 是,只改标签 | + +常用参数:`plan` 支持 `--lang zh|en`、`--profile tiered|simple`、`--hash-duplicates`、`--large-mb N`、`--target-percent P`、`--no-dir-sizes`、`--no-permanent-delete-options`、`--now ISO`、`--allow-outside-home`、`--allow-referenced`;`apply` 支持 `--dry-run`、`--allow-permanent-delete`、`--audit PATH`;`review` 的 `--serve` 是必须显式加的,另外支持 `--port N`、`--no-open`、`--allow-permanent-delete`;`undo` 支持 `--dry-run`,除了 audit.jsonl 也能传清单 tsv;`status` 支持 `--lang`、`--allow-outside-home`;`clear-tags` 支持重复出现的 `--from tags.txt`、`--all-lists`、`--tag NAME`、`--dry-run`。 + +## 给 Agent 用 + +```bash +npx skills@latest add LearnPrompt/carl-file-organizer +``` + +Agent 拿到的边界很窄:它能跑 `plan`,能读 `plan.json`,只对待判断且允许改道的那部分文件按文件名做语义判断,把判断写进批准文件的 `overrides` 里,列给你看,等你回复。真正执行永远要你自己在对话里说一句执行,Agent 不会替你按下那个按钮。完整规则在 [skills/carl-file-organizer/SKILL.md](./skills/carl-file-organizer/SKILL.md),字段表在 [docs/plan-schema.md](./docs/plan-schema.md)。 + +## plan.json 长什么样 + +```json +{ + "schema_version": 2, + "lang": "zh", + "source_root_portable": "$HOME/Downloads", + "managed_dir_portable": "$HOME/Downloads/00_下载目录管理", + "names": { + "library.docs.pdf": "20_知识库/文档资料/PDF", + "archive.zip.zip": "90_归档/压缩包/ZIP" + }, + "actions": [ + { + "id": "9c1f0a7b2d3e4f55", + "kind": "move", + "filename": "report.pdf", + "destination_key": "library.docs.pdf", + "reason": {"zh": "文档按类型归档到 20_知识库/文档资料/PDF"}, + "tier": "routine", + "approvable": true + } + ], + "approved_action_ids": [], + "overrides": [] +} +``` + +完整字段表在 [docs/plan-schema.md](./docs/plan-schema.md),审核页的视觉规范在 [docs/review-page.md](./docs/review-page.md)。 + +## 老师们 + +| 老师 | 教的是 | 在这里变成了什么 | +|---|---|---| +| [John Locke](https://en.wikipedia.org/wiki/Commonplace_book) | commonplace book,能检索比完美分类更重要 | 收件箱先接住一切,待判断不强行归类 | +| [Melvil Dewey](https://en.wikipedia.org/wiki/Dewey_Decimal_Classification) | 用编号分区 | 00/10/20/60/90 这套编号,但只到三层就封顶 | +| [S. R. Ranganathan](https://en.wikipedia.org/wiki/S._R._Ranganathan) | 分面分类,一件东西可以有多个机器可读的属性 | `destination_key`、`tier`、`rule` 各管一面,不挤成一个标签 | +| Aby Warburg | 好邻居法则,把相关的放在一起,哪怕分类学上它们不属于同一类 | 分类原则的来源,也是压缩包和解压目录成对处理的直觉来源 | +| [Niklas Luhmann](https://en.wikipedia.org/wiki/Niklas_Luhmann) | 卡片盒的价值在关系和理由,不在卡片本身 | 每条 action 都带一句 `reason`,不是光给个目的地 | +| [Paul Otlet](https://en.wikipedia.org/wiki/Paul_Otlet) | 给人读的卡片,配一份给机器读的索引 | `report.html` 给人,`plan.json` 给程序和 Agent | +| [David Allen](https://en.wikipedia.org/wiki/Getting_Things_Done) | GTD 的 capture、clarify、review、act 四步 | 就是 plan、待判断、review、apply | +| [Andrej Karpathy](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) | 面向 Agent 可读的知识库分区思路 | `names` 和 `destination_key` 这套机器优先的分区键 | +| [KKKKhazix / storage-analyzer](https://github.com/KKKKhazix/khazix-skills/tree/main/storage-analyzer) | 网页上一键处置的体验 | `review --serve` 页面上直接点按钮处置 | + +署名:LearnPrompt / 卡尔的AI沃茨。 + +## 开发 + +```bash +python3 -m unittest discover -s tests -v +python3 scripts/demo_fixture.py /tmp/gn-demo +``` + +`scripts/demo_fixture.py` 造一份假的下载目录,静置期内外的文件、成对压缩包、重复副本、敏感命名、构建产物都齐全,CI 的冒烟测试和这份 README 的截图用的都是它,不会碰任何真实文件。CI 跑 macOS 和 Linux 各三个 Python 版本的单元测试,再实跑一遍 README 里这套命令,加一次打包检查。 + +## License + +MIT diff --git a/skills/ops/carl-file-organizer/SKILL.md b/skills/ops/carl-file-organizer/SKILL.md new file mode 100644 index 0000000..7e41e7f --- /dev/null +++ b/skills/ops/carl-file-organizer/SKILL.md @@ -0,0 +1,90 @@ +--- +name: carl-file-organizer +description: 文件整理：整理下载目录或任意散乱文件夹,先出方案再执行。用户说「整理下载目录」「收拾一下 Downloads」「下载文件夹太乱了」「帮我归档文件」「文件太多了理一理」「整理一下这个文件夹」「organize my downloads」「clean up my downloads folder」「tidy this folder」「sort these files」「my downloads is a mess」时使用。它跑 carl-file-organizer 命令行工具,只对工具标为待判断且允许改道的文件做语义判断,任何移动、进废纸篓或删除都要用户在聊天里明确说执行才做。 +--- + +# Carl File Organizer + +你是这套工具的操作员,不是决策者。它自己已经用规则把大部分文件的去向定好了,你只处理剩下那部分需要看文件名和上下文才能判断的,把判断交回去给用户确认,最后由用户点头才落地。 + +## 什么时候用,什么时候不用 + +用户想整理某个文件夹里的散件,尤其是下载目录,用它。 +用户要清系统缓存、清 node_modules、找磁盘大户这类整机存储清理,那不是这个工具的活,直接说明白,不要硬套。 +用户只是问某个文件该放哪,不需要跑工具,直接回答就行。 + +## 第一步,确认工具在 + +跑 `carl-file-organizer --version`。不存在就把后面所有命令都换成这个前缀: + +``` +uvx --from git+https://github.com/LearnPrompt/carl-file-organizer carl-file-organizer <子命令...> +``` + +两条都不行,告诉用户怎么装,停下,不要自己想办法绕过去。 + +## 第二步,只读扫描 + +```bash +carl-file-organizer plan <目录> --lang zh +``` + +目录不写默认 `~/Downloads`,用户没指定就用默认值,不要自己猜别的路径。用户说英文目录名就把 `--lang` 换成 `en`。 + +这一步只读,不动任何文件。产物落在目标目录下的管理目录里:中文是 `<目录>/00_下载目录管理/`,英文是 `<目录>/00_File_Organizer/`,包含 `plan.json` 和 `report.html`。 + +## 第三步,读 plan.json,只转述不判断 + +plan.json 的结构里,`actions` 是全部候选动作,`groups` 是几选一的选择题,`approved_action_ids` 和 `overrides` 目前都是空的,等你和用户一起填。 + +先看 `summary`,按 `tier` 报一下总数:多少条常规移动(`routine`)、多少组压缩包配对和重复副本、多少条可再生产物、多少条静置未到期或者被判定为禁刀区 / 正被占用 / 被外部引用(这些统称 hold,一律不能批准,只读原因),多少条待判断。 + +`groups` 里的每一组都带着 `options`,每个选项自带一份 `action_ids`。你的活是把这些选项摆给用户看,让用户选,选完直接把那个选项的 `action_ids` 整体抄进批准名单,不要自己去 `actions` 里挑单条、不要自己拼凑,这是唯一被允许的映射方式。 + +## 第四步,只对待判断且可改道的做语义判断 + +`actions` 里 `reroutable` 为 true 的那些才是你能碰的待判断项,其余的一律原样转述,不要建议改动。 + +对每一条,只根据文件名、扩展名、大小、修改时间去判断它更适合放进 `plan.json` 的 `names` 里哪个 key,不要打开文件、不要读内容。名字里带着密钥、密码、令牌、证书、身份信息一类线索的,不要自己往敏感区搬,工具已经在敏感判断上有独立的一套规则,拿不准就原样跳过,交给用户自己在页面上处理。 + +拿不准分类的,不写判断,让它留在收件箱的待判断区,不要为了给出答案硬猜。 + +把你的判断整理成一张表给用户看,列出文件名、你判给哪个分区、一句理由,然后停下等用户回复,不要替用户拍板。 + +## 第五步,写批准文件 + +用户确认之后,把 `plan.json` 原样复制成同一个管理目录下的 `approved.json`,只改这几处,其余字段一个字都不能动: + +- `approved_action_ids`:用户同意的动作 id,包括他直接同意的常规移动、他在组里选的那个选项的 `action_ids`、以及他同意进废纸篓的可再生产物和冷存候选。永远不要往这里加 `kind` 是 `delete` 的 id,除非用户明确说要永久删除并且你确认过命令行会带 `--allow-permanent-delete`。 +- `overrides`:上一步整理出的 `{action_id, destination_key}` 列表,`destination_key` 必须原样取自 `plan.json` 的 `names`,不要自己编路径。 +- `approved_at`、`approved_by`:当前时间和 `agent:<你的名字>`。 + +工具会按 `id` 重新核对每一条,任何字段被改过都会被当成篡改直接拒绝执行,所以不要顺手改别的地方。 + +## 第六步,执行门禁 + +先跑一遍 dry-run,把输出原样贴给用户: + +```bash +carl-file-organizer apply <管理目录>/approved.json --dry-run +``` + +只有用户在聊天里明确说了执行、apply、动手、可以了这类话,才去掉 `--dry-run` 跑真的一遍。用户说看看、先别动、再想想,都不算授权,继续停在只读或 dry-run。 + +几条硬规矩,任何情况都不能破: + +- 永远不用 `rm`、`mv`、`trash`、`osascript` 这类命令自己动文件,所有动作必须走 `carl-file-organizer apply`。 +- 永远不在命令里加 `--allow-permanent-delete`,这个开关只能由用户自己在终端里加。 +- 工具拒绝执行、报错或者把某一批标为 refused 的,原样把那句话转告用户,不要自己想办法绕开或者重试出别的路径。 +- `overrides` 之外没有第二条改道通道,不要直接编辑 `destination`。 + +## 第七步,回报 + +按这个顺序说完就停,不要多说: + +- 这一轮做到了哪一步:只读盘点 / 已 dry-run / 已执行。 +- 移动了多少项,进废纸篓多少项,跳过和被拒绝的各多少项、为什么。 +- 待判断里你判了几条、用户改了几条、还剩几条没处理。 +- 没碰过的范围:禁刀区、静置期没到的、正被占用或被外部引用的。 +- `plan.json`、`approved.json`、`audit.jsonl` 的路径。 +- 一句话提醒:`carl-file-organizer undo` 能把这轮移动的都挪回去,`carl-file-organizer clear-tags` 能清掉页面留下的 Finder 标签。 diff --git a/skills/ops/carl-file-organizer/agents/openai.yaml b/skills/ops/carl-file-organizer/agents/openai.yaml new file mode 100644 index 0000000..c144120 --- /dev/null +++ b/skills/ops/carl-file-organizer/agents/openai.yaml @@ -0,0 +1,3 @@ +interface: + display_name: "文件整理 · Carl File Organizer" + short_description: "Plan file moves, get approval, then apply exactly that" diff --git a/skills/ops/carl-file-organizer/assets/report_template.html b/skills/ops/carl-file-organizer/assets/report_template.html new file mode 100644 index 0000000..7408889 --- /dev/null +++ b/skills/ops/carl-file-organizer/assets/report_template.html @@ -0,0 +1,899 @@ + + + + + + +__REPORT_TITLE__ + + + + + +__REPORT_BODY__ + + + + + + + diff --git a/skills/ops/carl-file-organizer/docs/plan-schema.md b/skills/ops/carl-file-organizer/docs/plan-schema.md new file mode 100644 index 0000000..67ccc37 --- /dev/null +++ b/skills/ops/carl-file-organizer/docs/plan-schema.md @@ -0,0 +1,295 @@ +# plan.json 契约（schema_version 2） + +审核页、本地服务、执行器、undo 和 Agent 都只读这一个文件，谁也不去猜结构。 +样例在仓库根的 `tests/fixtures/carl_file_organizer/plan-v2-sample.json`， +本文每一节都能在那份样例里找到对应的行； +`tests/test_cfo_fixture_contract.py` 按本文的约束逐条断言，改结构就要一起改这三处。 + +一句话记住整份文件：`actions` 是候选清单，`groups` 是几选一的题目， +`approved_action_ids` 是唯一的批准名单，`overrides` 是唯一的改道通道。 + +## 顶层字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `schema_version` | int | 固定 2。执行器见到别的值直接拒绝。 | +| `generator` | object | `{name, version}`，version 取自 `carl_file_organizer.__version__`。 | +| `created_at` | string | 计划时刻，带时区。跟随 `now`，所以同一个 `--now` 跑两遍出来的计划是一样的。 | +| `generated_at` | string | 真实生成时刻（墙钟），带时区。排查问题时看这一个。 | +| `now` | string | 静置期的计算基准。apply 阶段不重算，复现问题时用 `--now` 固定它。 | +| `lang` | string | `zh` 或 `en`，本次输出用的语言。 | +| `lang_source` | string | `flag` / `dotfile` / `env` / `existing-dirs` / `apple-languages` / `posix-locale` / `default`。 | +| `profile` | object | `{name, source, dotfile, hash}`。`source` 是 `builtin` 或 `builtin+dotfile`，`hash` 是合并后 profile 的短摘要，apply 时对不上只提醒不中断。 | +| `source_root` / `source_root_portable` | string | 绝对路径只出现在 plan.json 里；清单、日志、审计一律写 `$HOME/...` 形式。 | +| `managed_dir` / `managed_dir_portable` | string | 可见的管理目录，中文是 `00_下载目录管理`，英文是 `00_File_Organizer`。plan.json 和 report.html 都落在这里。 | +| `names` | object | 本次解析出来的 key → 相对路径，例如 `library.docs.pdf` → `20_知识库/文档资料/PDF`。页面和 Agent 直接显示这里的值，不必回头读 profile。 | +| `capabilities` | object | `{trash_backend, finder_tags, permanent_delete_offered}`。`trash_backend` 是 `finder`（macOS）/ `win32`（Windows 回收站）/ `gio`（Linux）/ `none`，为 none 时废纸篓选项不可批准。 | +| `scan_policy` | object | 本次扫描的边界与上限，写下来是为了让人知道哪些地方根本没看。除设计里那几项，还有 `allow_referenced` 与 `skipped{hidden,symlinks,partitions,artifacts}` 两个计数用的字段。 | +| `disk` | object | 卷容量与目标占用，移动同卷释放 0 字节这句话也写在里面。 | +| `mess` | object | 这个目录现在有多乱，见下。 | +| `summary` | object | 见下。 | +| `pinned` | array | 本次生效的常驻白名单，名字或 glob。 | +| `actions` | array | 候选动作，见下。 | +| `groups` | array | 成组的选择题，见下。 | +| `notes` | object 或 null | Agent 写的人话说明，见下。生成时是 null。 | +| `approved_action_ids` | array | 页面、服务和 Agent 唯一需要填的字段。 | +| `overrides` | array | 待判断项的改道，见下。 | +| `approved_at` | string 或 null | 批准时刻。 | +| `approved_by` | string 或 null | `html-static` / `serve` / `agent:<名字>`。 | + +`summary` 里 `entries` / `files` / `dirs` / `bytes` 数的是扫描到的顶层条目， +`by_tier` 和 `by_kind` 数的是 action 条数（同一个文件可能同时有移动和处置两条候选，所以两组数不相等是正常的）。 +`potential_reclaimable_bytes` 按 subject 去重后累加，只是个信息量，不代表建议； +`proposed_reclaimed_bytes` 在生成时永远是 0，执行完才按实际写进审计。 + +`by_color` 是 `{green|yellow|red: {actions, bytes}}`。`actions` 按条数算， +所有颜色加起来等于 `len(actions)`；`bytes` 是这些 action 的 `size_bytes` 之和， +跟 `by_tier` 一样会重复计入同一个文件（一个文件可以同时有一条绿色移动和一条黄色处置）， +它描述的是形状不是总量。 + +## mess：这个目录有多乱 + +```json +"mess": { + "score": 59, + "color": "yellow", + "color_source": "rule", + "counts": {"loose": 12, "aging": 1, "pending": 2, "groups": 3, "top_dirs": 3}, + "reason": {"zh": "...", "en": "..."} +} +``` + +| 字段 | 说明 | +|---|---| +| `score` | 0 到 100，越大越乱。只决定进度条长度，不决定颜色。 | +| `color` | `green` / `yellow` / `red`。 | +| `color_source` | `rule` 或 `notes`，谁定的这个颜色。 | +| `rule_color` | 只在被 notes 覆盖后出现，保存规则原本算出来的颜色。 | +| `line` | 只在 notes 给了一句话时出现。 | +| `counts` | `loose` 顶层散件数、`aging` 静置中、`pending` 待判断（按 subject 去重）、`groups` 组数、`top_dirs` 顶层目录数。 | +| `reason` | `{zh, en}`，规则给的一句话。 | + +阈值与分数公式在 `references/tiers.md`，代码里对应 `planner.MESS_THRESHOLDS`。 +Agent 可以在 notes.json 里覆盖 `color`，覆盖后 `color_source` 变成 `notes`， +规则的答案退到 `rule_color`。 + +## action 字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `id` | string | `sha256(subject_id \0 kind \0 destination_portable)[:16]`。同一个文件的移动、废纸篓、永久删除是三条不同 id。 | +| `subject_id` | string | `sha256(realpath \0 subject_kind \0 size \0 mtime_ns)[:16]`。执行前实时重算，对不上就跳过。 | +| `kind` | string | `move` / `trash` / `delete` / `hold`。 | +| `subject_kind` | string | `file` 或 `dir`。目录整个搬，不拆内部结构。 | +| `source` / `source_portable` | string | 源路径。 | +| `filename` | string | 条目名，目录移动后保持原名。 | +| `destination` / `destination_portable` | string 或 null | 目标路径；`trash` / `delete` / `hold` 为 null。 | +| `destination_key` | string 或 null | `names` 里的 key，页面按 key 显示分区名。 | +| `size_bytes` / `modified_ns` / `age_hours` | number | 体积、修改时间、已静置小时数。 | +| `rule` | string | 判定依据，取值见下表。 | +| `reason` | object | `{zh, en}`，一句人话说清为什么。 | +| `detail` | object 或 null | `{zh, en}`，补充证据，例如静置了多久。 | +| `confidence` | number | 0 到 1。未知扩展名是 0.55，扩展名规则命中是 0.98。 | +| `tier` | string | `routine` / `sensitive` / `regenerable` / `duplicate` / `cold` / `aging` / `pinned` / `forbidden` / `in_use` / `referenced`。 | +| `group_id` | string 或 null | 属于某个组时填组 id。 | +| `approvable` | bool | `hold` 恒为 false。执行器见到不可批准的 id 出现在名单里直接抛错。 | +| `reroutable` | bool | 只有 rule 属于 `unknown-ext` / `dir:pending` / `dir:derivative` 的移动才为 true，也只有它们能被 `overrides` 改道。 | +| `default_selected` | bool | 生成时一律 false，勾选由人来做。 | +| `requires` | array | `delete` 是 `["allow-permanent-delete"]`。 | +| `reclaims_bytes` | number | 移动是 0，废纸篓和永久删除是条目体积。 | +| `restore_method` | object 或 null | `{zh, en}`。移动给一条可直接执行的 mv，废纸篓写放回原处，永久删除写清楚不可恢复。 | +| `tags` | array | 扫描到的非临时 Finder 标签。 | +| `hints` | array | 只提示不改判定，例如 `git-repo`、`derivative-copy:-worktree-`。 | +| `color` | string | `green` / `yellow` / `red`。页面的按钮跟着它走，所以它由 kind、tier 与 reroutable 三个字段算出来，永远不手写。判定表在 `references/tiers.md`。 | +| `hold_reason` | object 或 null | hold 时必填，`{zh, en}`，静置类还带 `ready_at`，禁刀区带 `label`，常驻带 `pattern`（命中的那条白名单）。 | +| `guard` | object | 保底层的取证结果，见下。 | +| `dir_stats` | object 或 null | 目录才有，`{files, bytes, truncated}`，`truncated` 为 true 表示统计到上限就停了。 | + +### rule 取值 + +`ext:<后缀>`、`sensitive:exact` / `sensitive:suffix` / `sensitive:marker` / `sensitive:manual`、 +`dir:pending`、`dir:derivative`、`pair`、`tag:<标签>`、`name:<规则 id>`、`conflict`、 +`unknown-ext`、`dup:sha256`、`dup:name`、`regenerable:<目录名>`、`cold:large`、`cold:installer`、 +`hold:aging`、`hold:pinned`、`hold:forbidden`、`hold:in_use`、`hold:referenced`。 + +`conflict` 是目标位置已经被占了，条目转进重复待确认区并带上时间戳改名。 +时间戳插在扩展名前面，所以改完还是能双击打开： +`quarterly-report.pdf` → `00_收件箱/重复待确认/quarterly-report-2026-09-02_12-00-00.pdf`， +`release.tar.gz` → `release-2026-09-02_12-00-00.tar.gz`。 +目录和没有扩展名的条目保持 `<原名>-<时间戳>`。 + +### guard 字段 + +保底层查的是谁还在用这个路径，命中就把条目变成 hold，让人自己去改引用。 + +| 键 | 内容 | +|---|---| +| `open_by` | `[{command, pid}]`，来自 lsof。命中即 tier `in_use`，不可降级。 | +| `referenced_in` | `[{file, line, form}]`，来自 launchd、cron、shell 配置和用户自己加的引用文件。命中即 tier `referenced`，`plan --allow-referenced` 可降级成黄字提示。 | +| `incoming_symlinks` | 指进这个条目的软链路径。 | +| `shape` | 目录形态，例如 `git-dir`、`git-worktree`、`venv`、`dotenv`、`node-project`；文件是 `executable`、`shebang`。worktree 直接进 hold，其余只加提示。 | + +除这四个必填键，`guard` 还可能带两个可选键：`open_by_status` 为 `unknown` 表示这台机器上没有 lsof 或者 lsof 超时了， +「没人开着它」这句话只是查不出来而不是查过；`uncommitted` 是 git 仓库未提交的条目数。读的人按有则用、无则忽略处理。 + +lsof 报出来的 Spotlight 一类只读守护进程（`mdworker`、`quicklookd`、`fseventsd` 等）不算占用。 +刚下载完的文件几秒内就会被索引，把它们算进去等于整个下载目录永远动不了。 + +## group 字段 + +| 字段 | 说明 | +|---|---| +| `group_id` | 组 id，成员 action 的 `group_id` 指向它。 | +| `kind` | `pair`（压缩包与解压目录）、`duplicate`（重复文件）、`regenerable`（构建产物）。 | +| `tier` | pair 与 duplicate 都是 `duplicate`，regenerable 是 `regenerable`。 | +| `stem` | 归组用的词干。 | +| `evidence` | pair 是 `name`；duplicate 是 `sha256` 或 `name-pattern`；regenerable 是 `basename`。 | +| `sha256` | 只有内容验证过的重复组才有值。 | +| `members` | 成员的 `subject_id` 列表。 | +| `member_labels` | subject_id → 给人看的名字。 | +| `reason` | `{zh, en}`。 | +| `options` | 选项数组，见下。 | +| `color` | 恒为 `yellow`。组是一道选择题，不管选哪个都会有东西被处置，这就是需要人看一眼的定义。 | +| `default_choice` | 单选框的默认位置。压缩包配对默认留目录；名字像副本但内容没比对过的重复组默认都保留。 | +| `potential_bytes` | 采纳默认选项能腾出的字节数。 | + +`options[]` 里每项是 `{key, label{zh,en}, action_ids[], requires[]}`。 +`action_ids` 是组选择映射到批准名单的唯一通道，页面和 Agent 都不许自己推导； +里面出现的每个 id 都在 `actions` 里，而且 `approvable` 一定为 true。 +只要选项里包含永久删除，`requires` 就带 `allow-permanent-delete`。 + +压缩包配对给四个选项：都保留、留目录、留压缩包、留目录并永久删除压缩包。 +重复组给每个成员一个留它的选项外加一个都保留。 +构建产物组只有全部进废纸篓和全部永久删除两项，不选就等于保留。 + +## notes：Agent 写的人话 + +规则给的理由是每条规则一句固定话，读起来像通知。真正有用的是「这是什么、 +为什么在这儿、删了会怎样」，那要一个看过这个目录的 Agent 来写。 +notes 就是这一层，它只加文字，不改任何决策。 + +Agent 写一份 notes.json： + +```json +{ + "folder_line": "这个目录基本是上周做提案时下载的东西", + "mess": {"color": "yellow", "line": "看着乱，其实一半是同一批素材"}, + "actions": { + "0112233445566a7b": { + "what": "第三方给的合同 PDF 扫描件", + "why": "扩展名规则命中，归到文档资料", + "if_removed": "邮箱里还有原件，丢了能要回来" + } + }, + "groups": { + "dup-3ba91c04": {"what": "同一份素材导了两次", "why": "内容哈希一致", "if_removed": "留一份就够"} + } +} +``` + +然后跑： + +```bash +python3 scripts/organize.py build --notes notes.json +``` + +合并结果写回 `plan.notes`： + +| 字段 | 说明 | +|---|---| +| `schema` | 固定 1。 | +| `folder_line` | 整个目录一句话，或 null。 | +| `mess` | `{color?, line?}`，清理过的那份。 | +| `actions` | `action_id` → `{what?, why?, if_removed?}`，只留这三个键。 | +| `groups` | `group_id` → 同上。 | +| `warnings` | 合并时发现的问题，每条一句话。 | + +规矩就三条： + +1. **只收三个句子。** `what`、`why`、`if_removed` 之外的键一律丢掉并记一条 warning， + 所以 notes 里写 `"approvable": true` 或 `"destination": "..."` 不会有任何效果。 +2. **不认识的 id 只提醒。** 上一次扫描留下的 notes 拿来配这次的计划，多出来的条目 + 会被跳过并记一条 warning，不会让整份报告生不出来。 +3. **唯一能改的是目录颜色。** `mess.color` 会覆盖 `plan.mess.color`，规则的答案退到 + `rule_color`，`color_source` 记成 `notes`。条目的 `color` 一个字都不能改， + 因为页面的按钮跟着它走，那是安全边界。 + +单句超过 600 字会被截断，不会被丢掉。合并两次是替换不是叠加， +改完一句重跑一遍不会留下上一版。 + +Python 里直接用： + +```python +from carl_file_organizer.notes import merge_notes +plan = merge_notes(plan, notes) # 返回新的一份，原来的不动 +``` + +报告怎么渲染是 `build_report.py` 的事，这里只保证 plan.json 里有一份干净的 `notes`。 +没有 notes 时报告退回规则文案，一样能出。 + +## overrides + +```json +{"action_id": "0112233445566a7b", "destination_key": "work.db.sql"} +``` + +只对 `reroutable` 为 true 的 action 生效，`destination_key` 必须是 `names` 里已有的 key。 +执行器拿到 override 后自己重算目标路径并校验仍在 root 之内，任何写字面路径的尝试都会被拒。 +待判断区的下拉框和 Agent 的语义判断都走这一条路，别的改道方式一律没有。 + +## report.html 与绝对路径 + +plan.json 留着绝对路径，report.html 一个都不留。渲染前整份计划先过一遍 +`paths.strip_absolute()`：`source_root`、`managed_dir`、每条 action 的 `source` 与 +`destination` 这四类字段直接删掉，只留 `_portable` 双胞胎；`guard.referenced_in[].file`、 +`guard.referenced_in[].form` 与 `guard.incoming_symlinks` 都改写成 `$HOME` 形式；剩下任何 +还带着家目录前缀的字符串一律一起改写，没有例外，免得以后新加的字段又把用户名漏出去。 + +这条规则的理由很实在：审核页是能直接发给别人看的一页 HTML，绝对路径里带着账号名。 +serve 模式的 `/api/plan` 走同一条路，服务进程内存里的计划仍然是完整的，`/api/apply` +用的是内存那份，不受影响。 + +页面导出的 approved.json 因此也只有 `$HOME` 形式。apply 一侧用 +`paths.hydrate_absolute()` 按当前机器的家目录把四类字段展开回来，再做原有的全部校验， +所以 id 重算、越界检查、篡改检测都不用知道文件是从哪来的。 + +落盘的 token 只有 `$HOME` 一种，Windows 上也是。计划和报告是会跨机器传的东西， +一种拼法就只有一个解析器和一条隐私检查（整份文档一次 `startswith`）。 +`paths.expand_portable()` 两种都认，`%USERPROFILE%\Downloads` 和 +`$HOME/Downloads` 落到同一个地方；Windows 上给人看的时候由 +`paths.display_portable()` 换成 `%USERPROFILE%\...`，那只是渲染，不进文件。 + +## approved.json 与 plan.json 的关系 + +approved.json 就是 plan.json 本身，只多填三个字段：`approved_action_ids` 填上勾选的 id， +`overrides` 填上改道，`approved_at` 与 `approved_by` 记下谁在什么时候批的。 +`notes` 可以带着，执行器完全不读它。 +其余字段一个都不能改，执行器会按 `id` 重算并比对，改过 destination 的文件会被当成篡改直接拒绝。 + +执行前的校验按顺序是：schema 必须为 2；root 存在且不是软链； +批准的 id 必须都在已知清单里；同一个 subject 只能批准一条； +出现不可批准的 id 直接抛错；名单里有永久删除而命令行没加 `--allow-permanent-delete` 时整批拒绝，不是跳过。 + +apply 成功之后，用户传进来的 approved.json 会被复制一份到管理目录，叫 `approved-<时间戳>.json`。 +浏览器通常把导出文件下载到下载目录根部，不收走的话下次扫描就会被当成散件。 + +## 审计记录 + +`managed_dir/audit.jsonl` 每行一条，路径一律 `$HOME` 形式。 + +| 字段 | 说明 | +|---|---| +| `timestamp` | 这一条执行完的时刻。 | +| `run_id` | 一次 apply 的批次号，形如 `2026-09-02_10-20-11`。 | +| `action_id` / `subject_id` | 对应 plan 里的两个 id。 | +| `kind` | `move` / `trash` / `delete`。 | +| `source` / `destination` | `$HOME` 形式。 | +| `status` | `moved` / `trashed` / `deleted` / `dry-run` / `skipped` / `conflict` / `refused` / `failed`。 | +| `detail` | 状态的一句话解释。 | +| `size_bytes` | 条目体积。 | +| `manifest` | 这一批对应的 TSV 清单路径。 | +| `tagged` | 有没有成功打上临时标签。打标签失败只提醒，不影响这条动作的状态。 | +| `trash_backend` | 本次用的废纸篓后端。 | + +崩溃时以 audit 为准。清单会先落一行 PLANNED 再重写终态， +中途断电就停在 PLANNED，undo 和 status 都只信 audit，清单留给人读和查最后整理时间。 diff --git a/skills/ops/carl-file-organizer/docs/review-page.md b/skills/ops/carl-file-organizer/docs/review-page.md new file mode 100644 index 0000000..6a0d60e --- /dev/null +++ b/skills/ops/carl-file-organizer/docs/review-page.md @@ -0,0 +1,104 @@ +# 报告页视觉规范（report.html，v0.3 三色版） + +报告页是整个 skill 唯一给人看的界面。两个入口共用同一份模板：整理下载目录的页面读 plan.json，整机盘点的页面读 analysis.json。页面不加载任何外部资源，双击打开 file:// 也是零请求；同一份模板在 static 与 serve 两种模式下长得一样，只有底栏按钮的动作不同。 + +三个文件对应本文：`assets/report_template.html` 是模板（样式、脚本、顶部的 TEXT 文案表），`scripts/build_report.py` 负责把数据脱敏后渲染成正文并注入模板，仓库根 `tests/test_cfo_report_template.py` 逐条断言本文的约束。改样式先改模板，改文案只改 TEXT。 + +## 一句话原则 + +三色是核心，别的都让路。绿是放心做，黄是你看一眼，红是别动。强调色用墨色，页面里没有蓝，没有渐变，没有深色模式。危险动作靠文字说清楚，按钮权限跟颜色走。 + +## Token 表 + +| token | 值 | 用途 | +|---|---|---| +| `--bg` | `#f6f7f9` | 页面底、命令块底 | +| `--card` | `#ffffff` | 卡片底 | +| `--ink` | `#1d2129` | 正文、主按钮底、勾选框选中色，也是唯一的强调色 | +| `--muted` | `#86909c` | 次要文字、表头、路径 | +| `--line` | `#e5e6eb` | 1px 分隔线、次按钮描边 | +| `--green` / `--green-bg` | `#00b42a` / `#e8ffea` | 放心归位、可清 | +| `--yellow` / `--yellow-bg` | `#ff7d00` / `#fff7e8` | 你看一眼 | +| `--red` / `--red-bg` | `#f53f3f` / `#ffece8` | 别动 | +| `--other` | `#c9cdd4` | 磁盘条里的其他占用、无色的点 | +| `--radius` | `14px` | 卡片圆角；命令块、按钮、下拉用 `--radius-s` 8px | +| `--shadow` | `0 1px 3px rgba(0,0,0,.04), 0 8px 24px rgba(0,0,0,.04)` | 所有卡片同一个浅阴影 | + +字体：正文 `-apple-system, "PingFang SC", "Segoe UI", "Noto Sans CJK SC", "Helvetica Neue", Arial, sans-serif`，15px 行高 1.6；路径、命令、id 用 `ui-monospace, "SF Mono", Menlo, Consolas, "Noto Sans Mono CJK SC", monospace`，13px。标题只有页面标题 26px/700 和分区标题 18px/700，小节标题 13px 灰色大写字距。没有 web font。 + +## 布局（从上到下） + +``` +顶栏（sticky top，白底 92% 透明加轻微模糊，底边 1px） + 标题 · 目录或机器 · 扫描时间 · 模式；serve 时多一行「本页只在这台机器可见」 +页面标题 + 一句引导 +总览网格（两列，窄屏单列） + 整理页左：杂乱度大字 + 三色 pill + 计数小字 + 三色分段条 + 三色图例 + 一句话 + 盘点页左：每块磁盘一条（已用拆成 绿/黄/红/其他 四段加可用）+ 三色占比条 + 一句话 + 系统信息网格 + 右：执行建议卡「先做什么」，整理页按三色计数各一句，盘点页用 overview.priority；有绿项时给「勾选全部绿项」按钮 +Top 5 表（只有盘点页）：色点 · 大小 · 类型 · 名字 · 路径 · 一句话 +三色分区 green → yellow → red，每区一个
+ 分区头：色点 + 标题 + 计数 + 一句说明 + 工具（全部展开 / 本组全选 / serve 的执行本组 / 绿区的永久删除开关） + 每项一张
，左侧 4px 色条 +长期建议卡：盘点页 overview.long_term；整理页列静置中的到期时间，再放 undo 提示与三句固定说明 +底栏（sticky bottom）：已选计数 + 主按钮（static 导出，serve 执行） +``` + +三个分区永远都渲染，空的写一句本组没有条目。分段条的每一段用 `style="width:NN%"`，NN 按数量或字节向下取整，所以宽度之和永远不超过 100。 + +## 卡片 + +卡片头：勾选框（有权限才有）、名字、徽章（目录、敏感命名、待判断、去向、类型、组类型、可腾出）、大小、状态、折叠箭头。点头部空白处展开，点控件不展开。 + +卡片体：路径（等宽）、目录统计与标签、`这是什么 / 为什么这么判 / 动了会怎样 / 怎么恢复` 四行、命令块带复制按钮、控件行。 + +四行文案的来源：Agent 写了 notes 就用 notes（`notes.actions[id]`、`notes.groups[id]`、`notes.items[id]` 里的 `what / why / if_removed`），没写就回落到脚本的 `reason / detail / hold_reason / restore_method`（整理页）或条目自带的 `what / why / if_removed / restore`（盘点页）。空行不渲染。 + +## 按钮权限跟色走 + +| 颜色 | 整理页 | 盘点页 | +|---|---|---| +| green | 勾选执行；trash/delete 候选给单选，永久删除折叠在开关后并弹 confirm | 勾选；废纸篓单选默认选中，永久删除折叠在开关后 | +| yellow | 勾选移动或进废纸篓（都可逆）；配对与重复组的单选；待判断的去向下拉写进 overrides | 只有「移到废纸篓（可逆）」，且仅当 `trash_paths` 非空；serve 时多「打开所在位置」 | +| red | 没有任何 input，没有 `data-action="trash|delete"`；serve 时只有「打开所在位置」 | 同左 | + +判色顺序：数据里带 `color` 就用它；没有则整理页按 kind/tier 推（hold 或 forbidden/in_use/referenced/aging/pinned 是红，reroutable 或 duplicate/cold 是黄，其余绿；组里 regenerable 绿、pair/duplicate 黄），盘点页按 tier 和 `open_by`、`trash_paths` 推。 + +带永久删除的选项加 `needs-permanent` 类，`body[data-permanent="off"]` 时整体隐藏并 disabled；打开绿区的开关才显示。选中时弹 `confirm()`，文案含路径与大小，取消就退回默认选项。yellow 区永远不渲染永久删除选项，即便数据里有。 + +## 控件 data 属性 + +- 勾选框 `input.gn-pick`：整理页移动带 `data-action-id`，候选行带 `data-choice`；盘点页带 `data-item-id` 与 `data-choice`。 +- 单选 `input.gn-choice`：`data-action="trash|delete"`，永久删除再带 `data-kind="delete"`、`data-path`、`data-size`。 +- 组：`article[data-group-id]` 内 `input.gn-adopt` 与 `input.gn-option[data-action-ids]`，页面只按 `options[].action_ids` 映射，不自己推导。 +- 待判断下拉 `select.gn-dest`，`data-original` 记原去向，改了才写进 overrides。 +- 按钮 `data-action="copy"` 复制同块 ``；`data-action="reveal"` 带 `data-path`，只在 serve 渲染。 +- 状态回写按 `[data-ids]` 匹配 id，`.gn-status` 显示文字，`done` 时禁用该卡片全部控件。 + +## static 与 serve + +| | static | serve | +|---|---|---| +| 主按钮 | 整理页导出 `carl-file-organizer-approved.json`（plan 本身加 `approved_action_ids / overrides / approved_at / approved_by`）；盘点页导出 `carl-file-organizer-decisions.json`（`item_ids` 加每个 id 的 `actions` 映射） | 整理页 POST `/api/apply`，body `{action_ids, overrides, dry_run}`；盘点页 POST `/api/dispose`，按动作分两次，body `{item_ids, action: "trash"|"delete"}` | +| 其他按钮 | 无 | 停止服务（POST `/api/shutdown`）；整理页多「先试运行」；分区头多「执行本组已选」 | +| 打开所在位置 | 不渲染 | POST `/api/reveal`，body `{path_portable}` | +| token | 不嵌 | 嵌进 config，也接受 `?t=` 后立即从地址栏抹掉；请求头 `X-GN-Token` | +| 启动时 | 无请求 | 整理页 GET `/api/plan` 同步 `executed_ids` 与 `permanent_delete_enabled` | + +服务端响应 `{results: [{action_id 或 item_id, status, detail}]}`，status 取 `moved / trashed / deleted / dry-run / skipped / refused / failed`。 + +## 脱敏 + +页面和内嵌 JSON 里一个绝对路径都没有。渲染前整份数据过 `build_report.sanitize()`：`source_root`、`managed_dir`、每条 action 的 `source` 与 `destination`、每个 item 的 `path` 直接删掉，只留 `_portable` 双胞胎；其余任何字符串里出现家目录（按 plan 反推的 home，以及 `/Users/x`、`/home/x`、`C:\Users\x` 三种形态）都改写成 `$HOME`。测试扫整页不许出现 `/Users/`。 + +## 文案 + +所有固定文案都在模板顶部 `', html, re.S) + if not match: + raise ValueError("report_template.html has no report-text block") + return json.loads(match.group(1)) + + +def _lang_of(data: Dict[str, Any], lang: Optional[str]) -> str: + chosen = lang or data.get("lang") or "zh" + return chosen if chosen in LANGS else "zh" + + +class _T(object): + """``t("key", n=3)`` with fallback to English and then the key itself.""" + + def __init__(self, text: Dict[str, Dict[str, str]], lang: str) -> None: + self.text = text + self.lang = lang + + def __call__(self, key: str, **kw: Any) -> str: + table = self.text.get(self.lang) or {} + template = table.get(key) + if template is None: + template = (self.text.get("en") or {}).get(key) + if template is None: + template = key + if kw: + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + return template + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + + +def format_bytes(value: Union[int, float, None]) -> str: + size = float(value or 0) + for unit in ("B", "KB", "MB", "GB", "TB"): + if size < 1024 or unit == "TB": + return "{0:.1f} {1}".format(size, unit) if unit != "B" else "{0} B".format(int(size)) + size /= 1024 + return "{0:.1f} TB".format(size) + + +def _pick(value: Any, lang: str) -> str: + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or value.get("zh") or "") + if value is None: + return "" + return str(value) + + +def _e(value: Any) -> str: + return escape("" if value is None else str(value), quote=True) + + +def _embed_json(data: Any) -> str: + text = json.dumps(data, ensure_ascii=False) + return ( + text.replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("
", "\\u2028") + .replace("
", "\\u2029") + ) + + +def _display_time(value: Any) -> str: + text = str(value or "") + short = text + if len(text) >= 16 and text[10] == "T": + short = text[:10] + " " + text[11:16] + return '{1}'.format(_e(text), _e(short)) + + +def _color(value: Any, fallback: str) -> str: + return value if value in COLORS else fallback + + +def _widths(parts: List[Tuple[str, float]]) -> List[Tuple[str, int]]: + """Integer percentages, floored, so they never add up past 100.""" + + total = sum(max(0.0, float(v or 0)) for _, v in parts) + if total <= 0: + return [(k, 0) for k, _ in parts] + return [(k, int(max(0.0, float(v or 0)) * 100 // total)) for k, v in parts] + + +def _segbar(parts: List[Tuple[str, float]]) -> str: + segs = [] + for key, width in _widths(parts): + if width <= 0: + continue + segs.append(''.format(_e(key), width)) + return '
{0}
'.format("".join(segs)) + + +# -------------------------------------------------------------------------- +# kind detection, sanitising, notes +# -------------------------------------------------------------------------- + + +def detect_kind(data: Dict[str, Any]) -> str: + schema = str(data.get("schema") or "") + if "storage" in schema or "analysis" in schema: + return "storage" + if "actions" in data and data.get("schema_version") == 2: + return "organize" + if "items" in data or "disks" in data: + return "storage" + if "actions" in data: + return "organize" + raise ValueError("cannot tell whether this is a plan.json or an analysis.json") + + +def _home_of(data: Dict[str, Any], home: Optional[Union[str, Path]]) -> str: + if home is not None: + return str(home) + root = str(data.get("source_root") or "") + portable = str(data.get("source_root_portable") or "") + if root and portable == HOME_TOKEN: + return root + prefix = HOME_TOKEN + "/" + if root and portable.startswith(prefix): + relative = Path(portable[len(prefix):]).parts + whole = Path(root).parts + if len(whole) > len(relative) and whole[-len(relative):] == relative: + return str(Path(*whole[: -len(relative)])) + return str(Path.home()) + + +def _portable_text(text: str, base: str) -> str: + if base and base != "/": + if text == base: + text = HOME_TOKEN + else: + text = text.replace(base + "/", HOME_TOKEN + "/").replace(base + "\\", HOME_TOKEN + "\\") + for pattern in _HOME_PATTERNS: + text = pattern.sub(HOME_TOKEN, text) + return text + + +def _scrub(value: Any, base: str) -> Any: + if isinstance(value, dict): + return {k: _scrub(v, base) for k, v in value.items()} + if isinstance(value, list): + return [_scrub(item, base) for item in value] + if isinstance(value, str): + return _portable_text(value, base) + return value + + +def sanitize(data: Any, home: Optional[Union[str, Path]] = None) -> Any: + """A copy carrying no absolute path: the ``paths.strip_absolute`` rule, re-stated. + + ``source_root``, ``managed_dir`` and every action's ``source`` and + ``destination`` are dropped (their ``_portable`` twins stay); every other + string that starts with the home directory, or looks like one on macOS, + Linux or Windows, is rewritten to ``$HOME``. + """ + + if not isinstance(data, dict): + return data + base = _home_of(data, home) + out = copy.deepcopy(data) + for field in PLAN_ABSOLUTE_FIELDS: + out.pop(field, None) + for action in out.get("actions") or []: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + action.pop(field, None) + for item in out.get("items") or []: + if isinstance(item, dict): + item.pop("path", None) + return _scrub(out, base) + + +def merge_notes(data: Dict[str, Any], notes: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """``notes.json`` on top of whatever ``data["notes"]`` already carries.""" + + merged: Dict[str, Any] = copy.deepcopy(data.get("notes") or {}) if isinstance(data.get("notes"), dict) else {} + for key, value in (notes or {}).items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + inner = dict(merged[key]) + for k2, v2 in value.items(): + if isinstance(v2, dict) and isinstance(inner.get(k2), dict): + tmp = dict(inner[k2]) + tmp.update(v2) + inner[k2] = tmp + else: + inner[k2] = v2 + merged[key] = inner + else: + merged[key] = value + out = dict(data) + out["notes"] = merged + return out + + +def _note_for(notes: Dict[str, Any], table: str, ids: List[str], lang: str) -> Dict[str, str]: + bucket = notes.get(table) or {} + for i in ids: + entry = bucket.get(i) + if isinstance(entry, dict): + return {k: _pick(v, lang) for k, v in entry.items() if isinstance(v, (str, dict))} + return {} + + +# -------------------------------------------------------------------------- +# colours for a plan that predates them +# -------------------------------------------------------------------------- + + +def action_color(action: Dict[str, Any]) -> str: + given = action.get("color") + if given in COLORS: + return given + kind = action.get("kind", "move") + tier = action.get("tier", "routine") + if kind == "hold" or tier in RED_TIERS: + return "red" + if action.get("reroutable") or tier in YELLOW_TIERS: + return "yellow" + return "green" + + +def group_color(group: Dict[str, Any]) -> str: + given = group.get("color") + if given in COLORS: + return given + return "green" if group.get("kind") == "regenerable" else "yellow" + + +def _mess_fallback(plan: Dict[str, Any], t: _T) -> Dict[str, Any]: + actions = [a for a in (plan.get("actions") or []) if isinstance(a, dict)] + groups = plan.get("groups") or [] + summary = plan.get("summary") or {} + entries = int(summary.get("entries") or len({a.get("subject_id") for a in actions})) + pending = len([a for a in actions if a.get("kind") == "move" and a.get("reroutable")]) + holds = len([a for a in actions if a.get("kind") == "hold"]) + aging = len([a for a in actions if a.get("kind") == "hold" and a.get("tier") == "aging"]) + top_dirs = len({a.get("subject_id") for a in actions if a.get("subject_kind") == "dir" and a.get("kind") != "hold"}) + score = int(min(100, round(entries * 1.5 + pending * 5 + len(groups) * 6 + holds * 2))) + color = "green" if score < 35 else "yellow" if score < 70 else "red" + return { + "score": score, + "color": color, + "counts": {"scattered": entries, "pending": pending, "duplicate": len(groups), "aging": aging, "top_dirs": top_dirs}, + "reason": t("mess_fallback", entries=entries, pending=pending, groups=len(groups), holds=holds), + "source": "fallback", + } + + +def _by_color_fallback(plan: Dict[str, Any]) -> Dict[str, Dict[str, int]]: + out = {c: {"count": 0, "bytes": 0} for c in COLORS} + seen = set() + for a in plan.get("actions") or []: + if not isinstance(a, dict): + continue + sid = str(a.get("subject_id") or a.get("id")) + if sid in seen: + continue + seen.add(sid) + c = action_color(a) + out[c]["count"] += 1 + out[c]["bytes"] += int(a.get("size_bytes") or 0) + return out + + +def _normalise_by_color(raw: Any, fallback: Dict[str, Dict[str, int]]) -> Dict[str, Dict[str, int]]: + if not isinstance(raw, dict): + return fallback + out = {} + for c in COLORS: + v = raw.get(c) + if isinstance(v, dict): + count = v.get("count") + if count is None: + count = v.get("actions", v.get("items", 0)) + out[c] = {"count": int(count or 0), "bytes": int(v.get("bytes") or 0)} + elif isinstance(v, (int, float)): + out[c] = {"count": int(v), "bytes": fallback.get(c, {}).get("bytes", 0)} + else: + out[c] = {"count": 0, "bytes": 0} + return out + + +# -------------------------------------------------------------------------- +# shared fragments +# -------------------------------------------------------------------------- + + +def _copy_block(command: str, t: _T) -> str: + return ( + '
{0}' + '
' + ).format(_e(command), t("copy")) + + +def _reveal_button(path: str, mode: str, t: _T) -> str: + if mode != "serve" or not path: + return "" + return ''.format( + _e(path), t("reveal") + ) + + +def _dl(rows: List[Tuple[str, str]]) -> str: + body = "".join("
{0}
{1}
".format(_e(k), v) for k, v in rows if v) + return "
{0}
".format(body) if body else "" + + +def _card( + *, + ids: List[str], + subject: str, + color: str, + name: str, + badges: List[str], + size: Any, + head_extra: str = "", + pick: str = "", + body: str, + open_by_default: bool = False, + extra_attrs: str = "", +) -> str: + return ( + '
' + '
' + '{pick}' + '{name}{badges}{head_extra}' + '{size}' + '' + '' + "
" + '
{body}
' + "
" + ).format( + open=" open" if open_by_default else "", + color=_e(color), + ids=_e(" ".join(i for i in ids if i)), + sid=_e(subject), + extra=extra_attrs, + pick=pick, + name=_e(name), + badges="".join(badges), + head_extra=head_extra, + size=format_bytes(size), + body=body, + ) + + +def _section( + key: str, + color: str, + title: str, + desc: str, + cards: List[str], + *, + mode: str, + t: _T, + selectable: bool, + tools_extra: str = "", +) -> str: + tools = [] + if cards: + tools.append( + ''.format( + t("expand_all"), t("collapse_all") + ) + ) + if selectable and cards: + tools.append( + ''.format( + t("btn_select_all"), t("btn_clear_all") + ) + ) + if mode == "serve": + tools.append(''.format(t("btn_section"))) + tools.append(tools_extra) + body = "".join(cards) or '

{0}

'.format(t("empty")) + return ( + '
' + '

{title}' + '{count}

{desc}

' + '
{tools}
' + "{body}
" + ).format(color=_e(color), key=_e(key), title=_e(title), count=len(cards), desc=_e(desc), tools="".join(tools), body=body) + + +# -------------------------------------------------------------------------- +# organize page +# -------------------------------------------------------------------------- + + +def _guard_lines(action: Dict[str, Any], lang: str, t: _T) -> List[str]: + guard = action.get("guard") or {} + lines: List[str] = [] + hold = action.get("hold_reason") + if isinstance(hold, dict) and hold.get("ready_at"): + lines.append(_e(t("ready_at", time=hold.get("ready_at")))) + if isinstance(hold, dict) and hold.get("label"): + lines.append('{0}'.format(_e(hold.get("label")))) + open_by = guard.get("open_by") or [] + if open_by: + procs = ", ".join( + "{0} (pid {1})".format(_e(p.get("command")), _e(p.get("pid"))) if isinstance(p, dict) else _e(p) for p in open_by + ) + lines.append("{0}: {1}".format(t("open_by"), procs)) + refs = guard.get("referenced_in") or [] + if refs: + items = [] + for ref in refs: + if isinstance(ref, dict): + where = "{0}:{1}".format(ref.get("file", ""), ref.get("line", "")) + form = ref.get("form") + items.append(_e(where) + (" ({0})".format(_e(form)) if form else "")) + else: + items.append(_e(ref)) + lines.append("{0}: {1}".format(t("referenced_in"), "; ".join(items))) + links = guard.get("incoming_symlinks") or [] + if links: + lines.append("{0}: {1}".format(t("symlinks"), _e("; ".join(str(x) for x in links)))) + shape = guard.get("shape") or [] + if shape: + lines.append("{0}: {1}".format(t("shape"), _e(" · ".join(str(s) for s in shape)))) + return lines + + +def _action_badges(action: Dict[str, Any], t: _T) -> List[str]: + badges = [] + if action.get("subject_kind") == "dir": + badges.append('{0}'.format(t("dir_label"))) + if action.get("tier") == "sensitive": + badges.append('{0}'.format(t("sensitive_label"))) + return badges + + +def _action_extras(action: Dict[str, Any], t: _T) -> str: + bits = [] + stats = action.get("dir_stats") + if isinstance(stats, dict): + key = "dir_files_trunc" if stats.get("truncated") else "dir_files" + bits.append('
{0}
'.format(_e(t(key, n=stats.get("files", 0))))) + tags = [x for x in (action.get("tags") or []) if x] + if tags: + bits.append('
{0}: {1}
'.format(t("tags"), _e(" · ".join(str(x) for x in tags)))) + hints = [str(h) for h in (action.get("hints") or []) if h] + if hints: + bits.append('
{0}: {1}
'.format(t("hint"), _e(" · ".join(hints)))) + return "".join(bits) + + +def _explain(action: Dict[str, Any], note: Dict[str, str], lang: str, t: _T, *, what_default: str = "") -> str: + """what / why / if_removed: the note when the Agent wrote one, the rule text otherwise.""" + + what = note.get("what") or what_default or _pick(action.get("reason"), lang) + why = note.get("why") or _pick(action.get("detail"), lang) + hold = _pick(action.get("hold_reason"), lang) + if not why and hold and hold != what: + why = hold + restore = _pick(action.get("restore_method"), lang) + if_removed = note.get("if_removed") or "" + if not if_removed and action.get("kind") == "trash": + if_removed = t("restore_trash") + if not if_removed and action.get("kind") == "delete": + if_removed = t("restore_delete") + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(if_removed))] + if restore and action.get("kind") != "move": + rows.append((t("restore"), _e(restore))) + return _dl(rows) + + +def _path_line(path: str, t: _T) -> str: + return '
{0}
'.format(_e(path)) if path else "" + + +def _card_move(action: Dict[str, Any], color: str, names: Dict[str, str], notes: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + aid = str(action.get("id", "")) + note = _note_for(notes, "actions", [aid], lang) + dest = names.get(str(action.get("destination_key") or ""), "") + if not dest: + dest_path = action.get("destination_portable") or "" + dest = str(Path(str(dest_path)).parent) if dest_path else "" + path = str(action.get("source_portable") or "") + pending = bool(action.get("reroutable")) + controls_allowed = color != "red" + pick = "" + if controls_allowed: + pick = ''.format( + _e(aid), _e(action.get("filename")) + ) + badges = _action_badges(action, t) + if pending: + badges.append('{0}'.format(t("pending_label"))) + if dest: + badges.append('→ {0}'.format(_e(dest))) + body = [_path_line(path, t), _action_extras(action, t), _explain(action, note, lang, t)] + restore = _pick(action.get("restore_method"), lang) + if restore: + body.append('
{0}
'.format(t("restore"))) + body.append(_copy_block(restore, t)) + controls = [] + if pending and controls_allowed: + current = str(action.get("destination_key") or "") + options = [] + for key, value in sorted(names.items(), key=lambda kv: (str(kv[1]), kv[0])): + options.append(''.format(_e(key), " selected" if key == current else "", _e(value))) + controls.append( + ''.format( + t("change_dest"), _e(aid), _e(current), "".join(options) + ) + ) + controls.append(_reveal_button(path, mode, t)) + if any(controls): + body.append('
{0}
'.format("".join(controls))) + return _card( + ids=[aid], subject=str(action.get("subject_id", "")), color=color, name=str(action.get("filename") or ""), + badges=badges, size=action.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _card_candidate( + trash: Optional[Dict[str, Any]], delete: Optional[Dict[str, Any]], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T, + *, trash_available: bool, delete_offered: bool, +) -> str: + action = trash or delete or {} + sid = str(action.get("subject_id", "")) + ids = [str(a["id"]) for a in (trash, delete) if a and a.get("id")] + note = _note_for(notes, "actions", ids, lang) + name = "cand-" + (sid or "-".join(ids)) + path = str(action.get("source_portable") or "") + choices: List[str] = [] + if color != "red": + if trash is not None and trash_available and trash.get("approvable", True): + choices.append( + ''.format( + n=_e(name), v=_e(trash.get("id")), label=t("opt_trash") + ) + ) + if color == "green" and delete is not None and delete_offered and delete.get("approvable", True): + choices.append( + ''.format( + n=_e(name), v=_e(delete.get("id")), p=_e(path or delete.get("filename")), + s=_e(format_bytes(delete.get("size_bytes"))), label=t("opt_delete"), + ) + ) + pick = "" + if choices: + pick = ''.format(_e(name), _e(action.get("filename"))) + badges = _action_badges(action, t) + badges.append('{1}'.format(color, t("opt_trash") if trash is not None else t("opt_delete"))) + body = [_path_line(path, t), _action_extras(action, t), _explain(action, note, lang, t)] + controls = choices + [_reveal_button(path, mode, t)] + if any(controls): + body.append('
{0}
'.format("".join(controls))) + elif color != "red": + body.append('
{0}
'.format(t("keep_here"))) + return _card( + ids=ids, subject=sid, color=color, name=str(action.get("filename") or ""), badges=badges, + size=action.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _card_hold(action: Dict[str, Any], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + aid = str(action.get("id", "")) + note = _note_for(notes, "actions", [aid], lang) + path = str(action.get("source_portable") or "") + hold = _pick(action.get("hold_reason"), lang) + reason = _pick(action.get("reason"), lang) + what = note.get("what") or hold or reason + why = note.get("why") or (reason if reason != what else "") + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(note.get("if_removed") or ""))] + body = [_path_line(path, t), _action_extras(action, t), _dl(rows)] + guard = _guard_lines(action, lang, t) + if guard: + body.append("".join('
{0}
'.format(line) for line in guard)) + reveal = _reveal_button(path, mode, t) + if reveal: + body.append('
{0}
'.format(reveal)) + badges = _action_badges(action, t) + tier = str(action.get("tier") or "") + if tier: + badges.append('{0}'.format(_e(tier.replace("_", " ")))) + head_extra = '{0}'.format(_e(what)) if what else "" + return _card( + ids=[aid], subject=str(action.get("subject_id", "")), color=color, name=str(action.get("filename") or ""), + badges=badges, size=action.get("size_bytes"), head_extra=head_extra, body="".join(body), + ) + + +def _card_group( + group: Dict[str, Any], color: str, by_subject: Dict[str, Dict[str, Any]], by_id: Dict[str, Dict[str, Any]], + notes: Dict[str, Any], lang: str, mode: str, t: _T, *, trash_available: bool, delete_offered: bool, +) -> str: + gid = str(group.get("group_id", "")) + note = _note_for(notes, "groups", [gid], lang) + labels = group.get("member_labels") or {} + members = [str(m) for m in (group.get("members") or [])] + member_rows = [] + total = 0 + for sid in members: + action = by_subject.get(sid) or {} + label = labels.get(sid) or action.get("filename") or sid + source = action.get("source_portable") or "" + total += int(action.get("size_bytes") or 0) + member_rows.append( + '
  • {0}{1}{2}{3}
    • '.format( + _e(label), format_bytes(action.get("size_bytes")), + '{0}'.format(_e(source)) if source else "", + _reveal_button(str(source), mode, t), + ) + ) + all_ids: List[str] = [] + options_html = [] + default = group.get("default_choice") + radio_name = "grp-" + gid + for option in group.get("options") or []: + key = str(option.get("key", "")) + ids = [str(i) for i in (option.get("action_ids") or [])] + for i in ids: + if i not in all_ids: + all_ids.append(i) + needs_permanent = PERMANENT_FLAG in (option.get("requires") or []) or any( + by_id.get(i, {}).get("kind") == "delete" for i in ids + ) + if color == "red": + continue + if needs_permanent and (not delete_offered or color != "green"): + continue + if not trash_available and any(by_id.get(i, {}).get("kind") == "trash" for i in ids): + continue + attrs = [ + 'type="radio"', 'class="gn-option"', 'name="{0}"'.format(_e(radio_name)), 'value="{0}"'.format(_e(key)), + 'data-action-ids="{0}"'.format(_e(" ".join(ids))), + ] + if key == default: + attrs.append("checked") + attrs.append('data-default="1"') + if needs_permanent: + attrs.append('data-action="delete"') + attrs.append('data-kind="delete"') + attrs.append('data-path="{0}"'.format(_e(" / ".join(str(labels.get(m, m)) for m in members)))) + attrs.append('data-size="{0}"'.format(_e(format_bytes(group.get("potential_bytes"))))) + attrs.append("disabled") + else: + attrs.append('data-action="{0}"'.format("trash" if any(by_id.get(i, {}).get("kind") == "trash" for i in ids) else "move")) + options_html.append( + ''.format( + cls=" needs-permanent" if needs_permanent else "", attrs=" ".join(attrs), label=_e(_pick(option.get("label"), lang)) + ) + ) + pick = "" + if options_html: + pick = ''.format(t("adopt_group")) + potential = group.get("potential_bytes") or 0 + badges = ['{0}'.format(_e(str(group.get("kind") or "group")))] + if potential: + badges.append('{1}'.format(color, _e(t("potential", bytes=format_bytes(potential))))) + reason = _pick(group.get("reason"), lang) + rows = [ + (t("what"), _e(note.get("what") or reason)), + (t("why"), _e(note.get("why") or (_e(group.get("evidence") or "") if note.get("what") else ""))), + (t("if_removed"), _e(note.get("if_removed") or "")), + ] + body = ['
      {0}
    '.format("".join(member_rows)), _dl(rows)] + if options_html: + body.append('
    {0}
    '.format("".join(options_html))) + else: + body.append('
    {0}
    '.format(t("keep_here"))) + title = " · ".join(str(labels.get(m, m)) for m in members) + return _card( + ids=all_ids, subject=gid, color=color, name=title, badges=badges, size=total, pick=pick, body="".join(body), + extra_attrs=' data-group-id="{0}"'.format(_e(gid)), + ) + + +def _organize_sections(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> Tuple[str, Dict[str, int]]: + actions: List[Dict[str, Any]] = [a for a in (plan.get("actions") or []) if isinstance(a, dict)] + groups: List[Dict[str, Any]] = [g for g in (plan.get("groups") or []) if isinstance(g, dict)] + names: Dict[str, str] = {str(k): str(v) for k, v in (plan.get("names") or {}).items()} + notes: Dict[str, Any] = plan.get("notes") or {} + capabilities = plan.get("capabilities") or {} + trash_available = capabilities.get("trash_backend", "finder") != "none" + delete_offered = bool(capabilities.get("permanent_delete_offered", True)) + + by_id = {str(a.get("id")): a for a in actions} + by_subject: Dict[str, Dict[str, Any]] = {} + for a in actions: + by_subject.setdefault(str(a.get("subject_id", a.get("id"))), a) + grouped_ids = set() + for g in groups: + for option in g.get("options") or []: + for i in option.get("action_ids") or []: + grouped_ids.add(str(i)) + + buckets: Dict[str, List[Tuple[int, str]]] = {c: [] for c in COLORS} + candidates: Dict[str, Dict[str, Optional[Dict[str, Any]]]] = {} + order: List[str] = [] + + for a in actions: + kind = a.get("kind", "move") + aid = str(a.get("id")) + color = action_color(a) + if kind == "hold": + buckets[color].append((4, _card_hold(a, color, notes, lang, mode, t))) + continue + if aid in grouped_ids: + continue + if kind == "move": + rank = 1 if a.get("reroutable") else 0 + buckets[color].append((rank, _card_move(a, color, names, notes, lang, mode, t))) + continue + if kind in ("trash", "delete"): + sid = str(a.get("subject_id", aid)) + if sid not in candidates: + candidates[sid] = {"trash": None, "delete": None} + order.append(sid) + candidates[sid][kind] = a + continue + buckets[color].append((0, _card_move(a, color, names, notes, lang, mode, t))) + + for g in groups: + color = group_color(g) + buckets[color].append( + (2, _card_group(g, color, by_subject, by_id, notes, lang, mode, t, trash_available=trash_available, delete_offered=delete_offered)) + ) + for sid in order: + trash, delete = candidates[sid]["trash"], candidates[sid]["delete"] + color = action_color(trash or delete or {}) + buckets[color].append( + (3, _card_candidate(trash, delete, color, notes, lang, mode, t, trash_available=trash_available, delete_offered=delete_offered)) + ) + + permanent = "" + if delete_offered: + enabled = capabilities.get("permanent_delete_enabled", True) + disabled = " disabled" if mode == "serve" and not enabled else "" + note = t("permanent_not_enabled") if (mode == "serve" and not enabled) else t("permanent_static_note") + permanent = ''.format( + disabled, t("toggle_permanent"), _e(note) + ) + + counts = {c: len(buckets[c]) for c in COLORS} + html = "".join( + [ + _section("green", "green", t("sec_green"), t("sec_green_desc"), [h for _, h in sorted(buckets["green"], key=lambda x: x[0])], mode=mode, t=t, selectable=True, tools_extra=permanent), + _section("yellow", "yellow", t("sec_yellow"), t("sec_yellow_desc"), [h for _, h in sorted(buckets["yellow"], key=lambda x: x[0])], mode=mode, t=t, selectable=True), + _section("red", "red", t("sec_red"), t("sec_red_desc"), [h for _, h in sorted(buckets["red"], key=lambda x: x[0])], mode=mode, t=t, selectable=False), + ] + ) + return html, counts + + +def _organize_overview(plan: Dict[str, Any], lang: str, mode: str, t: _T, counts: Dict[str, int]) -> str: + notes = plan.get("notes") or {} + mess = plan.get("mess") if isinstance(plan.get("mess"), dict) else _mess_fallback(plan, t) + mess_color = _color(mess.get("color"), "yellow") + score = mess.get("score") + by_color = _normalise_by_color((plan.get("summary") or {}).get("by_color"), _by_color_fallback(plan)) + line = _pick((notes.get("mess") or {}).get("line"), lang) if isinstance(notes.get("mess"), dict) else "" + if isinstance(notes.get("mess"), dict) and notes["mess"].get("color") in COLORS: + mess_color = notes["mess"]["color"] + if not line: + line = _pick(mess.get("reason"), lang) + chips = [] + for key, value in (mess.get("counts") or {}).items(): + label = t("count_" + str(key)) + chips.append("{0} {1}".format(_e(label if label != "count_" + str(key) else key), _e(value))) + legend = "".join( + '{1} {2} · {3}'.format( + c, t("legend_" + c), by_color[c]["count"], format_bytes(by_color[c]["bytes"]) + ) + for c in COLORS + ) + big = '{0}/100'.format(_e(score)) if isinstance(score, (int, float)) else "" + overview = ( + '

    {title}

    ' + '
    {big}{label}
    ' + '
    {chips}
    ' + "{bar}" + '
    {legend}
    ' + '

    {line}

    ' + "
    " + ).format( + title=t("mess_title"), big=big, color=mess_color, label=t("mess_" + mess_color), chips="".join(chips), + bar=_segbar([(c, by_color[c]["count"]) for c in COLORS]), legend=legend, line=_e(line), + ) + folder_line = _pick(notes.get("folder_line"), lang) + advice_items = [] + if counts["green"]: + advice_items.append("
  • {0}
    • ".format(_e(t("advice_green", n=counts["green"])))) + if counts["yellow"]: + advice_items.append("
  • {0}
    • ".format(_e(t("advice_yellow", n=counts["yellow"])))) + if counts["red"]: + advice_items.append("
  • {0}
    • ".format(_e(t("advice_red", n=counts["red"])))) + if not advice_items: + advice_items.append("
  • {0}
    • ".format(_e(t("advice_none")))) + tools = "" + if counts["green"]: + tools = '
    '.format(t("btn_pick_green")) + advice = ( + '

    {title}

    {folder}
      {items}
    {tools}
    ' + ).format( + title=t("advice_title"), folder='

    {0}

    '.format(_e(folder_line)) if folder_line else "", + items="".join(advice_items), tools=tools, + ) + return '
    {0}{1}
    '.format(overview, advice) + + +def _organize_longterm(plan: Dict[str, Any], lang: str, t: _T) -> str: + aging = [] + for a in plan.get("actions") or []: + if not isinstance(a, dict) or a.get("kind") != "hold": + continue + hold = a.get("hold_reason") if isinstance(a.get("hold_reason"), dict) else {} + if hold.get("ready_at"): + aging.append("
  • {0}
    • ".format(_e(t("aging_line", name=a.get("filename"), time=hold.get("ready_at"))))) + aging_html = "" + if aging: + aging_html = "

    {0}

      {1}
    ".format(t("aging_title"), "".join(aging)) + capabilities = plan.get("capabilities") or {} + notes = [t("undo_hint"), t("pledge"), t("pledge_more"), t("disk_note")] + if capabilities.get("trash_backend", "finder") == "none": + notes.append(t("no_trash_backend")) + return '
    {0}

    {1}

    {2}
    '.format( + aging_html, t("longterm_title"), "".join("

    {0}

    ".format(_e(n)) for n in notes) + ) + + +def _organize_bar(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + cells = [ + (t("bar_dir"), '{0}'.format(_e(plan.get("source_root_portable") or ""))), + (t("bar_time"), '{0}'.format(_display_time(plan.get("created_at") or plan.get("now") or ""))), + (t("bar_mode"), '{0}'.format(t("mode_" + mode))), + ] + return _bar(t("title_organize"), cells, mode, t) + + +def _bar(brand: str, cells: List[Tuple[str, str]], mode: str, t: _T) -> str: + items = "".join('{0}{1}'.format(_e(k), v) for k, v in cells) + serve = '
    {0}
    '.format(t("serve_note")) if mode == "serve" else "" + return '
    {0}{1}
    {2}
    '.format( + _e(brand), items, serve + ) + + +def _footer(kind: str, mode: str, t: _T) -> str: + if mode == "serve": + buttons = [''.format(t("btn_shutdown"))] + if kind == "organize": + buttons.append(''.format(t("btn_dry"))) + buttons.append(''.format(t("btn_apply"))) + hint = "" + else: + label = t("btn_export") if kind == "organize" else t("btn_export_decisions") + buttons = [''.format(label)] + hint = '
    {0}
    '.format(t("footer_static_hint")) + return ( + '
    ' + '
    {hint}
    ' + '
    {buttons}
    ' + "
    " + ).format(hint=hint, buttons="".join(buttons)) + + +def render_organize_body(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + sections, counts = _organize_sections(plan, lang, mode, t) + return "".join( + [ + _organize_bar(plan, lang, mode, t), + '
    ', + '

    {0}

    {1}

    '.format(t("heading_organize"), t("lede_organize")), + _organize_overview(plan, lang, mode, t, counts), + sections, + _organize_longterm(plan, lang, t), + '

    {0}

    '.format(t("print_hint")), + "
    ", + _footer("organize", mode, t), + ] + ) + + +# -------------------------------------------------------------------------- +# storage page +# -------------------------------------------------------------------------- + + +def item_color(item: Dict[str, Any]) -> str: + given = item.get("color") + if given in COLORS: + return given + tier = str(item.get("tier") or "") + if tier in RED_TIERS or item.get("open_by"): + return "red" + if tier in YELLOW_TIERS or not item.get("trash_paths"): + return "yellow" + return "green" + + +def _kind_label(kind: Any, t: _T) -> str: + key = "kind_" + str(kind or "other") + label = t(key) + return str(kind) if label == key else label + + +def _card_item(item: Dict[str, Any], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T, *, delete_offered: bool) -> str: + iid = str(item.get("id", "")) + note = _note_for(notes, "items", [iid], lang) + path = str(item.get("path_portable") or "") + trash_paths = [str(p) for p in (item.get("trash_paths") or []) if p] + action = str(item.get("action") or ("trash" if trash_paths else "keep")) + name = "item-" + iid + choices: List[str] = [] + if color != "red" and trash_paths and action in ("trash", "delete"): + choices.append( + ''.format( + n=_e(name), v=_e(iid), label=t("opt_trash") + ) + ) + if color == "green" and delete_offered: + choices.append( + ''.format( + n=_e(name), v=_e(iid), p=_e(path), s=_e(format_bytes(item.get("size_bytes"))), label=t("opt_delete") + ) + ) + pick = "" + if choices: + pick = ''.format( + _e(name), _e(iid), _e(item.get("name")) + ) + badges = ['{0}'.format(_e(_kind_label(item.get("kind"), t)))] + if item.get("open_by"): + badges.append('{0}'.format(t("open_by"))) + what = note.get("what") or _pick(item.get("what"), lang) or _pick(item.get("note"), lang) + why = note.get("why") or _pick(item.get("why"), lang) + if_removed = note.get("if_removed") or _pick(item.get("if_removed"), lang) + restore = _pick(item.get("restore"), lang) + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(if_removed)), (t("restore"), _e(restore))] + body = [_path_line(path, t), _dl(rows)] + extra_note = _pick(item.get("note"), lang) + if extra_note and extra_note != what: + body.append('
    {0}: {1}
    '.format(t("hint"), _e(extra_note))) + if trash_paths and (len(trash_paths) > 1 or trash_paths[0] != path): + body.append( + '
    {0}
      {1}
    '.format( + t("trash_paths"), "".join('
  • {0}
    • '.format(_e(p)) for p in trash_paths) + ) + ) + kill = [str(p) for p in (item.get("kill_processes") or []) if p] + if kill: + body.append('
    {0}: {1}
    '.format(t("kill_processes"), _e(" · ".join(kill)))) + open_by = item.get("open_by") or [] + if open_by: + procs = ", ".join( + "{0} (pid {1})".format(_e(p.get("command")), _e(p.get("pid"))) if isinstance(p, dict) else _e(p) for p in open_by + ) + body.append('
    {0}: {1}
    '.format(t("open_by"), procs)) + commands = [str(c) for c in (item.get("commands") or []) if c] + if commands: + body.append('
    {0}
    '.format(t("commands"))) + body.extend(_copy_block(c, t) for c in commands) + controls = choices + [_reveal_button(path, mode, t)] + if any(controls): + body.append('
    {0}
    '.format("".join(controls))) + return _card( + ids=[iid], subject=iid, color=color, name=str(item.get("name") or path), badges=badges, + size=item.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _storage_sections(data: Dict[str, Any], lang: str, mode: str, t: _T) -> Tuple[str, Dict[str, int]]: + items = [i for i in (data.get("items") or []) if isinstance(i, dict)] + notes = data.get("notes") or {} + capabilities = data.get("capabilities") or {} + delete_offered = bool(capabilities.get("permanent_delete_offered", True)) + buckets: Dict[str, List[str]] = {c: [] for c in COLORS} + for item in items: + color = item_color(item) + buckets[color].append(_card_item(item, color, notes, lang, mode, t, delete_offered=delete_offered)) + permanent = "" + if delete_offered: + enabled = capabilities.get("permanent_delete_enabled", True) + disabled = " disabled" if mode == "serve" and not enabled else "" + note = t("permanent_not_enabled") if (mode == "serve" and not enabled) else t("permanent_static_note") + permanent = ''.format( + disabled, t("toggle_permanent"), _e(note) + ) + counts = {c: len(buckets[c]) for c in COLORS} + html = "".join( + [ + _section("green", "green", t("st_sec_green"), t("st_sec_green_desc"), buckets["green"], mode=mode, t=t, selectable=True, tools_extra=permanent), + _section("yellow", "yellow", t("st_sec_yellow"), t("st_sec_yellow_desc"), buckets["yellow"], mode=mode, t=t, selectable=True), + _section("red", "red", t("st_sec_red"), t("st_sec_red_desc"), buckets["red"], mode=mode, t=t, selectable=False), + ] + ) + return html, counts + + +def _disk_segments(disk: Dict[str, Any], items: List[Dict[str, Any]], primary: bool) -> List[Tuple[str, float]]: + mount = str(disk.get("mount") or "") + colored = {c: 0.0 for c in COLORS} + for item in items: + where = str(item.get("disk") or "") + if where == mount or (not where and primary): + colored[item_color(item)] += float(item.get("size_bytes") or 0) + used = float(disk.get("used_bytes") or 0) + total = float(disk.get("total_bytes") or 0) + free = float(disk.get("free_bytes") or max(0.0, total - used)) + other = max(0.0, used - sum(colored.values())) + return [("green", colored["green"]), ("yellow", colored["yellow"]), ("red", colored["red"]), ("other", other), ("free", free)] + + +def _storage_overview(data: Dict[str, Any], lang: str, mode: str, t: _T, counts: Dict[str, int]) -> str: + items = [i for i in (data.get("items") or []) if isinstance(i, dict)] + disks = [d for d in (data.get("disks") or []) if isinstance(d, dict)] + machine = data.get("machine") or {} + summary = data.get("summary") or {} + scan = data.get("scan") or {} + overview = data.get("overview") or {} + notes = data.get("notes") or {} + by_color = _normalise_by_color(summary.get("by_color"), _by_color_items(items)) + + disk_html = [] + for index, disk in enumerate(disks): + primary = bool(disk.get("primary")) or (index == 0 and not any(d.get("primary") for d in disks)) + used = disk.get("used_bytes") + pct = disk.get("used_percent") + pct_text = "{0:.0f}%".format(float(pct)) if isinstance(pct, (int, float)) else "" + disk_html.append( + '
    {name} {mount}' + '{used} · {pct}
    {bar}' + '
    {lg}{ly}' + '{lr}{lo}{lf}
    '.format( + name=_e(disk.get("name") or disk.get("mount")), mount=_e(disk.get("mount") or ""), + used=_e(t("disk_used", used=format_bytes(used), total=format_bytes(disk.get("total_bytes")))), pct=pct_text, + bar=_segbar(_disk_segments(disk, items, primary)), + lg=t("legend_green"), ly=t("legend_yellow"), lr=t("legend_red"), lo=t("seg_other"), + lf=_e(t("disk_free", free=format_bytes(disk.get("free_bytes")))), + ) + ) + sys_rows = [ + (t("sys_host"), machine.get("hostname")), + (t("sys_os"), " ".join(str(x) for x in (machine.get("os"), machine.get("arch")) if x)), + (t("sys_cpu"), machine.get("cpu")), + (t("sys_mem"), format_bytes(machine.get("memory_bytes")) if machine.get("memory_bytes") else None), + (t("sys_uptime"), t("sys_uptime_days", n=machine.get("uptime_days")) if machine.get("uptime_days") is not None else None), + (t("sys_scanned"), format_bytes(summary.get("scanned_bytes")) if summary.get("scanned_bytes") else None), + (t("sys_reclaim"), format_bytes(summary.get("reclaimable_bytes")) if summary.get("reclaimable_bytes") else None), + (t("sys_elapsed"), "{0:.0f} s".format(float(scan.get("elapsed_s"))) if isinstance(scan.get("elapsed_s"), (int, float)) else None), + (t("sys_denied"), ", ".join(str(x) for x in (scan.get("denied") or [])) or None), + ] + sys_html = "".join( + '
    {0}{1}
    '.format(_e(k), _e(v)) for k, v in sys_rows if v + ) + legend = "".join( + '{1} {2} · {3}'.format( + c, t("legend_" + c), by_color[c]["count"], format_bytes(by_color[c]["bytes"]) + ) + for c in COLORS + ) + headline = _pick(notes.get("headline"), lang) or _pick(overview.get("headline"), lang) + left = ( + '

    {dt}

    {disks}' + '

    {lt}

    {bar}
    {legend}
    {line}' + '

    {st}

    {sys}
    ' + ).format( + dt=t("disk_title"), disks="".join(disk_html), lt=t("color_split"), bar=_segbar([(c, by_color[c]["bytes"]) for c in COLORS]), + legend=legend, line='

    {0}

    '.format(_e(headline)) if headline else "", st=t("sys_title"), sys=sys_html, + ) + priority = overview.get("priority") or [] + li = [] + for entry in priority: + text = _pick(entry.get("text") if isinstance(entry, dict) else entry, lang) + if text: + li.append("
  • {0}
    • ".format(_e(text))) + if not li: + for c in COLORS: + if counts[c]: + li.append("
  • {0}
    • ".format(_e(t("advice_" + c, n=counts[c])))) + tools = "" + if counts["green"]: + tools = '
    '.format(t("btn_pick_green")) + right = '

    {0}

      {1}
    {2}
    '.format(t("advice_title"), "".join(li), tools) + return '
    {0}{1}
    '.format(left, right) + + +def _by_color_items(items: List[Dict[str, Any]]) -> Dict[str, Dict[str, int]]: + out = {c: {"count": 0, "bytes": 0} for c in COLORS} + for item in items: + c = item_color(item) + out[c]["count"] += 1 + out[c]["bytes"] += int(item.get("size_bytes") or 0) + return out + + +def _storage_top5(data: Dict[str, Any], lang: str, t: _T) -> str: + top = [x for x in (data.get("top5") or data.get("top") or []) if isinstance(x, dict)] + if not top: + return "" + by_id = {str(i.get("id")): i for i in (data.get("items") or []) if isinstance(i, dict)} + rows = [] + for entry in top[:5]: + item = by_id.get(str(entry.get("id")), {}) + color = _color(entry.get("color"), item_color(item) if item else "yellow") + note = _pick(entry.get("note"), lang) or _pick(item.get("what"), lang) + rows.append( + '{size}{kind}' + '{name}{path}{note}'.format( + c=color, size=format_bytes(entry.get("size_bytes") or item.get("size_bytes")), + kind=_e(_kind_label(entry.get("kind") or item.get("kind"), t)), name=_e(entry.get("name") or item.get("name")), + path=_e(entry.get("path_portable") or item.get("path_portable") or ""), note=_e(note), + ) + ) + head = "".join("{0}".format(_e(t(k))) for k in ("th_color", "th_size", "th_kind", "th_name", "th_path", "th_note")) + return '

    {0}

    {1}{2}
    '.format( + t("top5_title"), head, "".join(rows) + ) + + +def _storage_longterm(data: Dict[str, Any], lang: str, t: _T) -> str: + overview = data.get("overview") or {} + notes = data.get("notes") or {} + lines = notes.get("long_term") if isinstance(notes.get("long_term"), list) else overview.get("long_term") or [] + li = "".join("
  • {0}
    • ".format(_e(_pick(x, lang))) for x in lines if _pick(x, lang)) + if not li: + return "" + return '

    {0}

      {1}
    '.format(t("longterm_title"), li) + + +def _storage_bar(data: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + machine = data.get("machine") or {} + cells = [ + (t("bar_machine"), '{0}'.format(_e(" · ".join(str(x) for x in (machine.get("hostname"), machine.get("os")) if x)))), + (t("bar_time"), '{0}'.format(_display_time(data.get("created_at") or ""))), + (t("bar_mode"), '{0}'.format(t("mode_" + mode))), + ] + return _bar(t("title_storage"), cells, mode, t) + + +def render_storage_body(data: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + sections, counts = _storage_sections(data, lang, mode, t) + return "".join( + [ + _storage_bar(data, lang, mode, t), + '
    ', + '

    {0}

    {1}

    '.format(t("heading_storage"), t("lede_storage")), + _storage_overview(data, lang, mode, t, counts), + _storage_top5(data, lang, t), + sections, + _storage_longterm(data, lang, t), + '

    {0}

    '.format(t("print_hint")), + "
    ", + _footer("storage", mode, t), + ] + ) + + +# -------------------------------------------------------------------------- +# public API +# -------------------------------------------------------------------------- + + +def render( + data: Dict[str, Any], + *, + mode: str = "static", + token: str = "", + lang: Optional[str] = None, + notes: Optional[Dict[str, Any]] = None, + kind: Optional[str] = None, + template_path: Optional[Union[str, Path]] = None, + home: Optional[Union[str, Path]] = None, +) -> str: + """The finished page for ``data`` (a plan.json or an analysis.json dict).""" + + if mode not in MODES: + raise ValueError("mode must be one of {0}".format(", ".join(MODES))) + kind = kind or detect_kind(data) + if kind not in KINDS: + raise ValueError("kind must be one of {0}".format(", ".join(KINDS))) + template = load_template(template_path) + text = template_text(template) + clean = sanitize(merge_notes(data, notes), home) + lang = _lang_of(clean, lang) + t = _T(text, lang) + if mode != "serve": + token = "" + capabilities = dict(clean.get("capabilities") or {}) + config = { + "kind": kind, + "mode": mode, + "token": token, + "lang": lang, + "capabilities": capabilities, + "generator": "build_report.py {0}".format(__version__), + } + body = render_organize_body(clean, lang, mode, t) if kind == "organize" else render_storage_body(clean, lang, mode, t) + mapping = { + "LANG": "zh-CN" if lang == "zh" else "en", + "TITLE": _e(t("title_" + kind)), + "KIND": kind, + "MODE": mode, + "DATA": _embed_json(clean), + "CONFIG": _embed_json(config), + "BODY": body, + } + return re.sub(r"__REPORT_(LANG|TITLE|KIND|MODE|DATA|CONFIG|BODY)__", lambda m: mapping[m.group(1)], template) + + +def _read_json(path: Union[str, Path]) -> Dict[str, Any]: + with open(str(path), "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Render the three-colour report page.") + parser.add_argument("input", help="plan.json or analysis.json") + parser.add_argument("--notes", help="notes.json written by the Agent (what / why / if_removed per id)") + parser.add_argument("-o", "--output", help="where to write report.html (default: next to the input)") + parser.add_argument("--mode", choices=MODES, default="static") + parser.add_argument("--token", default="", help="serve-mode token embedded in the page") + parser.add_argument("--lang", choices=LANGS, default=None) + parser.add_argument("--kind", choices=KINDS, default=None, help="override the auto-detected page kind") + args = parser.parse_args(argv) + + data = _read_json(args.input) + notes = _read_json(args.notes) if args.notes else None + html = render(data, mode=args.mode, token=args.token, lang=args.lang, notes=notes, kind=args.kind) + output = Path(args.output) if args.output else Path(args.input).resolve().parent / "report.html" + output.write_text(html, encoding="utf-8") + sys.stdout.write("{0}\n".format(output)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py new file mode 100644 index 0000000..a4bf366 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py @@ -0,0 +1,3 @@ +"""Carl File Organizer: human-first file organization.""" + +__version__ = "0.3.0" diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py new file mode 100644 index 0000000..a049ad7 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py new file mode 100644 index 0000000..31f7ec9 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py @@ -0,0 +1,298 @@ +"""Rule matching: extensions, sensitive naming, pinned names, derived copies. + +Two eras live side by side here. ``CATEGORY_EXTENSIONS`` and :func:`classify` +are the original small taxonomy and stay exactly as they were, because the +``simple`` profile and the first test file are built on them. Everything else +is profile driven and reads its data out of a :class:`~carl_file_organizer.config.Config`. + +The sensitive check is the part worth reading twice. It never opens a file; it +only looks at the name, in four passes of decreasing certainty: + + exact_names the whole name, compared case-insensitively + exact_stems the name minus its suffix, compared as a whole word + suffixes the suffix alone + markers substring, and only for long or non-ASCII words + +Short words are never substring matched. That is the fix for the day a rule +containing ``r2`` swallowed ``pixverse-r2.md``. +""" + +from __future__ import annotations + +import fnmatch +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Set + +from .scanner import split_suffix + + +CATEGORY_EXTENSIONS: Dict[str, Set[str]] = { + "Documents": { + ".doc", + ".docx", + ".epub", + ".key", + ".md", + ".numbers", + ".pages", + ".pdf", + ".ppt", + ".pptx", + ".rtf", + ".txt", + ".xls", + ".xlsx", + }, + "Images": { + ".avif", + ".gif", + ".heic", + ".jpeg", + ".jpg", + ".png", + ".svg", + ".tif", + ".tiff", + ".webp", + }, + "Video": {".avi", ".m4v", ".mkv", ".mov", ".mp4", ".webm"}, + "Audio": {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".wav"}, + "Archives": { + ".7z", + ".bz2", + ".gz", + ".rar", + ".tar", + ".tgz", + ".xz", + ".zip", + }, + "Installers": {".apk", ".dmg", ".exe", ".iso", ".msi", ".pkg"}, + "Code & Data": { + ".c", + ".cpp", + ".css", + ".csv", + ".go", + ".html", + ".ipynb", + ".java", + ".js", + ".json", + ".jsx", + ".py", + ".rb", + ".rs", + ".sql", + ".swift", + ".toml", + ".ts", + ".tsx", + ".xml", + ".yaml", + ".yml", + }, +} + + +def classify(path: Path) -> tuple: + """Return a human-readable category, reason, and confidence.""" + suffix = path.suffix.lower() + for category, extensions in CATEGORY_EXTENSIONS.items(): + if suffix in extensions: + return category, "{0} file".format(suffix or "extensionless"), 0.98 + return "Other", "file type is not in the small default taxonomy", 0.55 + + +# --------------------------------------------------------------------------- +# profile driven matching +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Rule: + key: str # the suffix that matched, e.g. ".pdf" + dest_key: str + min_age_hours: int + reason_key: str + label: Optional[Dict[str, str]] = None + + +@dataclass(frozen=True) +class SensitiveHit: + layer: str # "exact" | "suffix" | "marker" -> becomes rule "sensitive:" + needle: str + where: str # "exact_names" | "exact_stems" | "suffixes" | "markers" + + +@dataclass(frozen=True) +class NamePattern: + id: str + match: str # "contains" | "prefix" | "glob" + needle: str + dest: Optional[str] # literal path relative to root + dest_key: Optional[str] # or a key in profile names + + +def _profile(config: Any) -> Mapping[str, Any]: + return getattr(config, "profile", config) + + +def _section(config: Any, name: str) -> Mapping[str, Any]: + section = _profile(config).get(name) + return section if isinstance(section, Mapping) else {} + + +def _list(config: Any, section: str, key: str) -> Sequence[Any]: + values = _section(config, section).get(key) + return values if isinstance(values, (list, tuple)) else () + + +def match_rule(suffix: str, config: Any) -> Optional[Rule]: + """Look a suffix up in the profile rule table.""" + + rules = _section(config, "rules") + folded = str(suffix).casefold() + entry = rules.get(folded) + if not isinstance(entry, Mapping): + return None + aging = _section(config, "aging") + default_hours = int(aging.get("default_hours", 48) or 0) + minimum = entry.get("min_age_hours") + return Rule( + key=folded, + dest_key=str(entry.get("dest")), + min_age_hours=int(default_hours if minimum is None else minimum), + reason_key=str(entry.get("reason") or "ext_other"), + label=entry.get("label") if isinstance(entry.get("label"), Mapping) else None, + ) + + +def is_sensitive(name: str, config: Any) -> Optional[SensitiveHit]: + """Name-only credential detection. Contents are never read.""" + + text = str(name) + folded = text.casefold() + stem, suffix = split_suffix(text) + folded_stem = stem.casefold() + + for candidate in _list(config, "sensitive", "exact_names"): + if folded == str(candidate).casefold(): + return SensitiveHit("exact", str(candidate), "exact_names") + + for candidate in _list(config, "sensitive", "exact_stems"): + if folded_stem == str(candidate).casefold(): + return SensitiveHit("exact", str(candidate), "exact_stems") + + for candidate in _list(config, "sensitive", "suffixes"): + needle = str(candidate).casefold() + if suffix and suffix == needle: + return SensitiveHit("suffix", str(candidate), "suffixes") + # A dot-prefixed suffix such as ".env" also matches "prod.env" when the + # name carries no suffix of its own. + if not suffix and folded.endswith(needle) and folded != needle: + return SensitiveHit("suffix", str(candidate), "suffixes") + + minimum = int(_section(config, "sensitive").get("marker_min_len", 5) or 5) + for candidate in _list(config, "sensitive", "markers"): + marker = str(candidate) + ascii_only = all(ord(char) < 128 for char in marker) + if ascii_only and len(marker) < minimum: + # Refused at load time; skipped here so a hand-edited profile in the + # wild degrades to "missed" rather than "matched everything". + continue + if marker.casefold() in folded: + return SensitiveHit("marker", marker, "markers") + return None + + +def derivative_hint(name: str, config: Any) -> Optional[str]: + """Return the hint substring when a name looks like a scratch copy.""" + + folded = str(name).casefold() + for hint in _profile(config).get("derivative_hints") or (): + needle = str(hint).casefold() + if needle and needle in folded: + return str(hint) + return None + + +def is_regenerable_name(name: str, config: Any) -> bool: + basenames = _list(config, "regenerable", "basenames") + return any(str(name) == str(candidate) for candidate in basenames) + + +def regenerable_restore(name: str, config: Any) -> Dict[str, str]: + restore = _section(config, "regenerable").get("restore") + table = restore if isinstance(restore, Mapping) else {} + entry = table.get(str(name)) or table.get("_default") or {} + if isinstance(entry, Mapping): + return {"zh": str(entry.get("zh", "")), "en": str(entry.get("en", ""))} + return {"zh": str(entry), "en": str(entry)} + + +def match_pinned(name: str, config: Any) -> Optional[str]: + """Exact name or fnmatch glob against the pinned allow list.""" + + text = str(name) + for pattern in _profile(config).get("pinned") or (): + candidate = str(pattern) + if text == candidate or fnmatch.fnmatch(text, candidate): + return candidate + return None + + +def match_name_pattern(name: str, config: Any) -> Optional[NamePattern]: + """User written name rules; substring matching needs at least four characters.""" + + text = str(name) + folded = text.casefold() + _stem, suffix = split_suffix(text) + for raw in _profile(config).get("name_patterns") or (): + if not isinstance(raw, Mapping): + continue + needle = str(raw.get("needle") or "") + if not needle: + continue + suffixes = raw.get("suffixes") + if isinstance(suffixes, (list, tuple)) and suffixes: + allowed = {str(item).casefold() for item in suffixes} + if suffix not in allowed: + continue + how = str(raw.get("match") or "contains") + folded_needle = needle.casefold() + if how == "prefix": + hit = folded.startswith(folded_needle) + elif how == "glob": + hit = fnmatch.fnmatch(text, needle) + else: + hit = len(folded_needle) >= 4 and folded_needle in folded + if not hit: + continue + return NamePattern( + id=str(raw.get("id") or needle), + match=how, + needle=needle, + dest=str(raw["dest"]) if raw.get("dest") else None, + dest_key=str(raw["dest_key"]) if raw.get("dest_key") else None, + ) + return None + + +def is_archive_suffix(suffix: str, config: Any) -> bool: + folded = str(suffix).casefold() + return any( + folded == str(item).casefold() for item in _profile(config).get("archive_suffixes") or () + ) + + +def is_installer_suffix(suffix: str, config: Any) -> bool: + folded = str(suffix).casefold() + return any( + folded == str(item).casefold() for item in _profile(config).get("installer_suffixes") or () + ) + + +def forbidden_markers(config: Any) -> List[str]: + return [str(item) for item in _list(config, "forbidden", "markers")] diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py new file mode 100644 index 0000000..145889b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py @@ -0,0 +1,238 @@ +"""Command-line interface: argument parsing and dispatch only. + +Every subcommand resolves to ``.run(args)``. No business logic lives +here, so the subcommands and their flag names are frozen from day zero and each +work package only fills in its own module. + + plan -> carl_file_organizer.planner.run + build -> carl_file_organizer.notes.run + apply -> carl_file_organizer.executor.run + review -> carl_file_organizer.server.run + undo -> carl_file_organizer.undo.run + status -> carl_file_organizer.status.run + clear-tags -> carl_file_organizer.tags.run + +A module that has no ``run`` yet produces a one-line message and exit code 2, +never an ImportError traceback. +""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.util +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, List, Optional + +from . import __version__, i18n + +#: subcommand name -> module inside this package that owns it +COMMAND_MODULES = { + "plan": "planner", + "build": "notes", + "apply": "executor", + "review": "server", + "undo": "undo", + "status": "status", + "clear-tags": "tags", +} + + +def _iso(text: str) -> datetime: + try: + return datetime.fromisoformat(text) + except ValueError as error: + raise argparse.ArgumentTypeError( + "--now must be an ISO 8601 timestamp, for example 2026-09-02T10:15:00" + ) from error + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="carl-file-organizer", + description="Human-first file organization with explicit approval.", + ) + parser.add_argument("--version", action="version", version=__version__) + subparsers = parser.add_subparsers(dest="command", required=True) + + # -- plan --------------------------------------------------------------- + plan = subparsers.add_parser("plan", help="scan read-only and write a review page") + plan.add_argument( + "source", + nargs="?", + type=Path, + default=Path.home() / "Downloads", + help="folder to scan (default: ~/Downloads)", + ) + plan.add_argument("--lang", choices=["zh", "en"], default=None) + plan.add_argument("--profile", choices=["tiered", "simple"], default=None) + plan.add_argument("--hash-duplicates", action="store_true") + plan.add_argument("--large-mb", type=int, default=500) + plan.add_argument("--target-percent", type=float, default=90.0) + plan.add_argument("--no-dir-sizes", dest="dir_sizes", action="store_false") + plan.add_argument( + "--no-permanent-delete-options", + dest="permanent_delete_options", + action="store_false", + ) + plan.add_argument("--now", type=_iso, default=None, help="fix the clock (tests and replays)") + plan.add_argument("--allow-outside-home", action="store_true") + plan.add_argument( + "--allow-referenced", + action="store_true", + help="downgrade referenced-elsewhere holds to a warning", + ) + plan.set_defaults(dir_sizes=True, permanent_delete_options=True) + + # -- build -------------------------------------------------------------- + build = subparsers.add_parser( + "build", help="lay an agent's notes.json over a plan.json" + ) + build.add_argument("plan", type=Path, help="the plan.json to annotate") + build.add_argument( + "--notes", + type=Path, + default=None, + help="notes.json written by the agent; without it the plan is only rewritten", + ) + build.add_argument( + "--output", + type=Path, + default=None, + help="where to write the annotated plan (default: in place)", + ) + build.add_argument( + "--report", + action="store_true", + help="also re-render report.html next to the output plan", + ) + build.add_argument("--lang", choices=["zh", "en"], default=None) + + # -- apply -------------------------------------------------------------- + apply_cmd = subparsers.add_parser("apply", help="apply only the approved actions") + apply_cmd.add_argument("approval", type=Path) + apply_cmd.add_argument("--dry-run", action="store_true") + apply_cmd.add_argument("--allow-permanent-delete", action="store_true") + apply_cmd.add_argument("--audit", type=Path, default=None) + + # -- review ------------------------------------------------------------- + review = subparsers.add_parser("review", help="open the review page") + review.add_argument("plan", type=Path) + review.add_argument("--serve", action="store_true", required=True) + review.add_argument("--port", type=int, default=0) + review.add_argument("--no-open", dest="open_browser", action="store_false") + review.add_argument("--allow-permanent-delete", action="store_true") + review.set_defaults(open_browser=True) + + # -- undo --------------------------------------------------------------- + undo = subparsers.add_parser("undo", help="move things back where they came from") + undo.add_argument("record", type=Path, help="audit.jsonl or a manifest .tsv") + undo.add_argument("--dry-run", action="store_true") + + # -- status ------------------------------------------------------------- + status = subparsers.add_parser("status", help="last tidy-up and current mess") + status.add_argument( + "source", nargs="?", type=Path, default=Path.home() / "Downloads" + ) + status.add_argument("--lang", choices=["zh", "en"], default=None) + status.add_argument("--allow-outside-home", action="store_true") + + # -- clear-tags --------------------------------------------------------- + clear = subparsers.add_parser("clear-tags", help="remove the temporary Finder tag") + clear.add_argument( + "source", nargs="?", type=Path, default=Path.home() / "Downloads" + ) + clear.add_argument("--from", dest="from_lists", action="append", type=Path, default=None) + clear.add_argument("--all-lists", action="store_true") + clear.add_argument("--tag", default=None) + clear.add_argument("--dry-run", action="store_true") + + return parser + + +def _module_run(module_name: str) -> Optional[Callable[[Any], int]]: + """Return ``module.run`` when it exists, without letting ImportError escape.""" + + full_name = "{0}.{1}".format(__package__, module_name) + try: + if importlib.util.find_spec(full_name) is None: + return None + except (ImportError, ValueError): + return None + module = importlib.import_module(full_name) + run = getattr(module, "run", None) + return run if callable(run) else None + + +def _not_implemented(command: str, module_name: str, lang: str) -> int: + print( + "error: " + + i18n.t( + "cli_not_implemented", + lang, + command=command, + module="{0}.{1}".format(__package__, module_name), + ), + file=sys.stderr, + ) + return 2 + + +def _prepare_config(args: argparse.Namespace) -> Any: + """Shared front half of plan/status: refuse bad roots, resolve settings.""" + + from . import config as config_module + from . import paths + + root = paths.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + cfg = config_module.resolve_config(root, lang=args.lang, profile=getattr(args, "profile", None)) + for message in cfg.warnings: + print(i18n.t("warning_prefix", cfg.lang) + message, file=sys.stderr) + return cfg + + +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + command = args.command + module_name = COMMAND_MODULES[command] + lang = getattr(args, "lang", None) or "en" + + try: + if command == "plan": + from . import config as config_module + + cfg = _prepare_config(args) + lang = cfg.lang + written = config_module.save_dotfile(cfg, only_if_missing=True) + if written is not None: + print(i18n.t("cli_dotfile_written", cfg.lang, path=written)) + print(i18n.t("cli_lang_source", cfg.lang, lang=cfg.lang, source=cfg.lang_source)) + args.config = cfg + run = _module_run(module_name) + if run is None: + return _not_implemented(command, module_name, cfg.lang) + return int(run(args)) + + if command == "status": + cfg = _prepare_config(args) + lang = cfg.lang + args.config = cfg + + run = _module_run(module_name) + if run is None: + return _not_implemented(command, module_name, lang) + return int(run(args)) + except KeyboardInterrupt: + return 130 + except Exception as error: # noqa: BLE001 - CLI boundary: one clean line, never a traceback + print("error: {0}".format(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py new file mode 100644 index 0000000..caaceb3 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py @@ -0,0 +1,544 @@ +"""Language detection, profile loading and merging, per-folder settings file. + +Everything downstream reads a single :class:`Config`. The contract frozen here: + + resolve_config(root, *, lang=None, profile=None) -> Config + dir_name(config, key) -> "20_知识库/文档资料/PDF" + dest_dir(config, key) -> Path + managed_dir(config) -> Path + partition_prefixes(config) -> ("00_", "10_", ...) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from . import paths + +DOTFILE_NAME = ".carl-file-organizer.json" +DEFAULT_PROFILE = "tiered" +LANG_ENV = "CARL_FILE_ORGANIZER_LANG" +LANGS = ("zh", "en") +MAX_NAME_DEPTH = 3 + +#: Keys whose list values are unioned when a dotfile overrides a built-in profile. +UNION_LIST_KEYS = ( + ("pinned",), + ("derivative_hints",), + ("archive_suffixes",), + ("installer_suffixes",), + ("sensitive", "exact_names"), + ("sensitive", "exact_stems"), + ("sensitive", "suffixes"), + ("sensitive", "markers"), + ("forbidden", "basenames"), + ("forbidden", "suffixes"), + ("forbidden", "markers"), + ("forbidden", "glob"), + ("regenerable", "basenames"), + ("guard", "reference_files"), + ("guard", "reference_scan_roots"), + ("dedupe", "copy_patterns"), +) + +#: Sections merged key by key, with the dotfile value winning per key. +SCALAR_SECTIONS = ("aging", "tags", "dedupe", "guard", "regenerable", "sensitive") + +#: Top-level keys the dotfile simply overwrites. +OVERRIDE_KEYS = ("lang", "profile", "large_bytes", "managed_dir", "transient_tag", "review_log") + +#: Settings that record a choice rather than a rule, and so stay out of the digest. +HASH_IGNORED_KEYS = ("lang",) + + +class LanguageAmbiguous(ValueError): + """Both language layouts exist in the same folder and no language was given.""" + + +@dataclass +class Config: + root: Path + lang: str + lang_source: str + profile_name: str + profile: Dict[str, Any] + profile_source: str + dotfile_path: Optional[Path] + profile_hash: str + warnings: List[str] = field(default_factory=list) + + +# -------------------------------------------------------------------------- +# language detection +# -------------------------------------------------------------------------- + + +def normalize_lang(text: Optional[str]) -> Optional[str]: + if not text: + return None + folded = str(text).strip().casefold().replace("_", "-") + if folded.startswith("zh") or "hans" in folded or "hant" in folded: + return "zh" + if folded.startswith("en"): + return "en" + return None + + +def _apple_languages(timeout_s: float = 1.0) -> Optional[str]: + """First entry of ``defaults read -g AppleLanguages``; ``None`` when unavailable.""" + + if sys.platform != "darwin": + return None + try: + result = subprocess.run( + ["defaults", "read", "-g", "AppleLanguages"], + capture_output=True, + text=True, + timeout=timeout_s, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + match = re.search(r'"?([A-Za-z]{2}[A-Za-z0-9\-_]*)"?', result.stdout) + return match.group(1) if match else None + + +def _posix_locale(env: Mapping[str, str]) -> Optional[str]: + for key in ("LC_ALL", "LC_MESSAGES", "LANG"): + value = env.get(key) + if value: + return value.split(".")[0] + return None + + +def _sniff_dir_names(profile: Mapping[str, Any]) -> Dict[str, List[str]]: + """Directory names that betray which language layout a folder already uses.""" + + probes: Dict[str, List[str]] = {"zh": [], "en": []} + names = profile.get("names") or {} + candidates = [names.get("inbox"), profile.get("managed_dir")] + for entry in candidates: + if not isinstance(entry, Mapping): + continue + zh_name = entry.get("zh") + en_name = entry.get("en") + if not zh_name or not en_name or zh_name == en_name: + continue + probes["zh"].append(str(zh_name)) + probes["en"].append(str(en_name)) + return probes + + +def _sniff_existing_dirs(root: Path, profile: Mapping[str, Any]) -> List[str]: + found: List[str] = [] + probes = _sniff_dir_names(profile) + for lang in LANGS: + for name in probes[lang]: + try: + if (Path(root) / name).is_dir(): + found.append(lang) + break + except OSError: + continue + return found + + +def detect_lang( + root: Path, + override: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + dotfile: Optional[Mapping[str, Any]] = None, + profile: Optional[Mapping[str, Any]] = None, +) -> Tuple[str, str]: + """Return ``(lang, source)``. + + Chain: flag > dotfile > ``CARL_FILE_ORGANIZER_LANG`` > existing directories > + AppleLanguages > POSIX locale > ``en``. + """ + + environ = os.environ if env is None else env + + if override: + chosen = normalize_lang(override) + if chosen is None: + raise ValueError("unsupported language: {0}".format(override)) + return chosen, "flag" + + settings = dotfile if dotfile is not None else (load_dotfile(root) or {}) + chosen = normalize_lang(settings.get("lang") if isinstance(settings, Mapping) else None) + if chosen: + return chosen, "dotfile" + + chosen = normalize_lang(environ.get(LANG_ENV)) + if chosen: + return chosen, "env" + + if profile: + found = _sniff_existing_dirs(root, profile) + if len(found) > 1: + raise LanguageAmbiguous( + "both language layouts exist in {0}; pass --lang zh or --lang en".format(root) + ) + if len(found) == 1: + return found[0], "existing-dirs" + + chosen = normalize_lang(_apple_languages()) + if chosen: + return chosen, "apple-languages" + + chosen = normalize_lang(_posix_locale(environ)) + if chosen: + return chosen, "posix-locale" + + return "en", "default" + + +# -------------------------------------------------------------------------- +# profile loading and merging +# -------------------------------------------------------------------------- + + +def load_builtin_profile(name: str) -> Dict[str, Any]: + """Read ``carl_file_organizer/profiles/.json`` from the installed package.""" + + from importlib.resources import files # 3.9+ + + if not re.fullmatch(r"[a-z0-9_\-]+", name or ""): + raise ValueError("unknown profile: {0!r}".format(name)) + resource = files("carl_file_organizer.profiles").joinpath("{0}.json".format(name)) + try: + text = resource.read_text(encoding="utf-8") + except (FileNotFoundError, OSError) as error: + raise ValueError("unknown profile: {0}".format(name)) from error + profile = json.loads(text) + validate_profile(profile) + return profile + + +def load_dotfile(root: Path) -> Optional[Dict[str, Any]]: + """Read ``/.carl-file-organizer.json``; ``None`` when it does not exist.""" + + path = Path(root) / DOTFILE_NAME + if not path.is_file(): + return None + raw = path.read_text(encoding="utf-8") + try: + data = json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError( + "{0} is not valid JSON (line {1}, column {2}): {3}".format( + path, error.lineno, error.colno, error.msg + ) + ) from error + if not isinstance(data, dict): + raise ValueError("{0} must contain a JSON object".format(path)) + return data + + +def _union(base: Sequence[Any], extra: Sequence[Any]) -> List[Any]: + merged: List[Any] = [] + seen = set() + for item in list(base) + list(extra): + marker = json.dumps(item, sort_keys=True, ensure_ascii=False) if isinstance(item, (dict, list)) else item + if marker in seen: + continue + seen.add(marker) + merged.append(item) + return merged + + +def _union_name_patterns(base: Sequence[Any], extra: Sequence[Any]) -> List[Any]: + merged: List[Any] = [] + index: Dict[str, int] = {} + for item in list(base) + list(extra): + if isinstance(item, dict) and item.get("id"): + key = str(item["id"]) + if key in index: + merged[index[key]] = item + continue + index[key] = len(merged) + merged.append(item) + return merged + + +def _get_in(data: Mapping[str, Any], path: Sequence[str]) -> Any: + current: Any = data + for step in path: + if not isinstance(current, Mapping) or step not in current: + return None + current = current[step] + return current + + +def _set_in(data: Dict[str, Any], path: Sequence[str], value: Any) -> None: + current = data + for step in path[:-1]: + current = current.setdefault(step, {}) + current[path[-1]] = value + + +def validate_profile(profile: Mapping[str, Any]) -> None: + """Refuse footguns before anything reads the profile.""" + + sensitive = profile.get("sensitive") or {} + minimum = int(sensitive.get("marker_min_len", 5) or 5) + for marker in sensitive.get("markers", []) or []: + text = str(marker) + if all(ord(char) < 128 for char in text) and len(text) < minimum: + raise ValueError( + "sensitive.markers entry {0!r} is too short: ASCII markers are substring " + "matched and must be at least {1} characters. Put short words in " + "sensitive.exact_stems instead.".format(text, minimum) + ) + names = profile.get("names") or {} + for key in names: + if len(str(key).split(".")) > MAX_NAME_DEPTH: + raise ValueError( + "names key {0!r} is deeper than {1} levels".format(key, MAX_NAME_DEPTH) + ) + + +def merge_profile(base: Mapping[str, Any], override: Optional[Mapping[str, Any]]) -> Dict[str, Any]: + """Sparse overlay of a dotfile on top of a built-in profile. + + ``names`` and ``rules`` merge key by key; list-shaped settings are unioned; + ``lang`` / ``aging`` / ``large_bytes`` and friends are overwritten. + """ + + merged: Dict[str, Any] = json.loads(json.dumps(base, ensure_ascii=False)) + if not override: + validate_profile(merged) + return merged + + for section in ("names", "rules"): + incoming = override.get(section) + if not isinstance(incoming, Mapping): + continue + target = merged.setdefault(section, {}) + for key, value in incoming.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), Mapping): + combined = dict(target[key]) + combined.update(value) + target[key] = combined + else: + target[key] = value + + for section in SCALAR_SECTIONS: + incoming = override.get(section) + if not isinstance(incoming, Mapping): + continue + target = merged.setdefault(section, {}) + for key, value in incoming.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), Mapping): + combined = dict(target[key]) + combined.update(value) + target[key] = combined + else: + target[key] = value + + # Unions read from ``base`` on purpose: the scalar pass above may have + # replaced a list wholesale, and a union must never lose built-in entries. + for path in UNION_LIST_KEYS: + incoming = _get_in(override, path) + if not isinstance(incoming, list): + continue + current = _get_in(base, path) + _set_in(merged, path, _union(current if isinstance(current, list) else [], incoming)) + + if isinstance(override.get("name_patterns"), list): + current = base.get("name_patterns") + merged["name_patterns"] = _union_name_patterns( + current if isinstance(current, list) else [], override["name_patterns"] + ) + + for key in OVERRIDE_KEYS: + if key in override: + value = override[key] + if key in ("managed_dir", "transient_tag", "review_log") and isinstance(value, Mapping): + combined = dict(merged.get(key) or {}) + combined.update(value) + merged[key] = combined + else: + merged[key] = value + + validate_profile(merged) + return merged + + +def profile_hash(profile: Mapping[str, Any]) -> str: + """Short digest of the rules in a merged profile; stored in plan.json to catch drift. + + The chosen language is left out on purpose. The first ``plan`` in a folder + hashes the settings and then writes the settings file, which records the + language; hashing that back in would make every second run announce that + the settings had changed. Language decides what the folders are called, not + what goes into them, so it is not drift. An edited rule or a new sensitive + name still moves the digest. + """ + + material = { + key: value for key, value in profile.items() if key not in HASH_IGNORED_KEYS + } + encoded = json.dumps(material, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:12] + + +# -------------------------------------------------------------------------- +# resolution and persistence +# -------------------------------------------------------------------------- + + +def resolve_config( + root: Path, + *, + lang: Optional[str] = None, + profile: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, +) -> Config: + """Load the dotfile, pick the profile and the language, return a Config.""" + + root_path = Path(root).expanduser() + settings = load_dotfile(root_path) + profile_name = profile or (settings or {}).get("profile") or DEFAULT_PROFILE + base = load_builtin_profile(str(profile_name)) + merged = merge_profile(base, settings) + + warnings: List[str] = [] + lang_value, lang_source = detect_lang( + root_path, lang, env=env, dotfile=settings or {}, profile=merged + ) + if lang_source == "default": + warnings.append( + "no language signal found; falling back to en (pass --lang zh to change it)" + ) + + return Config( + root=root_path, + lang=lang_value, + lang_source=lang_source, + profile_name=str(profile_name), + profile=merged, + profile_source="builtin+dotfile" if settings else "builtin", + dotfile_path=(root_path / DOTFILE_NAME) if settings is not None else None, + profile_hash=profile_hash(merged), + warnings=warnings, + ) + + +def save_dotfile(config: Config, *, only_if_missing: bool = True) -> Optional[Path]: + """Write the smallest useful settings file; never write the merged profile back.""" + + path = Path(config.root) / DOTFILE_NAME + if only_if_missing and path.exists(): + return None + payload = {"profile": config.profile_name, "lang": config.lang} + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + config.dotfile_path = path + return path + + +# -------------------------------------------------------------------------- +# name resolution +# -------------------------------------------------------------------------- + + +def _pick_lang(entry: Any, lang: str) -> str: + if isinstance(entry, Mapping): + value = entry.get(lang) + if value: + return str(value) + value = entry.get("en") + if value: + return str(value) + for item in entry.values(): + if item: + return str(item) + return "" + return str(entry) + + +def dir_name(config: Config, key: str) -> str: + """``"library.docs.pdf"`` -> ``"20_知识库/文档资料/PDF"``. + + Parent keys are derived from the dotted prefix chain; three levels maximum. + """ + + parts = str(key).split(".") + if len(parts) > MAX_NAME_DEPTH: + raise ValueError( + "destination key {0!r} is deeper than {1} levels".format(key, MAX_NAME_DEPTH) + ) + names = config.profile.get("names") or {} + segments: List[str] = [] + for index in range(1, len(parts) + 1): + prefix = ".".join(parts[:index]) + if prefix not in names: + raise KeyError("profile {0} has no name for {1!r}".format(config.profile_name, prefix)) + segments.append(paths.safe_dir_name(_pick_lang(names[prefix], config.lang))) + return "/".join(segments) + + +def dest_dir(config: Config, key: str) -> Path: + return Path(config.root) / dir_name(config, key) + + +def managed_dir(config: Config) -> Path: + name = _pick_lang(config.profile.get("managed_dir"), config.lang) or "00_File_Organizer" + return Path(config.root) / paths.safe_dir_name(name) + + +def transient_tag(config: Config) -> str: + return _pick_lang(config.profile.get("transient_tag"), config.lang) or "Moved by File Organizer" + + +def transient_tags(config: Config) -> Tuple[str, ...]: + """Every language spelling of the temporary tag, for stripping user tags.""" + + entry = config.profile.get("transient_tag") or {} + if isinstance(entry, Mapping): + return tuple(str(value) for value in entry.values() if value) + return (str(entry),) + + +def review_log_name(config: Config) -> str: + return _pick_lang(config.profile.get("review_log"), config.lang) or "review-log.md" + + +def partition_dir_names(config: Config) -> Tuple[str, ...]: + """Top-level partition directory names in every language, plus the managed folder.""" + + names = config.profile.get("names") or {} + found: List[str] = [] + for key, entry in names.items(): + if "." in str(key): + continue + if isinstance(entry, Mapping): + found.extend(str(value) for value in entry.values() if value) + elif entry: + found.append(str(entry)) + managed = config.profile.get("managed_dir") or {} + if isinstance(managed, Mapping): + found.extend(str(value) for value in managed.values() if value) + return tuple(sorted(set(found))) + + +def partition_prefixes(config: Config) -> Tuple[str, ...]: + """Numeric partition prefixes such as ``("00_", "10_", "20_", "60_", "90_")``.""" + + prefixes = set() + for name in partition_dir_names(config): + match = re.match(r"^(\d+_)", name) + if match: + prefixes.add(match.group(1)) + return tuple(sorted(prefixes)) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py new file mode 100644 index 0000000..a327b04 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py @@ -0,0 +1,1337 @@ +"""Execution in three stages: validate the plan, preflight each action, then act. + +Nothing here decides anything. The plan already said what could happen and a +person already said which of it should happen; this module's whole job is to +refuse everything else, and to leave a trail that lets the person put it all +back. + +The stages, in order: + + validate_plan the approval file as a whole: schema, root, the id list + preflight_action one item, against the filesystem as it is right now + execute_action the actual move, trash or delete, plus verification + apply_approved_plan wires the three together and writes the paper trail + +Two rules shape everything below. A permanent delete needs an explicit flag +and refuses the entire batch when it is missing, because a silent skip would +let someone believe their disk got emptier. And a move that does not verify is +put straight back where it came from, and the rest of the batch stops. +""" + +from __future__ import annotations + +import copy +import json +import os +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +from . import i18n, ids +from . import manifest as manifest_module +from . import paths +from . import tags as tags_module +from . import trash as trash_module +from .guard import is_observer + +#: Literal on purpose: the executor must not depend on the planner to know +#: which plan format it can run. +SCHEMA_VERSION = 2 + +SUPPORTED_KINDS = ("move", "trash", "delete") + +#: Only these rules produce an action a person may reroute from the page. +REROUTABLE_RULES = ("unknown-ext", "dir:pending", "dir:derivative") + +#: Statuses are lowercase in audit.jsonl and uppercase in the TSV manifests. +STATUS_MANIFEST = { + "planned": "PLANNED", + "moved": "MOVED", + "trashed": "TRASHED", + "deleted": "DELETED", + "dry-run": "DRY-RUN", + "skipped": "SKIPPED", + "conflict": "CONFLICT", + "refused": "REFUSED", + "failed": "FAILED", +} + +DEFAULT_LSOF_TIMEOUT_S = 3.0 +DIR_ENTRY_CAP = 20000 +DIR_TIME_BUDGET_S = 1.5 + +TEXT = { + "source_missing": { + "zh": "源已经不在原处，跳过", + "en": "The source is no longer there; skipped", + }, + "source_symlink": { + "zh": "路径里有软链，拒绝执行", + "en": "A symlink sits in this path; refused", + }, + "outside_root": { + "zh": "源或目标不在整理目录里，拒绝执行", + "en": "Source or destination falls outside the folder; refused", + }, + "forbidden": { + "zh": "禁刀区，拒绝执行（{label}）", + "en": "No-go zone; refused ({label})", + }, + "changed": { + "zh": "计划生成之后这一项被改动过，跳过；重新 plan 就好", + "en": "This item changed after the plan was made; skipped, just run plan again", + }, + "kind_changed": { + "zh": "文件和目录的身份变了，跳过", + "en": "It is no longer the same kind of item; skipped", + }, + "destination_exists": { + "zh": "目标已经有同名的东西，不覆盖，源没有动", + "en": "The destination already exists; nothing was overwritten and the source stayed put", + }, + "destination_missing": { + "zh": "这条动作没有目标路径", + "en": "This action has no destination", + }, + "in_use": { + "zh": "现在有进程开着它（{who}），跳过", + "en": "A process still has it open ({who}); skipped", + }, + "ready": {"zh": "可以执行", "en": "ready"}, + "delete_no_flag": { + "zh": "批准名单里有永久删除，但没有加 --allow-permanent-delete，整批拒绝执行", + "en": ( + "The approval list contains a permanent delete but --allow-permanent-delete " + "was not given; the whole batch is refused" + ), + }, + "delete_not_regenerable": { + "zh": "只允许永久删除可再生的构建产物，{name} 不在名单里", + "en": "Permanent delete is only for regenerable build output, and {name} is not on that list", + }, + "delete_no_survivor": { + "zh": "重复组里没有确定保留的副本，拒绝永久删除", + "en": "No surviving copy is confirmed in this duplicate group; refusing the permanent delete", + }, + "delete_bad_tier": { + "zh": "这一项不属于可永久删除的类别（{tier}）", + "en": "This item is not in a permanently deletable class ({tier})", + }, + "delete_cold_dir": { + "zh": "冷存候选只允许删文件，不允许删目录", + "en": "Cold candidates may only be deleted as files, never as folders", + }, + "moved": {"zh": "已按批准移动", "en": "moved as approved"}, + "trashed": {"zh": "已移入废纸篓", "en": "moved to the trash"}, + "deleted": {"zh": "已永久删除", "en": "permanently deleted"}, + "dry_run": {"zh": "预演通过，没有真的动", "en": "validated; nothing was touched"}, + "verify_failed": { + "zh": "移动之后校验不通过（{detail}），已原路移回", + "en": "The move did not verify ({detail}); it was put back", + }, + "verify_failed_stuck": { + "zh": "移动之后校验不通过（{detail}），而且移回也失败了：{error}", + "en": "The move did not verify ({detail}) and moving it back also failed: {error}", + }, + "batch_stopped": { + "zh": "本批在前一条失败之后停下了，这一条没有执行", + "en": "The batch stopped after an earlier failure; this one was not attempted", + }, + "error": {"zh": "执行出错：{error}", "en": "execution error: {error}"}, + "verify_dst_missing": {"zh": "目标不存在", "en": "destination missing"}, + "verify_src_left": {"zh": "源还在原处", "en": "source still there"}, + "verify_size": { + "zh": "体积对不上（{before} -> {after}）", + "en": "size mismatch ({before} -> {after})", + }, + "verify_entries": { + "zh": "条目数对不上（{before} -> {after}）", + "en": "entry count mismatch ({before} -> {after})", + }, +} + +REVIEW_TEXT = { + "log_moved": { + "zh": "本次整理共移动 {count} 项；执行清单：`{manifest}`。", + "en": "This run moved {count} item(s); manifest: `{manifest}`.", + }, + "log_tagged": { + "zh": "本次移动目标已打 macOS Finder 标签 `{tag}`；标签清单：`{list}`。", + "en": "The moved items carry the macOS Finder tag `{tag}`; tag list: `{list}`.", + }, + "log_sensitive": { + "zh": "已将 {count} 项疑似敏感命名的东西移入 `{dest}`，未读取内容：{names}。", + "en": ( + "{count} item(s) with credential-looking names went to `{dest}`; " + "their contents were never read: {names}." + ), + }, + "log_conflict": { + "zh": "{names} 的目标位置已经有同名的东西，未覆盖，也未删除任一项，本次跳过。", + "en": ( + "{names} already had something at the destination; nothing was overwritten, " + "neither copy was deleted, and it was skipped." + ), + }, + "log_trashed": { + "zh": "已将 {count} 项移入废纸篓（后端 {backend}），在废纸篓里可以放回原处。", + "en": "{count} item(s) went to the trash (backend {backend}) and can be put back from there.", + }, + "log_deleted": { + "zh": "已永久删除 {count} 项可再生的构建产物：{names}。", + "en": "Permanently deleted {count} regenerable build output(s): {names}.", + }, + "log_skipped": { + "zh": "{count} 项在计划生成之后被动过或已经不在，本次跳过。", + "en": "{count} item(s) changed or vanished after the plan was made and were skipped.", + }, + "log_failed": { + "zh": "{count} 项执行失败并已原路移回，本批余下动作已停止。", + "en": "{count} item(s) failed, were put back, and the rest of the batch stopped.", + }, + "log_recheck": { + "zh": "复查结果：dry-run actions={count}。", + "en": "Recheck: dry-run actions={count}.", + }, +} + +CLI_TEXT = { + "cli_audit": {"zh": "审计：{path}", "en": "Audit: {path}"}, + "cli_manifest": {"zh": "清单：{path}", "en": "Manifest: {path}"}, + "cli_tag_list": {"zh": "标签清单：{path}", "en": "Tag list: {path}"}, + "cli_review_log": {"zh": "人工日志：{path}", "en": "Log: {path}"}, + "cli_approved_copy": {"zh": "批准文件已留档：{path}", "en": "Approval kept at: {path}"}, + "cli_dry_run": { + "zh": "预演模式：一个字节都没有写，清单和日志也不生成。", + "en": "Dry run: nothing was written, and no manifest or log was produced.", + }, + "cli_summary": { + "zh": "共 {total} 条：移动 {moved}，废纸篓 {trashed}，永久删除 {deleted}，跳过 {skipped}，失败 {failed}", + "en": ( + "{total} action(s): moved {moved}, trashed {trashed}, deleted {deleted}, " + "skipped {skipped}, failed {failed}" + ), + }, + "cli_profile_drift": { + "zh": "提醒：计划生成之后设置改过了（profile 摘要对不上），只提醒不中断。", + "en": "warning: settings changed after the plan was made (profile hash differs); continuing anyway.", + }, +} + + +class PlanError(ValueError): + """The approval file violates the execution contract.""" + + +class ExecutionRefused(PlanError): + """The batch is refused as a whole, on purpose, before anything is touched.""" + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key) or REVIEW_TEXT.get(key) or CLI_TEXT.get(key) or {} + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class ValidatedPlan: + plan: Dict[str, Any] + root: Path + home: Path + lang: str + actions: List[Dict[str, Any]] + by_id: Dict[str, Dict[str, Any]] + overrides: Dict[str, str] + allow_permanent_delete: bool + trash_backend: str + + +@dataclass +class Preflight: + action: Dict[str, Any] + ok: bool + status: str + detail: str + + +@dataclass +class ApplyReport: + run_id: str + dry_run: bool + results: List[Dict[str, Any]] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + manifest: Optional[Path] = None + tag_list: Optional[Path] = None + audit: Optional[Path] = None + review_log: Optional[Path] = None + approved_copy: Optional[Path] = None + recheck_actions: Optional[int] = None + trash_backend: str = trash_module.NONE + stopped: bool = False + + def status_of(self, action_id: str) -> Optional[str]: + for record in self.results: + if record.get("action_id") == action_id: + return str(record.get("status")) + return None + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def plan_home(plan: Dict[str, Any], home: Optional[Path] = None) -> Path: + """Recover the home directory the plan was written against. + + The arithmetic lives in :func:`carl_file_organizer.paths.plan_home`, which the + review page needs too; this name stays because the executor's callers know it. + """ + + return paths.plan_home(plan, home) + + +def _localized(value: Any, lang: str) -> str: + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or next(iter(value.values()), "")) + return "" if value is None else str(value) + + +def open_handles( + path: Path, *, timeout_s: float = DEFAULT_LSOF_TIMEOUT_S +) -> List[Dict[str, str]]: + """Processes holding ``path`` open, via ``lsof``. No lsof means no answer. + + This is the single-path spelling used just before a file is touched; the + plan-time batch lives in :mod:`carl_file_organizer.guard`. Both skip the same + indexing and preview daemons, via ``guard.is_observer``. + """ + + executable = shutil.which("lsof") + if not executable: + return [] + try: + result = subprocess.run( + [executable, "-F", "pcn", "--", str(path)], + capture_output=True, + text=True, + timeout=timeout_s, + ) + except (OSError, subprocess.SubprocessError): + return [] + if result.returncode not in (0, 1): + return [] + handles: List[Dict[str, str]] = [] + pid = "" + command = "" + for line in result.stdout.splitlines(): + tag, value = line[:1], line[1:].strip() + if tag == "p": + pid = value + command = "" + elif tag == "c": + command = value + elif tag == "n" and pid and not is_observer(command): + handles.append({"pid": pid, "command": command, "name": value}) + return handles + + +def _count_entries( + path: Path, *, cap: int = DIR_ENTRY_CAP, budget_s: float = DIR_TIME_BUDGET_S +) -> Tuple[int, bool]: + """Bounded recursive entry count; the flag says the count hit a limit.""" + + deadline = time.monotonic() + budget_s + total = 0 + stack = [Path(path)] + while stack: + current = stack.pop() + try: + entries = list(os.scandir(str(current))) + except OSError: + continue + for entry in entries: + total += 1 + if total >= cap or time.monotonic() > deadline: + return total, True + try: + if entry.is_dir(follow_symlinks=False): + stack.append(Path(entry.path)) + except OSError: + continue + return total, False + + +def _shape(path: Path) -> Dict[str, Any]: + """The facts a move has to preserve, captured just before it happens.""" + + target = Path(path) + if target.is_dir() and not target.is_symlink(): + count, truncated = _count_entries(target) + return {"kind": "dir", "entries": count, "truncated": truncated} + try: + size = target.stat().st_size + except OSError: + size = -1 + return {"kind": "file", "size": size} + + +def verify_move(src: Path, dst: Path, before: Dict[str, Any]) -> Tuple[bool, str]: + """Did the move actually land? Returns ``(ok, detail)`` and never raises.""" + + source = Path(src) + destination = Path(dst) + if not os.path.lexists(str(destination)): + return False, text("verify_dst_missing", "en") + if os.path.lexists(str(source)): + return False, text("verify_src_left", "en") + + if before.get("kind") == "dir": + after, after_truncated = _count_entries(destination) + if before.get("truncated") and after_truncated: + return True, "" + if after != before.get("entries"): + return ( + False, + text("verify_entries", "en", before=before.get("entries"), after=after), + ) + return True, "" + + try: + after_size = destination.stat().st_size + except OSError as error: + return False, str(error) + if int(before.get("size", -1)) >= 0 and after_size != before["size"]: + return False, text("verify_size", "en", before=before["size"], after=after_size) + return True, "" + + +# -------------------------------------------------------------------------- +# stage one: the approval file as a whole +# -------------------------------------------------------------------------- + + +def validate_plan( + plan: Dict[str, Any], + *, + allow_permanent_delete: bool, + home: Optional[Path] = None, +) -> ValidatedPlan: + """Check the approval file before a single byte moves. + + The order is part of the contract: schema, root, unknown ids, one action per + subject, unapprovable ids, tampering, reroutes, and finally the permanent + delete gate, which refuses the whole batch instead of skipping quietly. + """ + + if not isinstance(plan, dict): + raise PlanError("the approval file must contain a JSON object") + if plan.get("schema_version") != SCHEMA_VERSION: + raise PlanError( + "unsupported schema_version {0!r}: this build only runs version {1}".format( + plan.get("schema_version"), SCHEMA_VERSION + ) + ) + + # An approval exported from the review page carries only ``$HOME`` forms, + # because the page is shareable and absolute paths name the user. Expanding + # them here means every check below, and the whole preflight, keeps working + # on real paths without knowing where the file came from. + home_dir = plan_home(plan, home) + plan = paths.hydrate_absolute(plan, home_dir) + + root_text = str(plan.get("source_root") or "") + if not root_text: + raise PlanError("the plan has no source_root") + root = Path(root_text).expanduser() + if root.is_symlink(): + raise PlanError("the planned folder is a symlink: {0}".format(root)) + if not root.is_dir(): + raise PlanError("the planned folder is unavailable: {0}".format(root)) + + actions = plan.get("actions") + if not isinstance(actions, list): + raise PlanError("actions must be a list") + approved = plan.get("approved_action_ids") + if not isinstance(approved, list): + raise PlanError("approved_action_ids must be a list") + + by_id: Dict[str, Dict[str, Any]] = {} + for action in actions: + if not isinstance(action, dict) or not action.get("id"): + raise PlanError("every action needs an id") + by_id[str(action["id"])] = action + + approved_ids: List[str] = [] + seen: Set[str] = set() + for entry in approved: + key = str(entry) + if key in seen: + continue + seen.add(key) + approved_ids.append(key) + + unknown = [key for key in approved_ids if key not in by_id] + if unknown: + raise PlanError("approval contains unknown action ids: {0}".format(sorted(unknown))) + + lang = str(plan.get("lang") or "en") + + chosen: List[Dict[str, Any]] = [] + subjects: Dict[str, str] = {} + for key in approved_ids: + action = by_id[key] + subject = str(action.get("subject_id") or "") + if subject and subject in subjects: + raise PlanError( + "two approved actions target the same item: {0} and {1}".format( + subjects[subject], key + ) + ) + subjects[subject] = key + chosen.append(action) + + for action in chosen: + if not action.get("approvable", False): + raise PlanError( + "action {0} is not approvable ({1})".format( + action.get("id"), action.get("rule") + ) + ) + kind = str(action.get("kind") or "") + if kind not in SUPPORTED_KINDS: + raise PlanError( + "unsupported action kind {0!r} for {1}".format(kind, action.get("id")) + ) + _check_not_tampered(action, root, home_dir) + + working = [copy.deepcopy(action) for action in chosen] + working_by_id = {str(action["id"]): action for action in working} + overrides = _apply_overrides(plan, by_id, working_by_id, root, home_dir) + + if not allow_permanent_delete: + for action in working: + if action.get("kind") == "delete": + raise ExecutionRefused(text("delete_no_flag", lang)) + + capabilities = plan.get("capabilities") or {} + backend = str(capabilities.get("trash_backend") or trash_module.NONE) + + return ValidatedPlan( + plan=plan, + root=paths.realpath(root), + home=home_dir, + lang=lang, + actions=working, + by_id=by_id, + overrides=overrides, + allow_permanent_delete=allow_permanent_delete, + trash_backend=backend, + ) + + +def _check_not_tampered(action: Dict[str, Any], root: Path, home: Path) -> None: + """Recompute the action id from the very fields it was derived from.""" + + destination = action.get("destination") + portable = paths.portable(Path(str(destination)), home) if destination else None + expected = ids.action_id( + str(action.get("subject_id") or ""), str(action.get("kind") or ""), portable + ) + if expected != str(action.get("id")): + raise PlanError( + "action {0} was altered after the plan was written".format(action.get("id")) + ) + if destination is not None and not paths.inside_root(Path(str(destination)).parent, root): + raise PlanError( + "action {0} points outside the folder being organized".format(action.get("id")) + ) + + +def _apply_overrides( + plan: Dict[str, Any], + by_id: Dict[str, Dict[str, Any]], + working_by_id: Dict[str, Dict[str, Any]], + root: Path, + home: Path, +) -> Dict[str, str]: + """Reroute pending items, and only through the plan's own names table.""" + + raw = plan.get("overrides") or [] + if not isinstance(raw, list): + raise PlanError("overrides must be a list") + names = plan.get("names") or {} + if not isinstance(names, dict): + raise PlanError("names must be an object") + + resolved: Dict[str, str] = {} + for entry in raw: + if not isinstance(entry, dict): + raise PlanError("every override must be an object") + action_key = str(entry.get("action_id") or "") + key = str(entry.get("destination_key") or "") + if action_key not in by_id: + raise PlanError("override names an unknown action: {0}".format(action_key)) + original = by_id[action_key] + if original.get("kind") != "move" or not original.get("reroutable", False): + raise PlanError("action {0} cannot be rerouted".format(action_key)) + if str(original.get("rule") or "") not in REROUTABLE_RULES: + raise PlanError( + "action {0} has rule {1!r}, which is not reroutable".format( + action_key, original.get("rule") + ) + ) + if key not in names: + raise PlanError( + "override destination_key {0!r} is not in the plan's names table".format(key) + ) + filename = str(original.get("filename") or Path(str(original.get("source"))).name) + destination = Path(root) / str(names[key]) / filename + if not paths.inside_root(destination.parent, root): + raise PlanError("override for {0} escapes the folder".format(action_key)) + resolved[action_key] = key + target = working_by_id.get(action_key) + if target is not None: + target["destination"] = str(destination) + target["destination_portable"] = paths.portable(destination, home) + target["destination_key"] = key + target["rerouted"] = True + return resolved + + +# -------------------------------------------------------------------------- +# stage two: one item, against the filesystem as it is right now +# -------------------------------------------------------------------------- + + +def preflight_action( + action: Dict[str, Any], + root: Path, + config: Any, + *, + lang: str = "en", + open_handles_fn: Optional[Callable[[Path], Sequence[Any]]] = None, + duplicate_survivors: Optional[Iterable[str]] = None, + allow_permanent_delete: bool = True, +) -> Preflight: + """Everything that must still be true a moment before acting.""" + + source = Path(str(action.get("source") or "")) + kind = str(action.get("kind") or "") + + if not os.path.lexists(str(source)): + return Preflight(action, False, "skipped", text("source_missing", lang)) + if source.is_symlink() or paths.has_symlink_component(source, Path(root)): + return Preflight(action, False, "refused", text("source_symlink", lang)) + if not paths.inside_root(source, Path(root)): + return Preflight(action, False, "refused", text("outside_root", lang)) + + label = paths.is_forbidden(source, config) + if label: + return Preflight(action, False, "refused", text("forbidden", lang, label=label)) + + changed = _subject_changed(action, source, lang) + if changed is not None: + return Preflight(action, False, "skipped", changed) + + if kind == "move": + destination_text = action.get("destination") + if not destination_text: + return Preflight(action, False, "refused", text("destination_missing", lang)) + destination = Path(str(destination_text)) + if not paths.inside_root(destination.parent, Path(root)): + return Preflight(action, False, "refused", text("outside_root", lang)) + if paths.has_symlink_component(destination.parent, Path(root)): + return Preflight(action, False, "refused", text("source_symlink", lang)) + if os.path.lexists(str(destination)): + return Preflight(action, False, "conflict", text("destination_exists", lang)) + + if kind == "delete": + refusal = _delete_gate( + action, + source, + config, + lang=lang, + duplicate_survivors=duplicate_survivors, + allow_permanent_delete=allow_permanent_delete, + ) + if refusal is not None: + return Preflight(action, False, "refused", refusal) + + handler = open_handles_fn if open_handles_fn is not None else open_handles + try: + handles = list(handler(source) or []) + except Exception: # noqa: BLE001 - a guard that crashes must not block the batch + handles = [] + if handles: + who = ", ".join( + "{0}({1})".format(item.get("command") or item.get("name", "?"), item.get("pid", "?")) + if isinstance(item, dict) + else str(item) + for item in handles[:3] + ) + return Preflight(action, False, "skipped", text("in_use", lang, who=who)) + + return Preflight(action, True, "ready", text("ready", lang)) + + +def _subject_changed(action: Dict[str, Any], source: Path, lang: str = "en") -> Optional[str]: + """``None`` when the item is still exactly what the plan looked at.""" + + try: + stat = source.stat() + except OSError: + return text("source_missing", lang) + + live_kind = "dir" if source.is_dir() else "file" + recorded_kind = str(action.get("subject_kind") or live_kind) + if live_kind != recorded_kind: + return text("kind_changed", lang) + + recorded = str(action.get("subject_id") or "") + if not recorded: + return None + + live = ids.subject_id(source, live_kind, stat.st_size, stat.st_mtime_ns) + if live == recorded: + return None + + # Directory sizes are computed with a cap, so a planner may well have + # recorded a walked size rather than the inode size. Modification time is + # the real guard for a directory, so that second spelling is allowed, and + # only that one. + if live_kind == "dir": + try: + recorded_size = int(action.get("size_bytes") or 0) + except (TypeError, ValueError): + recorded_size = 0 + alternate = ids.subject_id(source, live_kind, recorded_size, stat.st_mtime_ns) + if alternate == recorded: + return None + + return text("changed", lang) + + +def _delete_gate( + action: Dict[str, Any], + source: Path, + config: Any, + *, + lang: str, + duplicate_survivors: Optional[Iterable[str]], + allow_permanent_delete: bool, +) -> Optional[str]: + """The second door in front of a permanent delete. ``None`` means open.""" + + if not allow_permanent_delete: + return text("delete_no_flag", lang) + + tier = str(action.get("tier") or "") + profile = getattr(config, "profile", {}) or {} + regenerable = (profile.get("regenerable") or {}).get("basenames") or [] + + if tier == "regenerable": + if source.name not in regenerable: + return text("delete_not_regenerable", lang, name=source.name) + return None + + if tier == "duplicate": + survivors = set(str(item) for item in (duplicate_survivors or [])) + subject = str(action.get("subject_id") or "") + if not (survivors - {subject}): + return text("delete_no_survivor", lang) + return None + + if tier == "cold": + if source.is_dir(): + return text("delete_cold_dir", lang) + return None + + return text("delete_bad_tier", lang, tier=tier or "unknown") + + +# -------------------------------------------------------------------------- +# stage three: act +# -------------------------------------------------------------------------- + + +def execute_action( + action: Dict[str, Any], + *, + dry_run: bool = False, + trash_backend: str = trash_module.NONE, + home: Optional[Path] = None, + root: Optional[Path] = None, + config: Any = None, + lang: str = "en", + trash_fn: Optional[Callable[..., None]] = None, +) -> Tuple[str, str]: + """Perform one approved action. Returns ``(status, detail)`` and never raises.""" + + kind = str(action.get("kind") or "") + source = Path(str(action.get("source") or "")) + + if dry_run: + return "dry-run", text("dry_run", lang) + + try: + if kind == "move": + destination = Path(str(action.get("destination"))) + if os.path.lexists(str(destination)): + return "conflict", text("destination_exists", lang) + destination.parent.mkdir(parents=True, exist_ok=True) + before = _shape(source) + shutil.move(str(source), str(destination)) + ok, detail = verify_move(source, destination, before) + if ok: + return "moved", text("moved", lang) + try: + if os.path.lexists(str(destination)) and not os.path.lexists(str(source)): + shutil.move(str(destination), str(source)) + except (OSError, shutil.Error) as error: + return "failed", text( + "verify_failed_stuck", lang, detail=detail, error=error + ) + return "failed", text("verify_failed", lang, detail=detail) + + if kind == "trash": + sender = trash_fn if trash_fn is not None else trash_module.move_to_trash + sender(source, backend=trash_backend, lang=lang) + return "trashed", text("trashed", lang) + + if kind == "delete": + if source.is_dir() and not source.is_symlink(): + if root is not None and paths.has_symlink_component(source, Path(root)): + return "refused", text("source_symlink", lang) + if config is not None and paths.is_forbidden(source, config): + return "refused", text("forbidden", lang, label="recheck") + shutil.rmtree(str(source)) + else: + os.unlink(str(source)) + return "deleted", text("deleted", lang) + + except trash_module.TrashError as error: + return "failed", str(error) + except Exception as error: # noqa: BLE001 - one bad item must not kill the batch + return "failed", text("error", lang, error=error) + + return "refused", text("delete_bad_tier", lang, tier=kind) + + +# -------------------------------------------------------------------------- +# the whole run +# -------------------------------------------------------------------------- + + +def _audit_record( + action: Dict[str, Any], + *, + status: str, + detail: str, + run_id: str, + home: Path, + manifest: str, + trash_backend: str, + tagged: bool = False, + now: Optional[datetime] = None, +) -> Dict[str, Any]: + destination = action.get("destination") + return { + "timestamp": (now or datetime.now().astimezone()).isoformat(timespec="seconds"), + "run_id": run_id, + "action_id": action.get("id"), + "subject_id": action.get("subject_id"), + "kind": action.get("kind"), + "source": paths.portable(Path(str(action.get("source"))), home), + "destination": paths.portable(Path(str(destination)), home) if destination else None, + "status": status, + "detail": detail, + "size_bytes": action.get("size_bytes", 0), + "manifest": manifest, + "tagged": tagged, + "trash_backend": trash_backend, + } + + +def _restore_method(action: Dict[str, Any], lang: str, home: Path) -> str: + kind = str(action.get("kind") or "") + if kind == "move" and action.get("destination"): + destination = paths.portable(Path(str(action["destination"])), home) + parent = paths.portable(Path(str(action["source"])).parent, home) + return 'mv "{0}" "{1}/"'.format(destination, parent) + if kind == "trash": + return i18n.reason("restore_trash", lang) + if kind == "delete": + return i18n.reason("restore_delete", lang) + return "" + + +def _manifest_row( + action: Dict[str, Any], status: str, *, moved_at: str, home: Path, lang: str +) -> Dict[str, Any]: + destination = action.get("destination") + return { + "moved_at": moved_at, + "status": STATUS_MANIFEST.get(status, status.upper()), + "original_path": paths.portable(Path(str(action.get("source"))), home), + "target_path": paths.portable(Path(str(destination)), home) if destination else "", + "size_bytes": action.get("size_bytes", 0), + "reason": _localized(action.get("reason"), lang), + "restore_method": _restore_method(action, lang, home), + "kind": action.get("kind"), + "action_id": action.get("id"), + } + + +def _tag_moved(destination: Path, transient: str) -> bool: + """Put the temporary tag on a moved item; a failure is a warning, not a stop.""" + + try: + if tags_module.add_tag(destination, transient): + return True + # Already carrying the tag from an earlier run counts as tagged, so the + # path still reaches the tag list that clear-tags reads. + return transient in tags_module.user_tags(destination, ()) + except Exception as error: # noqa: BLE001 - tagging never stops a run + print("tag_warning: {0}\t{1}".format(destination, error), file=sys.stderr) + return False + + +def _series_for(actions: Sequence[Dict[str, Any]]) -> str: + kinds = {str(action.get("kind")) for action in actions} + if kinds and kinds <= {"trash", "delete"}: + return "removals" + if any(str(action.get("rule")) == "conflict" for action in actions): + return "conflict-moves" + return "moves" + + +def _survivors(validated: ValidatedPlan) -> Dict[str, Set[str]]: + """Per group, the subjects that are kept rather than trashed or deleted.""" + + members: Dict[str, Set[str]] = {} + disposed: Dict[str, Set[str]] = {} + for group in validated.plan.get("groups") or []: + if not isinstance(group, dict): + continue + group_id = str(group.get("group_id") or "") + members[group_id] = {str(item) for item in (group.get("members") or [])} + disposed.setdefault(group_id, set()) + for action in validated.actions: + group_id = str(action.get("group_id") or "") + if not group_id: + continue + if action.get("kind") in ("trash", "delete"): + disposed.setdefault(group_id, set()).add(str(action.get("subject_id"))) + return { + group_id: (members.get(group_id, set()) - gone) + for group_id, gone in disposed.items() + } + + +def _review_bullets( + report: ApplyReport, + validated: ValidatedPlan, + *, + lang: str, + transient: str, + home: Path, +) -> List[str]: + """The sentences a person reads a week later to remember what happened.""" + + def names(records: Sequence[Dict[str, Any]], limit: int = 6) -> str: + shown = [Path(str(item.get("source"))).name for item in records[:limit]] + joined = "、".join(shown) if lang == "zh" else ", ".join(shown) + if len(records) > len(shown): + joined += "…" if lang == "zh" else ", ..." + return joined + + by_status: Dict[str, List[Dict[str, Any]]] = {} + for record in report.results: + by_status.setdefault(str(record.get("status")), []).append(record) + + bullets: List[str] = [] + moved = by_status.get("moved", []) + if moved: + bullets.append( + text( + "log_moved", + lang, + count=len(moved), + manifest=paths.portable(report.manifest, home) if report.manifest else "", + ) + ) + if report.tag_list is not None: + bullets.append( + text( + "log_tagged", + lang, + tag=transient, + list=paths.portable(report.tag_list, home), + ) + ) + + sensitive = [ + record + for record in moved + if str(validated.by_id.get(str(record.get("action_id")), {}).get("tier")) == "sensitive" + ] + if sensitive: + destination = Path(str(sensitive[0].get("destination") or "")).parent + bullets.append( + text( + "log_sensitive", + lang, + count=len(sensitive), + dest=str(destination), + names=names(sensitive), + ) + ) + + conflicts = by_status.get("conflict", []) + if conflicts: + bullets.append(text("log_conflict", lang, names=names(conflicts))) + + trashed = by_status.get("trashed", []) + if trashed: + bullets.append( + text("log_trashed", lang, count=len(trashed), backend=report.trash_backend) + ) + + deleted = by_status.get("deleted", []) + if deleted: + bullets.append(text("log_deleted", lang, count=len(deleted), names=names(deleted))) + + skipped = by_status.get("skipped", []) + by_status.get("refused", []) + if skipped: + bullets.append(text("log_skipped", lang, count=len(skipped))) + + failed = by_status.get("failed", []) + if failed: + bullets.append(text("log_failed", lang, count=len(failed))) + + if report.recheck_actions is not None: + bullets.append(text("log_recheck", lang, count=report.recheck_actions)) + + return bullets + + +def apply_approved_plan( + plan: Dict[str, Any], + *, + dry_run: bool = False, + allow_permanent_delete: bool = False, + now: Optional[datetime] = None, + recheck_fn: Optional[Callable[[], int]] = None, + continue_on_error: bool = False, + home: Optional[Path] = None, + config: Any = None, + audit: Optional[Path] = None, + open_handles_fn: Optional[Callable[[Path], Sequence[Any]]] = None, + trash_fn: Optional[Callable[..., None]] = None, +) -> ApplyReport: + """Run every approved action, in order, and write the paper trail. + + The order is deliberate. The manifest lands before anything moves, so an + interrupted run still leaves a page saying what was about to happen and how + to put it back. Each action appends its own audit line the moment it + finishes, which is what makes the audit the authority when the manifest is + stale. Tagging, the recheck and the human log all wait until the last file + has settled. + """ + + from . import config as config_module + + validated = validate_plan( + plan, allow_permanent_delete=allow_permanent_delete, home=home + ) + lang = validated.lang + root = validated.root + home_dir = validated.home + + if config is None: + config = config_module.resolve_config(root, lang=lang) + + moment = now or datetime.now().astimezone() + run_id = manifest_module.timestamp_text(moment) + moved_at = moment.isoformat(timespec="seconds") + + managed = config_module.managed_dir(config) + audit_file = Path(audit) if audit is not None else manifest_module.audit_path(managed) + + report = ApplyReport( + run_id=run_id, dry_run=dry_run, trash_backend=validated.trash_backend + ) + if not validated.actions: + report.counts = {"total": 0} + return report + + manifest_file = manifest_module.manifest_path( + managed, root.name, _series_for(validated.actions), moment + ) + manifest_text = paths.portable(manifest_file, home_dir) + + survivors = _survivors(validated) + checks: Dict[str, Preflight] = {} + for action in validated.actions: + group_id = str(action.get("group_id") or "") + checks[str(action["id"])] = preflight_action( + action, + root, + config, + lang=lang, + open_handles_fn=open_handles_fn, + duplicate_survivors=survivors.get(group_id), + allow_permanent_delete=allow_permanent_delete, + ) + + statuses: Dict[str, str] = {} + for action in validated.actions: + check = checks[str(action["id"])] + statuses[str(action["id"])] = "planned" if check.ok else check.status + + def rows() -> List[Dict[str, Any]]: + return [ + _manifest_row( + action, + statuses[str(action["id"])], + moved_at=moved_at, + home=home_dir, + lang=lang, + ) + for action in validated.actions + ] + + if not dry_run: + managed.mkdir(parents=True, exist_ok=True) + manifest_module.write_manifest(manifest_file, rows()) + report.manifest = manifest_file + + transient = config_module.transient_tag(config) + can_tag = (not dry_run) and tags_module.supported() + + stopped = False + tagged_paths: List[str] = [] + for action in validated.actions: + key = str(action["id"]) + check = checks[key] + if not check.ok: + status, detail = check.status, check.detail + elif stopped: + status, detail = "skipped", text("batch_stopped", lang) + else: + status, detail = execute_action( + action, + dry_run=dry_run, + trash_backend=validated.trash_backend, + home=home_dir, + root=root, + config=config, + lang=lang, + trash_fn=trash_fn, + ) + if status == "failed" and not continue_on_error: + stopped = True + statuses[key] = status + + # Tag the moved item before its audit line is written, so the line can + # tell the truth about whether the tag landed. + tagged = False + if status == "moved" and can_tag and action.get("destination"): + destination = Path(str(action["destination"])) + tagged = _tag_moved(destination, transient) + if tagged: + tagged_paths.append(paths.portable(destination, home_dir)) + + record = _audit_record( + action, + status=status, + detail=detail, + run_id=run_id, + home=home_dir, + manifest=manifest_text if not dry_run else "", + trash_backend=validated.trash_backend, + tagged=tagged, + now=moment, + ) + report.results.append(record) + if not dry_run: + manifest_module.append_audit([record], audit_file) + report.audit = audit_file + + report.stopped = stopped + + if not dry_run: + manifest_module.write_manifest(manifest_file, rows()) + + if tagged_paths: + tag_file = manifest_module.tag_list_path(managed, root.name, moment) + manifest_module.write_tag_list(tag_file, tagged_paths) + report.tag_list = tag_file + + if recheck_fn is not None: + try: + report.recheck_actions = int(recheck_fn()) + except Exception as error: # noqa: BLE001 - a recheck must not undo a good run + print("recheck_warning: {0}".format(error), file=sys.stderr) + + counts: Dict[str, int] = {"total": len(report.results)} + for record in report.results: + status = str(record.get("status")) + counts[status] = counts.get(status, 0) + 1 + report.counts = counts + + if not dry_run: + bullets = _review_bullets( + report, validated, lang=lang, transient=transient, home=home_dir + ) + if bullets: + report.review_log = manifest_module.append_review_log( + managed, + lang, + moment.date(), + bullets, + name=config_module.review_log_name(config), + ) + + return report + + +# -------------------------------------------------------------------------- +# apply subcommand +# -------------------------------------------------------------------------- + + +def _recheck_with_planner( + root: Path, config: Any, moment: datetime +) -> Optional[Callable[[], int]]: + """A real recheck when the planner is available, and nothing when it is not.""" + + try: + from . import planner as planner_module + except ImportError: + return None + build = getattr(planner_module, "build_plan", None) + if build is None: + return None + + def recheck() -> int: + import inspect + + options: Dict[str, Any] = {} + try: + parameters = inspect.signature(build).parameters + except (TypeError, ValueError): + parameters = {} + if "config" in parameters: + options["config"] = config + if "now" in parameters: + options["now"] = moment + if "dir_sizes" in parameters: + options["dir_sizes"] = False + fresh = build(root, **options) + actions = fresh.get("actions") or [] + return len([item for item in actions if item.get("approvable", True)]) + + return recheck + + +def run(args: Any) -> int: + """``carl-file-organizer apply [--dry-run] [--allow-permanent-delete]``.""" + + from . import config as config_module + + approval_path = Path(args.approval).expanduser() + plan = paths.hydrate_absolute(json.loads(approval_path.read_text(encoding="utf-8"))) + if not isinstance(plan, dict): + raise PlanError("the approval file must contain a JSON object") + + root = Path(str(plan.get("source_root") or "")).expanduser() + profile_name = (plan.get("profile") or {}).get("name") + config = config_module.resolve_config(root, lang=plan.get("lang"), profile=profile_name) + lang = config.lang + + recorded_hash = (plan.get("profile") or {}).get("hash") + if recorded_hash and recorded_hash != getattr(config, "profile_hash", ""): + print(text("cli_profile_drift", lang), file=sys.stderr) + + moment = datetime.now().astimezone() + report = apply_approved_plan( + plan, + dry_run=bool(args.dry_run), + allow_permanent_delete=bool(args.allow_permanent_delete), + now=moment, + recheck_fn=None + if args.dry_run + else _recheck_with_planner(config.root, config, moment), + config=config, + audit=Path(args.audit).expanduser() if getattr(args, "audit", None) else None, + ) + + for record in report.results: + print( + "{0}\t{1}\t{2}".format( + record.get("status"), record.get("source"), record.get("detail") + ) + ) + + print( + text( + "cli_summary", + lang, + total=report.counts.get("total", 0), + moved=report.counts.get("moved", 0), + trashed=report.counts.get("trashed", 0), + deleted=report.counts.get("deleted", 0), + skipped=report.counts.get("skipped", 0) + report.counts.get("refused", 0), + failed=report.counts.get("failed", 0), + ) + ) + + if report.dry_run: + print(text("cli_dry_run", lang)) + else: + if report.manifest: + print(text("cli_manifest", lang, path=report.manifest)) + if report.tag_list: + print(text("cli_tag_list", lang, path=report.tag_list)) + if report.audit: + print(text("cli_audit", lang, path=report.audit)) + if report.review_log: + print(text("cli_review_log", lang, path=report.review_log)) + copy_target = _keep_approval(approval_path, config, report.run_id) + if copy_target is not None: + report.approved_copy = copy_target + print(text("cli_approved_copy", lang, path=copy_target)) + + return 2 if (report.counts.get("failed", 0) or report.counts.get("refused", 0)) else 0 + + +def _keep_approval(approval: Path, config: Any, run_id: str) -> Optional[Path]: + """Tuck the approval file into the managed folder so it stops being clutter.""" + + from . import config as config_module + + try: + managed = config_module.managed_dir(config) + managed.mkdir(parents=True, exist_ok=True) + target = managed / "approved-{0}.json".format(run_id) + if paths.realpath(approval) == paths.realpath(target): + return target + shutil.copy2(str(approval), str(target)) + return target + except OSError as error: + print("keep_approval_warning: {0}".format(error), file=sys.stderr) + return None + + +#: The audit writer lives in manifest.py now; this keeps the old import working. +append_audit = manifest_module.append_audit diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py new file mode 100644 index 0000000..483fb6b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py @@ -0,0 +1,327 @@ +"""Pairing, duplicate detection and regenerable build output. + +Three questions get answered here, all of them read-only: + + pair_archives is this .zip the same thing as the folder next to it + find_duplicates are two files the same file wearing different names + find_regenerable is this subfolder something a build can recreate + +Nothing here decides anything. Each function returns raw groups, and the +planner turns them into actions and options. +""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import classifier +from .ids import subject_id as _subject_id +from .scanner import DirStats, Entry, dir_stats_bounded, split_suffix + +#: How a name gains a copy marker: "foo (1).pdf", "foo_1.pdf", "Archive 2". +COPY_PATTERNS = ( + re.compile(r"^(?P.+?) \((?P\d+)\)(?P\.[^.]+)?$"), + re.compile(r"^(?P.+?)_(?P\d+)(?P\.[^.]+)?$"), + re.compile(r"^(?P.+?) (?P\d+)$"), +) + +DEFAULT_HASH_MAX_BYTES = 2 * 1024 * 1024 * 1024 + + +@dataclass +class RawGroup: + kind: str # "pair" | "duplicate" + stem: str + evidence: str # "name" | "sha256" | "name-pattern" + members: List[Entry] + sha256: Optional[str] = None + display: str = "" + group_id: str = "" + + +@dataclass +class RegenerableHit: + path: Path + name: str + owner: Entry # the top level directory it was found inside + relative: str # "web-prototype/node_modules" + dir_stats: DirStats + modified_ns: int = 0 + subject_id: str = "" + age_hours: int = 0 + + +# --------------------------------------------------------------------------- +# name arithmetic +# --------------------------------------------------------------------------- + + +def copy_base(name: str) -> Tuple[str, bool]: + """``"foo (1).pdf"`` -> ``("foo.pdf", True)``; unchanged names return False.""" + + text = str(name) + for pattern in COPY_PATTERNS: + match = pattern.match(text) + if match: + base = match.group("base") + ext = match.groupdict().get("ext") or "" + return base + ext, True + return text, False + + +def _strip_copy_marker(name: str) -> str: + base, _found = copy_base(name) + return base + + +def archive_stem(name: str, config: Any = None) -> Optional[str]: + """Folded stem of an archive file name, or ``None`` when it is not an archive.""" + + text = str(name) + stem, suffix = split_suffix(text) + if not suffix: + return None + if config is not None: + if not classifier.is_archive_suffix(suffix, config): + return None + elif suffix not in {".zip", ".tar.gz", ".tgz", ".tar", ".7z", ".rar", ".gz", ".xz", ".bz2"}: + return None + return _strip_copy_marker(stem).casefold() + + +def dir_stem(name: str) -> str: + """``"Archive (1)"`` and ``"Archive 2"`` both fold down to ``"archive"``.""" + + return _strip_copy_marker(str(name)).casefold() + + +def file_sha256(path: Path, *, max_bytes: int = DEFAULT_HASH_MAX_BYTES) -> Optional[str]: + """Digest of a regular file; ``None`` when it is too big or unreadable.""" + + try: + size = os.path.getsize(str(path)) + except OSError: + return None + if size > max_bytes: + return None + digest = hashlib.sha256() + try: + with open(str(path), "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +# --------------------------------------------------------------------------- +# pairing +# --------------------------------------------------------------------------- + + +def pair_archives(entries: Sequence[Entry], config: Any = None) -> List[RawGroup]: + """An archive and an extracted folder that share a stem belong together.""" + + buckets: Dict[str, Dict[str, List[Entry]]] = {} + for entry in entries: + if entry.kind == "dir": + stem = dir_stem(entry.name) + side = "dirs" + else: + stem = archive_stem(entry.name, config) + side = "files" + if stem is None: + continue + if not stem: + continue + buckets.setdefault(stem, {"dirs": [], "files": []})[side].append(entry) + + groups: List[RawGroup] = [] + for stem in sorted(buckets): + bucket = buckets[stem] + if not bucket["dirs"] or not bucket["files"]: + continue + members = sorted(bucket["dirs"], key=lambda item: item.name.casefold()) + members += sorted(bucket["files"], key=lambda item: item.name.casefold()) + groups.append( + RawGroup( + kind="pair", + stem=stem, + evidence="name", + members=members, + display=members[0].name, + ) + ) + return groups + + +# --------------------------------------------------------------------------- +# duplicates +# --------------------------------------------------------------------------- + + +def find_duplicates( + entries: Sequence[Entry], + *, + hash_duplicates: bool = False, + hash_max_bytes: int = DEFAULT_HASH_MAX_BYTES, + exclude: Optional[Sequence[str]] = None, +) -> List[RawGroup]: + """Byte-identical files first, then names that merely look like copies.""" + + skip = set(exclude or ()) + files = [ + entry + for entry in entries + if entry.kind == "file" and entry.subject_id not in skip and entry.size_bytes > 0 + ] + + groups: List[RawGroup] = [] + hashed: Dict[str, List[Entry]] = {} + claimed: set = set() + + if hash_duplicates: + by_size: Dict[int, List[Entry]] = {} + for entry in files: + by_size.setdefault(entry.size_bytes, []).append(entry) + for size in sorted(by_size): + candidates = by_size[size] + if len(candidates) < 2: + continue + for entry in candidates: + digest = file_sha256(entry.path, max_bytes=hash_max_bytes) + if digest is None: + continue + hashed.setdefault(digest, []).append(entry) + for digest in sorted(hashed): + members = hashed[digest] + if len(members) < 2: + continue + ordered = sorted(members, key=lambda item: item.name.casefold()) + for entry in ordered: + claimed.add(entry.subject_id) + groups.append( + RawGroup( + kind="duplicate", + stem=ordered[0].name, + evidence="sha256", + members=ordered, + sha256=digest, + display=ordered[0].name, + ) + ) + + by_base: Dict[str, List[Entry]] = {} + for entry in files: + if entry.subject_id in claimed: + continue + base, _marked = copy_base(entry.name) + by_base.setdefault(base.casefold(), []).append(entry) + for base in sorted(by_base): + members = by_base[base] + if len(members) < 2: + continue + ordered = sorted(members, key=lambda item: item.name.casefold()) + groups.append( + RawGroup( + kind="duplicate", + stem=ordered[0].name, + evidence="name-pattern", + members=ordered, + display=ordered[0].name, + ) + ) + return groups + + +# --------------------------------------------------------------------------- +# regenerable build output +# --------------------------------------------------------------------------- + + +def find_regenerable( + entries: Sequence[Entry], + config: Any, + *, + max_depth: int = 3, + now: Any = None, + dir_limits: Optional[Dict[str, Any]] = None, +) -> List[RegenerableHit]: + """Walk top level folders looking for ``node_modules`` and friends. + + Depth is counted from the top level folder, so ``proj/node_modules`` is one + and ``proj/a/b/c/node_modules`` is four. ``.git`` and anything in a no-go + zone are never entered, and a match is never descended into. + """ + + from . import paths as paths_module + + limits = dir_limits or {} + hits: List[RegenerableHit] = [] + for entry in entries: + if entry.kind != "dir": + continue + if paths_module.is_forbidden(entry.path, config): + continue + stack: List[Tuple[Path, int]] = [(entry.path, 0)] + while stack: + current, depth = stack.pop() + if depth >= max_depth: + continue + try: + children = sorted(os.scandir(str(current)), key=lambda item: item.name) + except OSError: + continue + for child in children: + name = child.name + try: + if child.is_symlink() or not child.is_dir(follow_symlinks=False): + continue + except OSError: + continue + if name == ".git": + continue + child_path = Path(child.path) + if classifier.is_regenerable_name(name, config): + stats = dir_stats_bounded( + child_path, + int(limits.get("max_entries", 50000)), + float(limits.get("time_budget_s", 1.5)), + ) + try: + child_stat = child.stat(follow_symlinks=False) + modified_ns = child_stat.st_mtime_ns + stat_size = child_stat.st_size + except OSError: + modified_ns = entry.modified_ns + stat_size = 0 + relative = "{0}/{1}".format( + entry.name, child_path.relative_to(entry.path).as_posix() + ) + age_hours = entry.age_hours + if now is not None: + seconds = now.timestamp() - (modified_ns / 1_000_000_000) + age_hours = int(seconds // 3600) if seconds > 0 else 0 + hits.append( + RegenerableHit( + path=child_path, + name=name, + owner=entry, + relative=relative, + dir_stats=stats, + modified_ns=modified_ns, + subject_id=_subject_id(child_path, "dir", stat_size, modified_ns), + age_hours=age_hours, + ) + ) + continue + if paths_module.is_forbidden(child_path, config): + continue + stack.append((child_path, depth + 1)) + hits.sort(key=lambda hit: hit.relative) + return hits diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py new file mode 100644 index 0000000..978c506 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py @@ -0,0 +1,737 @@ +"""The safety net: who else is still using this path. + +Moving a file is reversible, but a launch agent that hard-codes its path is not +going to notice. So before anything is proposed, every candidate is asked four +read-only questions, each with a timeout, each degrading to "unknown" rather +than raising: + + G1 open_handles is a process holding it open right now + G2 config_references does launchd, cron or a shell rc file name it + G3 incoming_symlinks does a symlink somewhere else point into it + G4 dir_shape is it a worktree, a repo, a virtualenv, a project + G5 recently_modified was it touched in the last ten minutes + +Every entry point takes an injection hook (``lsof_runner``, ``reference_files``, +``scan_roots``, ``git_status_runner``) so tests never touch the real machine. +""" + +from __future__ import annotations + +import os +import shutil +import stat as stat_module +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +from . import paths as paths_module + +#: Files that routinely hard-code a path, relative to the home directory. +SHELL_RC_FILES = ( + ".zshrc", + ".zprofile", + ".zshenv", + ".bashrc", + ".bash_profile", + ".profile", +) + +LAUNCH_DIRS_HOME = ("Library/LaunchAgents",) +LAUNCH_DIRS_ABSOLUTE = ("/Library/LaunchAgents", "/Library/LaunchDaemons") + +#: Daemons that merely look at a file. Spotlight opens everything that lands in +#: a download folder within seconds, so without this list a fresh download is +#: always "in use" and the plan proposes nothing at all. These processes read, +#: index or preview; none of them breaks when a path changes. +OBSERVER_COMMANDS = ( + "mdworker", + "mds", + "mdimport", + "mdsync", + "mdbulkimport", + "quicklookd", + "qlmanage", + "QuickLookUIService", + "Spotlight", + "fseventsd", + "backupd", + "XProtect", + "diskimages-helper", + "revisiond", + "distnoted", +) + + +def is_observer(command: str) -> bool: + """True for a daemon that only reads the file and never owns its path.""" + + folded = str(command or "").casefold() + if not folded: + return False + return any(folded.startswith(name.casefold()) for name in OBSERVER_COMMANDS) + + +MAX_REFERENCE_FILES = 400 +MAX_REFERENCE_BYTES = 2 * 1024 * 1024 +SYMLINK_SCAN_BUDGET_S = 2.0 +SYMLINK_SCAN_MAX_ENTRIES = 40000 + + +@dataclass +class OpenHandles: + handles: List[Dict[str, Any]] = field(default_factory=list) + status: str = "ok" # "ok" | "unknown" + + def __bool__(self) -> bool: + return bool(self.handles) + + +@dataclass +class DirShape: + labels: List[str] = field(default_factory=list) + uncommitted: Optional[int] = None + + def __contains__(self, item: object) -> bool: + return item in self.labels + + +# --------------------------------------------------------------------------- +# G1: open file handles +# --------------------------------------------------------------------------- + + +def _default_lsof_runner(argv: Sequence[str], timeout_s: float): + return subprocess.run(list(argv), capture_output=True, text=True, timeout=timeout_s) + + +def _parse_lsof(text: str) -> List[Dict[str, Any]]: + found: List[Dict[str, Any]] = [] + seen = set() + for _name, handle in _parse_lsof_pairs(text): + marker = (handle["command"], handle["pid"]) + if marker in seen: + continue + seen.add(marker) + found.append(handle) + return found + + +def _parse_lsof_pairs(text: str) -> List[Tuple[str, Dict[str, Any]]]: + """``[(reported path, {command, pid})]`` from ``lsof -F pcn`` output.""" + + pairs: List[Tuple[str, Dict[str, Any]]] = [] + pid: Optional[int] = None + command: Optional[str] = None + for line in text.splitlines(): + if not line: + continue + tag, value = line[0], line[1:] + if tag == "p": + try: + pid = int(value) + except ValueError: + pid = None + command = None + elif tag == "c": + command = value + elif tag == "n": + if pid is None or is_observer(command or ""): + continue + pairs.append((value, {"command": command or "", "pid": pid})) + return pairs + + +def lsof_possible() -> bool: + """Can this machine answer "who has that file open" at all? + + Windows never can: there is no lsof, and the closest native answers + (handle.exe, Restart Manager) need a download or an elevated process, so the + honest answer is "unknown" rather than a silent "nobody". That distinction + matters because "nobody has it open" is what lets an item be moved, and + guessing it would be the one lie this tool cannot afford. + """ + + if sys.platform == "win32": + return False + return shutil.which("lsof") is not None + + +def open_handles( + path: Path, + timeout_s: float = 5.0, + *, + lsof_runner: Optional[Callable[..., Any]] = None, + is_dir: Optional[bool] = None, +) -> OpenHandles: + """Ask lsof who has ``path`` open. No lsof, or a timeout, means unknown.""" + + runner = lsof_runner + if runner is None: + if not lsof_possible(): + return OpenHandles([], "unknown") + runner = _default_lsof_runner + + directory = os.path.isdir(str(path)) if is_dir is None else bool(is_dir) + if directory: + argv = ["lsof", "-F", "pcn", "+d", str(path)] + else: + argv = ["lsof", "-F", "pcn", "--", str(path)] + + try: + result = runner(argv, timeout_s) + except subprocess.TimeoutExpired: + return OpenHandles([], "unknown") + except (OSError, subprocess.SubprocessError): + return OpenHandles([], "unknown") + except Exception: # noqa: BLE001 - an injected runner must never break a plan + return OpenHandles([], "unknown") + + stdout = getattr(result, "stdout", "") or "" + returncode = getattr(result, "returncode", 0) + if returncode not in (0, 1): + return OpenHandles([], "unknown") + return OpenHandles(_parse_lsof(stdout), "ok") + + +def probe_open_handles( + file_paths: Sequence[Path], + dir_paths: Sequence[Path] = (), + timeout_s: float = 5.0, + *, + lsof_runner: Optional[Callable[..., Any]] = None, +) -> Dict[str, OpenHandles]: + """One lsof call for all the files, one per directory. + + lsof is slow enough that asking it once per entry turns a plan into a + coffee break, so the files go in a single batch and only directories, + which need ``+d``, are asked one at a time. + """ + + results: Dict[str, OpenHandles] = {} + runner = lsof_runner + if runner is None: + if not lsof_possible(): + for item in list(file_paths) + list(dir_paths): + results[str(item)] = OpenHandles([], "unknown") + return results + runner = _default_lsof_runner + + files = [Path(item) for item in file_paths] + for item in files: + results[str(item)] = OpenHandles([], "ok") + + by_real = {} + for item in files: + try: + by_real[os.path.realpath(str(item))] = str(item) + except OSError: + continue + + chunk = 150 + for start in range(0, len(files), chunk): + batch = files[start : start + chunk] + argv = ["lsof", "-F", "pcn", "--"] + [str(item) for item in batch] + try: + result = runner(argv, timeout_s) + except Exception: # noqa: BLE001 - unknown beats crashing + for item in batch: + results[str(item)] = OpenHandles([], "unknown") + continue + if getattr(result, "returncode", 0) not in (0, 1): + for item in batch: + results[str(item)] = OpenHandles([], "unknown") + continue + for reported, handle in _parse_lsof_pairs(getattr(result, "stdout", "") or ""): + key = by_real.get(reported) or by_real.get(os.path.realpath(reported)) + if key is None: + continue + existing = results.setdefault(key, OpenHandles([], "ok")) + marker = (handle["command"], handle["pid"]) + if marker not in {(h["command"], h["pid"]) for h in existing.handles}: + existing.handles.append(handle) + + for item in dir_paths: + results[str(item)] = open_handles( + Path(item), timeout_s, lsof_runner=runner, is_dir=True + ) + return results + + +# --------------------------------------------------------------------------- +# G2: configuration references +# --------------------------------------------------------------------------- + + +def path_forms(path: Path, home: Optional[Path] = None) -> List[str]: + """The three spellings a config file might use for the same path.""" + + base = Path(home) if home is not None else Path.home() + absolute = str(path) + forms = [absolute] + try: + relative = Path(absolute).relative_to(base) + except ValueError: + return forms + forms.append("~/" + relative.as_posix()) + forms.append("$HOME/" + relative.as_posix()) + return forms + + +def _read_lines(path: Path) -> Optional[List[str]]: + try: + if not path.is_file(): + return None + if path.stat().st_size > MAX_REFERENCE_BYTES: + return None + return path.read_text(encoding="utf-8", errors="replace").splitlines() + except (OSError, ValueError): + return None + + +def default_reference_files(config: Any, home: Optional[Path] = None) -> List[Path]: + """Launch agents, launch daemons, shell rc files, plus the user's own list.""" + + base = Path(home) if home is not None else Path.home() + found: List[Path] = [] + for relative in LAUNCH_DIRS_HOME: + directory = base / relative + try: + found.extend(sorted(directory.glob("*.plist"))) + except OSError: + continue + for absolute in LAUNCH_DIRS_ABSOLUTE: + try: + found.extend(sorted(Path(absolute).glob("*.plist"))) + except OSError: + continue + for name in SHELL_RC_FILES: + found.append(base / name) + + profile = getattr(config, "profile", config) or {} + guard = profile.get("guard") if hasattr(profile, "get") else None + extra = (guard or {}).get("reference_files") or () + for item in extra: + found.append(paths_module.expand_portable(str(item), base).expanduser()) + return found[:MAX_REFERENCE_FILES] + + +def _crontab_text(crontab_runner: Optional[Callable[[], Any]]) -> Optional[List[str]]: + runner = crontab_runner + if runner is None: + if shutil.which("crontab") is None: + return None + + def runner(): # type: ignore[misc] + return subprocess.run( + ["crontab", "-l"], capture_output=True, text=True, timeout=3 + ) + + try: + result = runner() + except Exception: # noqa: BLE001 - no crontab is a normal machine, not an error + return None + if getattr(result, "returncode", 1) != 0: + return None + return (getattr(result, "stdout", "") or "").splitlines() + + +def load_reference_texts( + config: Any, + home: Optional[Path] = None, + *, + reference_files: Optional[Sequence[Any]] = None, + crontab_runner: Optional[Callable[[], Any]] = None, + include_crontab: bool = True, +) -> List[Tuple[str, List[str]]]: + """Read every reference source once, so per-entry matching is pure string work.""" + + base = Path(home) if home is not None else Path.home() + if reference_files is None: + candidates = default_reference_files(config, base) + else: + candidates = [Path(str(item)).expanduser() for item in reference_files] + + texts: List[Tuple[str, List[str]]] = [] + for candidate in candidates: + lines = _read_lines(Path(candidate)) + if lines is None: + continue + texts.append((paths_module.portable(Path(candidate), base), lines)) + if include_crontab: + cron = _crontab_text(crontab_runner) + if cron: + texts.append(("crontab", cron)) + return texts + + +def config_references( + path: Path, + config: Any, + home: Optional[Path] = None, + *, + reference_files: Optional[Sequence[Any]] = None, + crontab_runner: Optional[Callable[[], Any]] = None, + texts: Optional[Sequence[Tuple[str, List[str]]]] = None, +) -> List[Dict[str, Any]]: + """``[{file, line, form}]`` for every config file that names ``path``.""" + + base = Path(home) if home is not None else Path.home() + sources = ( + list(texts) + if texts is not None + else load_reference_texts( + config, + base, + reference_files=reference_files, + crontab_runner=crontab_runner, + include_crontab=reference_files is None, + ) + ) + forms = path_forms(Path(path), base) + found: List[Dict[str, Any]] = [] + for display, lines in sources: + for number, line in enumerate(lines, start=1): + for form in forms: + if form in line: + found.append({"file": display, "line": number, "form": form}) + break + return found + + +# --------------------------------------------------------------------------- +# G3: symlinks pointing in +# --------------------------------------------------------------------------- + + +def default_scan_roots(config: Any, home: Optional[Path] = None) -> List[Path]: + base = Path(home) if home is not None else Path.home() + profile = getattr(config, "profile", config) or {} + guard = profile.get("guard") if hasattr(profile, "get") else None + roots = (guard or {}).get("reference_scan_roots") or () + found: List[Path] = [] + for item in roots: + found.append(paths_module.expand_portable(str(item), base).expanduser()) + return found + + +def symlink_index( + config: Any, + home: Optional[Path] = None, + *, + scan_roots: Optional[Sequence[Any]] = None, + max_depth: int = 2, +) -> List[Tuple[str, Path]]: + """Every symlink under the scan roots, with the real path it resolves to. + + Built once per plan; the per-entry check is then a containment test. + """ + + base = Path(home) if home is not None else Path.home() + if scan_roots is None: + roots = default_scan_roots(config, base) + else: + roots = [Path(str(item)).expanduser() for item in scan_roots] + + skip_names = {"Library", "Applications", ".Trash", "node_modules", ".git"} + started = time.monotonic() + seen_entries = 0 + index: List[Tuple[str, Path]] = [] + visited = set() + + for root in roots: + stack: List[Tuple[Path, int]] = [(Path(root), 0)] + while stack: + current, depth = stack.pop() + try: + key = os.path.realpath(str(current)) + except OSError: + continue + if key in visited: + continue + visited.add(key) + if (time.monotonic() - started) > SYMLINK_SCAN_BUDGET_S: + return index + try: + iterator = list(os.scandir(str(current))) + except OSError: + continue + for item in iterator: + seen_entries += 1 + if seen_entries > SYMLINK_SCAN_MAX_ENTRIES: + return index + try: + if item.is_symlink(): + target = Path(os.path.realpath(item.path)) + index.append((paths_module.portable(Path(item.path), base), target)) + continue + if depth + 1 < max_depth and item.is_dir(follow_symlinks=False): + if item.name in skip_names or item.name.startswith("."): + continue + stack.append((Path(item.path), depth + 1)) + except OSError: + continue + return index + + +def incoming_symlinks( + path: Path, + config: Any, + home: Optional[Path] = None, + *, + scan_roots: Optional[Sequence[Any]] = None, + index: Optional[Sequence[Tuple[str, Path]]] = None, + max_depth: int = 2, +) -> List[str]: + """Portable paths of symlinks whose target lands inside ``path``.""" + + base = Path(home) if home is not None else Path.home() + entries = ( + list(index) + if index is not None + else symlink_index(config, base, scan_roots=scan_roots, max_depth=max_depth) + ) + target = Path(os.path.realpath(str(path))) + found: List[str] = [] + for display, resolved in entries: + if resolved == target: + found.append(display) + continue + try: + resolved.relative_to(target) + except ValueError: + continue + found.append(display) + return sorted(set(found)) + + +# --------------------------------------------------------------------------- +# G4: shape +# --------------------------------------------------------------------------- + + +def _default_git_status_runner(path: Path, timeout_s: float): + return subprocess.run( + ["git", "-C", str(path), "status", "--porcelain"], + capture_output=True, + text=True, + timeout=timeout_s, + ) + + +def dir_shape( + path: Path, + *, + git_status_runner: Optional[Callable[..., Any]] = None, + timeout_s: float = 2.0, + count_uncommitted: bool = True, +) -> DirShape: + """What kind of folder this is: worktree, repo, virtualenv, project.""" + + target = Path(path) + labels: List[str] = [] + uncommitted: Optional[int] = None + try: + if not target.is_dir(): + return DirShape([], None) + except OSError: + return DirShape([], None) + + git_path = target / ".git" + try: + if git_path.is_file(): + labels.append("git-worktree") + elif git_path.is_dir(): + labels.append("git-dir") + except OSError: + pass + + for name in (".venv", "venv", ".tox"): + try: + if (target / name).is_dir(): + labels.append("venv") + break + except OSError: + continue + + try: + if (target / ".env").exists(): + labels.append("dotenv") + except OSError: + pass + + try: + if (target / "package.json").is_file(): + labels.append("node-project") + except OSError: + pass + + if "git-dir" in labels and count_uncommitted: + runner = git_status_runner + if runner is None and shutil.which("git") is not None: + runner = _default_git_status_runner + if runner is not None: + try: + result = runner(target, timeout_s) + if getattr(result, "returncode", 1) == 0: + text = getattr(result, "stdout", "") or "" + uncommitted = len([line for line in text.splitlines() if line.strip()]) + except Exception: # noqa: BLE001 - a slow repo is not a planning failure + uncommitted = None + return DirShape(labels, uncommitted) + + +def file_shape(path: Path) -> List[str]: + """``executable`` and ``shebang`` markers for a plain file.""" + + labels: List[str] = [] + target = Path(path) + try: + mode = target.stat().st_mode + if mode & (stat_module.S_IXUSR | stat_module.S_IXGRP | stat_module.S_IXOTH): + labels.append("executable") + except OSError: + return labels + try: + with open(str(target), "rb") as handle: + if handle.read(2) == b"#!": + labels.append("shebang") + except OSError: + pass + return labels + + +# --------------------------------------------------------------------------- +# G5: freshly touched +# --------------------------------------------------------------------------- + + +def recently_modified(entry: Any, now: datetime, minutes: int = 10) -> bool: + """True when the entry was written inside the last ``minutes``.""" + + if minutes <= 0: + return False + modified_ns = getattr(entry, "modified_ns", None) + if modified_ns is None and isinstance(entry, Mapping): + modified_ns = entry.get("modified_ns") + if modified_ns is None: + return False + delta = now.timestamp() - (float(modified_ns) / 1_000_000_000) + return 0 <= delta < minutes * 60 + + +# --------------------------------------------------------------------------- +# one context per plan +# --------------------------------------------------------------------------- + + +@dataclass +class GuardContext: + home: Path + enabled: bool = True + reference_texts: List[Tuple[str, List[str]]] = field(default_factory=list) + symlinks: List[Tuple[str, Path]] = field(default_factory=list) + lsof_runner: Optional[Callable[..., Any]] = None + lsof_enabled: bool = True + lsof_timeout_s: float = 5.0 + git_status_runner: Optional[Callable[..., Any]] = None + recent_minutes: int = 10 + open_by: Dict[str, OpenHandles] = field(default_factory=dict) + + def prime(self, file_paths: Sequence[Path], dir_paths: Sequence[Path] = ()) -> None: + """Batch the lsof work for a whole scan before any entry is decided.""" + + if not self.enabled or not self.lsof_enabled: + return + self.open_by = probe_open_handles( + file_paths, dir_paths, self.lsof_timeout_s, lsof_runner=self.lsof_runner + ) + + +def build_context( + config: Any, + *, + home: Optional[Path] = None, + enabled: bool = True, + reference_files: Optional[Sequence[Any]] = None, + scan_roots: Optional[Sequence[Any]] = None, + lsof_runner: Optional[Callable[..., Any]] = None, + lsof_enabled: bool = True, + crontab_runner: Optional[Callable[[], Any]] = None, + git_status_runner: Optional[Callable[..., Any]] = None, +) -> GuardContext: + """Collect everything the per-entry checks need, once.""" + + base = Path(home) if home is not None else Path.home() + profile = getattr(config, "profile", config) or {} + guard = (profile.get("guard") if hasattr(profile, "get") else None) or {} + context = GuardContext( + home=base, + enabled=enabled, + lsof_runner=lsof_runner, + lsof_enabled=lsof_enabled, + lsof_timeout_s=float(guard.get("lsof_timeout_s", 5) or 5), + git_status_runner=git_status_runner, + recent_minutes=int(guard.get("recent_minutes", 10) or 0), + ) + if not enabled: + return context + context.reference_texts = load_reference_texts( + config, + base, + reference_files=reference_files, + crontab_runner=crontab_runner, + include_crontab=reference_files is None, + ) + context.symlinks = symlink_index(config, base, scan_roots=scan_roots) + return context + + +def inspect( + path: Path, + config: Any, + context: GuardContext, + *, + is_dir: bool = False, +) -> Dict[str, Any]: + """Run every check for one entry and return the plan.json ``guard`` block.""" + + empty = {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []} + if not context.enabled: + return empty + + handles = OpenHandles([], "unknown") + if context.lsof_enabled: + cached = context.open_by.get(str(path)) + if cached is not None: + handles = cached + else: + handles = open_handles( + path, + context.lsof_timeout_s, + lsof_runner=context.lsof_runner, + is_dir=is_dir, + ) + references = config_references(path, config, context.home, texts=context.reference_texts) + links = incoming_symlinks(path, config, context.home, index=context.symlinks) + if is_dir: + shape = dir_shape(path, git_status_runner=context.git_status_runner) + labels = list(shape.labels) + uncommitted = shape.uncommitted + else: + labels = file_shape(path) + uncommitted = None + block = { + "open_by": handles.handles, + "referenced_in": references, + "incoming_symlinks": links, + "shape": labels, + } + if handles.status == "unknown": + block["open_by_status"] = "unknown" + if uncommitted is not None: + block["uncommitted"] = uncommitted + return block diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py new file mode 100644 index 0000000..18a1012 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py @@ -0,0 +1,355 @@ +"""Bilingual strings: UI copy and plan reason text. + +Two entry points are frozen for every work package: + + t(key, lang, **kw) UI copy (buttons, headings, CLI lines) + reason(key, lang, **kw) one line of plan reasoning + +``both(key, **kw)`` is a convenience that returns ``{"zh": ..., "en": ...}``, +which is exactly the shape plan.json stores in ``action.reason`` / +``action.detail`` / ``group.reason``. + +Rules for editors: keys are only ever added, never renamed or removed; the zh +column avoids straight double quotes and reads like a person talking. +""" + +from __future__ import annotations + +from typing import Any, Dict, Mapping + +LANGS = ("zh", "en") +DEFAULT_LANG = "en" + +#: Reason text attached to actions and groups. +REASONS: Dict[str, Dict[str, str]] = { + # --- extension rules ------------------------------------------------- + "ext_doc": { + "zh": "文档按类型归档到 {dest}", + "en": "Document filed by type into {dest}", + }, + "ext_image": { + "zh": "图片按类型归档到 {dest}", + "en": "Image filed by type into {dest}", + }, + "ext_video": { + "zh": "视频按类型归档到 {dest}", + "en": "Video filed by type into {dest}", + }, + "ext_audio": { + "zh": "音频按类型归档到 {dest}", + "en": "Audio filed by type into {dest}", + }, + "ext_web": { + "zh": "网页文件按类型归档到 {dest}", + "en": "Web file filed by type into {dest}", + }, + "ext_script": { + "zh": "脚本归入工具与导出", + "en": "Script filed under tools and exports", + }, + "ext_data": { + "zh": "数据文件归入数据库与查询", + "en": "Data file filed under database and queries", + }, + "ext_archive": { + "zh": "压缩包静置满 {hours} 小时后归档", + "en": "Archive filed after sitting still for {hours}h", + }, + "ext_installer": { + "zh": "安装包静置满 {hours} 小时后归档", + "en": "Installer filed after sitting still for {hours}h", + }, + "ext_other": { + "zh": "按扩展名规则归档到 {dest}", + "en": "Filed by extension rule into {dest}", + }, + # --- sensitive ------------------------------------------------------- + "sensitive_isolated": { + "zh": "名字看着像凭证，先单独隔离，未读取内容", + "en": "The name looks like a credential, so it is isolated; contents were never read", + }, + "sensitive_manual": { + "zh": "你标成了敏感，移到敏感信息区，未读取内容", + "en": "Marked sensitive by you; moved to the sensitive area, contents were never read", + }, + # --- directories ----------------------------------------------------- + "dir_pending": { + "zh": "整个目录保留原名，先进待判断", + "en": "Whole folder keeps its name and waits in Pending", + }, + "dir_derivative": { + "zh": "看着像某个目录的衍生副本（{hint}），先进待判断", + "en": "Looks like a derived copy ({hint}); waits in Pending", + }, + # --- grouping -------------------------------------------------------- + "pair": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested", + }, + "dup_sha256": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough", + }, + "dup_name": { + "zh": "名字像副本，但内容没有逐字节比对过，默认都留着", + "en": "Names look like copies, but the contents were not compared; both are kept by default", + }, + "regenerable": { + "zh": "构建产物，删掉还能再生成（{restore}）", + "en": "Build output; it can be regenerated ({restore})", + }, + # --- cold storage candidates ---------------------------------------- + "cold_large": { + "zh": "体积较大（{size}），可以考虑处置", + "en": "Large item ({size}); worth a decision", + }, + "cold_installer": { + "zh": "安装包或压缩包用完通常就不需要了", + "en": "Installers and archives are usually disposable once used", + }, + # --- conflicts and unknowns ----------------------------------------- + "conflict": { + "zh": "目标位置已经有同名文件，不覆盖，改放重复待确认", + "en": "The destination already exists; nothing is overwritten, it goes to Duplicates Review", + }, + "unknown_ext": { + "zh": "扩展名不在规则表里，先进待判断，可以在页面上改去向", + "en": "Extension is not in the rule table; it waits in Pending and can be rerouted", + }, + "name_pattern": { + "zh": "命中你自己写的名字规则 {pattern}", + "en": "Matched your own name rule {pattern}", + }, + "tag_group": { + "zh": "按你打的 Finder 标签 {tag} 归组", + "en": "Grouped by your Finder tag {tag}", + }, + # --- holds ----------------------------------------------------------- + "hold_aging": { + "zh": "还在静置期，已放 {age} 小时，满 {needed} 小时再动", + "en": "Still settling: {age}h of the required {needed}h", + }, + "hold_pinned": { + "zh": "你把它挪回顶层过，算常驻，不再动它", + "en": "You put it back at the top level, so it stays put", + }, + "hold_forbidden": { + "zh": "禁刀区，只展示不处理（{label}）", + "en": "No-go zone, shown but never touched ({label})", + }, + "hold_in_use": { + "zh": "现在有进程开着它：{who}，先不动", + "en": "A process still has it open: {who}", + }, + "hold_referenced": { + "zh": "有配置或脚本引用了这个路径：{where}，动它之前先改引用", + "en": "A config or script references this path: {where}; fix the reference first", + }, + "hold_recent": { + "zh": "十分钟内刚改过，先放一放", + "en": "Changed within the last ten minutes; leaving it alone", + }, + "hold_worktree": { + "zh": "这是 git worktree，路径写死在别处，不能搬", + "en": "This is a git worktree; its path is recorded elsewhere and must not move", + }, + "hold_no_trash": { + "zh": "这台机器没有可用的废纸篓后端，处置选项不可用", + "en": "No trash backend on this machine, so disposal is unavailable", + }, + # --- hints ----------------------------------------------------------- + "hint_git_repo": { + "zh": "这是 git 仓库，搬之前建议先 push，未提交 {count} 项", + "en": "Git repository; consider pushing first, {count} uncommitted item(s)", + }, + "hint_venv": { + "zh": "里面有虚拟环境，路径写死在文件里，搬完需要重建", + "en": "Contains a virtualenv with absolute paths baked in; rebuild after moving", + }, + "hint_dotenv": { + "zh": "里面有 .env，搬之前确认没有别的程序在读它", + "en": "Contains a .env; check nothing else reads it before moving", + }, + "hint_derivative": { + "zh": "名字里带 {hint}，像是临时副本", + "en": "Name contains {hint}, which looks like a scratch copy", + }, + # --- details --------------------------------------------------------- + "detail_aged": { + "zh": "静置 {age} 小时，已满 {needed} 小时", + "en": "aged {age}h, past the {needed}h threshold", + }, + "detail_sensitive": { + "zh": "命中敏感词 {needle}，不看静置期", + "en": "matched {needle}; the settling period is skipped", + }, + "detail_pair": { + "zh": "和 {other} 同名成对，一起搬进 {dest}", + "en": "pairs with {other} by name; both go to {dest}", + }, + "detail_dup_unverified": { + "zh": "名字像副本，内容没有比对过", + "en": "the name looks like a copy; contents were not compared", + }, + "detail_referenced_warning": { + "zh": "别处引用了它（{where}），你让它照常处理，动完记得改引用", + "en": "It is referenced elsewhere ({where}); you allowed it anyway, so fix the reference afterwards", + }, + "detail_dir_truncated": { + "zh": "目录太大，统计到上限就停了，体积是下限值", + "en": "The folder is large; counting stopped at the cap, so the size is a lower bound", + }, + "detail_no_trash": { + "zh": "这台机器没有废纸篓后端，只给了永久删除", + "en": "No trash backend on this machine, so only permanent delete is offered", + }, + # --- restore methods ------------------------------------------------- + "restore_move": { + "zh": "mv {dest} {src}", + "en": "mv {dest} {src}", + }, + "restore_trash": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash", + }, + "restore_delete": { + "zh": "删掉就没了，只对可再生的构建产物用这一项", + "en": "Permanent; only use this for regenerable build output", + }, + # --- folder mess, one line per colour --------------------------------- + "mess_green": { + "zh": "散件 {loose} 件，没有等着你判断的东西，这个目录现在是清爽的", + "en": "{loose} loose items and nothing waiting on you; this folder is in good shape", + }, + "mess_yellow": { + "zh": "散件 {loose} 件，其中 {pending} 件要你看一眼，还没到失控的地步", + "en": "{loose} loose items, {pending} of them want a look from you; not out of hand yet", + }, + "mess_red": { + "zh": "散件 {loose} 件，{pending} 件待判断，另有 {groups} 组重复或可再生的东西，该花点时间了", + "en": ( + "{loose} loose items, {pending} waiting on you and {groups} duplicate or " + "regenerable groups; this one needs real time" + ), + }, +} + +#: UI copy: CLI lines, report headings, buttons. +UI: Dict[str, Dict[str, str]] = { + "app_name": {"zh": "文件整理", "en": "Carl File Organizer"}, + "tagline": { + "zh": "先站好队，再决定谁走", + "en": "Everything lines up first, then you decide what moves", + }, + "cli_plan_written": {"zh": "计划：{path}", "en": "Plan: {path}"}, + "cli_report_written": {"zh": "审核页：{path}", "en": "Review: {path}"}, + "cli_nothing_moved": { + "zh": "本次只读扫描，没有移动或删除任何文件。", + "en": "Read-only scan. Nothing was moved or deleted.", + }, + "cli_dotfile_written": {"zh": "已记住设置：{path}", "en": "Settings remembered: {path}"}, + "cli_notes_merged": { + "zh": "说明已并入计划：{path}（{count} 条提醒）", + "en": "Notes merged into the plan: {path} ({count} warning(s))", + }, + "cli_lang_source": {"zh": "语言：{lang}（来自 {source}）", "en": "Language: {lang} (from {source})"}, + "cli_not_implemented": { + "zh": "子命令 {command} 还没接上：{module} 里没有 run 入口。", + "en": "Subcommand {command} is not wired up yet: {module} provides no run() entry point.", + }, + "cli_lang_ambiguous": { + "zh": "这个目录里中英文分区都有，说不准该用哪套，请加 --lang zh 或 --lang en。", + "en": "This folder has both Chinese and English partitions; pass --lang zh or --lang en.", + }, + "warning_prefix": {"zh": "提醒：", "en": "warning: "}, + "section_pending_moves": {"zh": "待批准移动", "en": "Moves awaiting approval"}, + "section_groups": {"zh": "配对与重复", "en": "Pairs and duplicates"}, + "section_disposal": {"zh": "可再生与冷存候选", "en": "Regenerable and cold candidates"}, + "section_holding": {"zh": "静置中与常驻", "en": "Settling and pinned"}, + "section_forbidden": {"zh": "禁刀区", "en": "No-go zone"}, + "section_undecided": {"zh": "待判断", "en": "Undecided"}, + "button_export": {"zh": "导出批准文件", "en": "Export approved plan"}, + "button_apply": {"zh": "执行已勾选的项", "en": "Apply the checked items"}, + "button_show_delete": {"zh": "显示永久删除选项", "en": "Show permanent delete options"}, + "banner_reversible": { + "zh": "移动只改路径不删内容。哪个软件找不到文件了，用 undo 或者清单里的 mv 命令原路放回。", + "en": "Moving only changes paths. If something cannot find a file, use undo or the mv line in the manifest.", + }, + "banner_do_not_touch": { + "zh": "计划生成之后先别动这些目录，动过的会在执行时自动跳过。", + "en": "Do not touch these folders after the plan is generated; changed ones are skipped at apply time.", + }, + "option_keep_all": {"zh": "都保留，一起搬进待判断", "en": "Keep both, move them together"}, + "option_keep_one": {"zh": "留 {keep}，{drop} 进废纸篓", "en": "Keep {keep}, trash {drop}"}, + "option_keep_one_delete": { + "zh": "留 {keep}，{drop} 永久删除", + "en": "Keep {keep}, permanently delete {drop}", + }, + "option_trash_all": {"zh": "全部进废纸篓", "en": "Trash all of them"}, + "option_delete_all": {"zh": "全部永久删除", "en": "Permanently delete all of them"}, + "option_keep_all_plain": {"zh": "都保留", "en": "Keep both"}, + "cli_report_skipped": { + "zh": "审核页还没接上（{module}），这次只写了计划文件。", + "en": "The review page is not wired up yet ({module}); only the plan file was written.", + }, + "cli_scan_summary": { + "zh": "扫描到 {entries} 项，候选动作 {actions} 条，其中待批准 {approvable} 条。", + "en": "Scanned {entries} item(s), {actions} candidate action(s), {approvable} awaiting approval.", + }, + "status_last_run": {"zh": "最后整理时间", "en": "Last tidy-up"}, + "status_dirty_now": {"zh": "现在是否又脏了", "en": "Messy again?"}, + "status_never": {"zh": "还没整理过", "en": "not tidied yet"}, +} + + +def _lookup(table: Mapping[str, Mapping[str, str]], key: str, lang: str) -> str: + entry = table.get(key) + if entry is None: + return key + chosen = entry.get(lang) + if chosen is None: + chosen = entry.get(DEFAULT_LANG) + if chosen is None: + chosen = next(iter(entry.values()), key) + return chosen + + +def _format(template: str, kw: Mapping[str, Any]) -> str: + if not kw: + return template + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def t(key: str, lang: str = DEFAULT_LANG, /, **kw: Any) -> str: + """UI copy for ``key`` in ``lang``; unknown keys return the key itself. + + ``key`` and ``lang`` are positional-only so a template placeholder may be + called ``lang`` without colliding with the parameter. + """ + + return _format(_lookup(UI, key, lang), kw) + + +def reason(key: str, lang: str = DEFAULT_LANG, /, **kw: Any) -> str: + """Plan reasoning for ``key`` in ``lang``; unknown keys return the key itself.""" + + return _format(_lookup(REASONS, key, lang), kw) + + +def both(key: str, /, **kw: Any) -> Dict[str, str]: + """``{"zh": ..., "en": ...}`` for a reason key, ready to drop into plan.json.""" + + return {lang: reason(key, lang, **kw) for lang in LANGS} + + +def ui_both(key: str, /, **kw: Any) -> Dict[str, str]: + """``{"zh": ..., "en": ...}`` for a UI key.""" + + return {lang: t(key, lang, **kw) for lang in LANGS} + + +def has_reason(key: str) -> bool: + return key in REASONS diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py new file mode 100644 index 0000000..d95edca --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py @@ -0,0 +1,24 @@ +"""Stable identifiers shared by planner, executor, server and undo. + +subject_id identifies a file or directory as it was observed (path + kind + +size + mtime). action_id identifies one proposed operation on that subject. +Both are truncated SHA-256 hex digests; any change to the underlying file +invalidates the id, which is how tampering and drift are detected. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + + +def subject_id(path: Path, kind: str, size: int, modified_ns: int) -> str: + real = os.path.realpath(str(path)) + material = f"{real}\0{kind}\0{size}\0{modified_ns}".encode("utf-8") + return hashlib.sha256(material).hexdigest()[:16] + + +def action_id(subject: str, kind: str, destination_portable: str | None) -> str: + material = f"{subject}\0{kind}\0{destination_portable or ''}".encode("utf-8") + return hashlib.sha256(material).hexdigest()[:16] diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py new file mode 100644 index 0000000..aa82151 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py @@ -0,0 +1,369 @@ +"""Paper trail: TSV manifests, tag lists, audit log, the human review log. + +Four kinds of file live in the managed directory, and every one of them is +written for a person to read first and a program second: + + downloads-moves-.tsv what moved, with a restore command per row + downloads-move-tags-.txt the paths that got the temporary Finder tag + audit.jsonl one JSON line per executed action + 人工调整日志.md / review-log.md a dated summary in plain sentences + +The first seven manifest columns are byte-for-byte the ones the reference +script has been writing since August, so old manifests stay readable and old +readers keep working; ``kind`` and ``action_id`` are appended after them. + +Paths inside these files are always written in ``$HOME/...`` form. +""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass, field +from datetime import date, datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence + +MANIFEST_COLUMNS = ( + "moved_at", + "status", + "original_path", + "target_path", + "size_bytes", + "reason", + "restore_method", + "kind", + "action_id", +) + +#: The seven columns the reference script wrote; anything after them is extra. +LEGACY_COLUMNS = MANIFEST_COLUMNS[:7] + +MANIFEST_SERIES = ("conflict-moves", "moves", "removals", "undo") + +#: Series that count as a tidy-up run when answering "when was this last done". +TIDY_SERIES = ("conflict-moves", "moves", "removals") + +AUDIT_NAME = "audit.jsonl" +TAG_LIST_SERIES = "move-tags" +TIMESTAMP_FORMAT = "%Y-%m-%d_%H-%M-%S" + +REVIEW_LOG_NAMES = {"zh": "人工调整日志.md", "en": "review-log.md"} +REVIEW_LOG_TITLES = {"zh": "# 人工调整日志", "en": "# Review log"} + +_MANIFEST_RE = re.compile( + r"^(?P.+?)-(?Pconflict-moves|moves|removals|undo)-" + r"(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.tsv$" +) + +#: Manifest statuses that mean the action really happened. +DONE_STATUSES = ("MOVED", "TRASHED", "DELETED", "RESTORED") + + +@dataclass +class RunInfo: + """One past run, read from a manifest file name and its rows.""" + + path: Path + series: str + timestamp: Optional[datetime] + moved_at: str + rows: int + done: int + bytes: int = 0 + statuses: Dict[str, int] = field(default_factory=dict) + + +# -------------------------------------------------------------------------- +# names and small helpers +# -------------------------------------------------------------------------- + + +def safe_field(value: Any) -> str: + """Flatten a value so it cannot break the tab separated layout.""" + + return str(value).replace("\t", " ").replace("\r", " ").replace("\n", " ") + + +def slug(name: str) -> str: + """``"Downloads"`` -> ``"downloads"``; used as the manifest name prefix.""" + + folded = str(name).strip().casefold() + folded = re.sub(r"[\s/\\]+", "-", folded) + folded = re.sub(r"[^0-9a-z一-鿿._-]+", "", folded) + folded = folded.strip("-._") + return folded or "folder" + + +def timestamp_text(ts: datetime) -> str: + return ts.strftime(TIMESTAMP_FORMAT) + + +def manifest_path(managed: Path, root_name: str, series: str, ts: datetime) -> Path: + """``downloads-moves-2026-09-02_10-20-11.tsv`` inside the managed directory.""" + + if series not in MANIFEST_SERIES: + raise ValueError("unknown manifest series: {0}".format(series)) + return Path(managed) / "{0}-{1}-{2}.tsv".format( + slug(root_name), series, timestamp_text(ts) + ) + + +def tag_list_path(managed: Path, root_name: str, ts: datetime) -> Path: + """``downloads-move-tags-2026-09-02_10-20-11.txt``.""" + + return Path(managed) / "{0}-{1}-{2}.txt".format( + slug(root_name), TAG_LIST_SERIES, timestamp_text(ts) + ) + + +def audit_path(managed: Path) -> Path: + return Path(managed) / AUDIT_NAME + + +def review_log_name(lang: str) -> str: + return REVIEW_LOG_NAMES.get(lang or "en", REVIEW_LOG_NAMES["en"]) + + +def _atomic_write(path: Path, text: str) -> Path: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + with temporary.open("w", encoding="utf-8", newline="") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + os.replace(str(temporary), str(target)) + return target + + +# -------------------------------------------------------------------------- +# manifests +# -------------------------------------------------------------------------- + + +def write_manifest(path: Path, rows: Sequence[Mapping[str, Any]]) -> Path: + """Write every row under :data:`MANIFEST_COLUMNS`, atomically.""" + + lines = ["\t".join(MANIFEST_COLUMNS)] + for row in rows: + lines.append( + "\t".join(safe_field(row.get(column, "")) for column in MANIFEST_COLUMNS) + ) + return _atomic_write(path, "\n".join(lines) + "\n") + + +def read_manifest(path: Path) -> List[Dict[str, str]]: + """Read a manifest by its own header line, so seven column files still parse.""" + + target = Path(path) + if not target.is_file(): + raise FileNotFoundError(str(target)) + text = target.read_text(encoding="utf-8") + lines = [line for line in text.split("\n") if line != ""] + if not lines: + return [] + header = lines[0].split("\t") + rows: List[Dict[str, str]] = [] + for line in lines[1:]: + fields = line.split("\t") + row: Dict[str, str] = {} + for index, column in enumerate(header): + row[column] = fields[index] if index < len(fields) else "" + rows.append(row) + return rows + + +def write_tag_list(path: Path, portable_paths: Iterable[str]) -> Path: + items = [safe_field(item) for item in portable_paths] + return _atomic_write(path, "\n".join(items) + ("\n" if items else "")) + + +def read_tag_list(path: Path) -> List[str]: + target = Path(path) + if not target.is_file(): + return [] + return [ + line.strip() + for line in target.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# -------------------------------------------------------------------------- +# audit log +# -------------------------------------------------------------------------- + + +def append_audit(results: Iterable[Mapping[str, Any]], path: Path) -> Path: + """Append audit records; this is the authority when a run is interrupted.""" + + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("a", encoding="utf-8") as handle: + for record in results: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + handle.flush() + os.fsync(handle.fileno()) + return target + + +def read_audit(path: Path) -> List[Dict[str, Any]]: + target = Path(path) + if not target.is_file(): + raise FileNotFoundError(str(target)) + records: List[Dict[str, Any]] = [] + for line in target.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped: + continue + try: + record = json.loads(stripped) + except ValueError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +# -------------------------------------------------------------------------- +# human review log +# -------------------------------------------------------------------------- + + +def append_review_log( + managed: Path, + lang: str, + day: date, + bullets: Sequence[str], + *, + name: Optional[str] = None, +) -> Path: + """Add ``bullets`` under a ``## YYYY-MM-DD`` heading, creating what is missing.""" + + target = Path(managed) / (name or review_log_name(lang)) + target.parent.mkdir(parents=True, exist_ok=True) + heading = "## {0}".format(day.isoformat()) + items = ["- {0}".format(safe_field(bullet)) for bullet in bullets if str(bullet).strip()] + if not items: + return target + + if target.is_file(): + lines = target.read_text(encoding="utf-8").split("\n") + else: + lines = [REVIEW_LOG_TITLES.get(lang or "en", REVIEW_LOG_TITLES["en"]), ""] + + start = None + for index, line in enumerate(lines): + if line.strip() == heading: + start = index + break + + if start is None: + while lines and lines[-1].strip() == "": + lines.pop() + lines.extend(["", heading, ""]) + lines.extend(items) + lines.append("") + else: + end = len(lines) + for index in range(start + 1, len(lines)): + if lines[index].startswith("## "): + end = index + break + section = lines[start:end] + while section and section[-1].strip() == "": + section.pop() + section.extend(items) + section.append("") + lines[start:end] = section + + target.write_text("\n".join(lines).rstrip("\n") + "\n", encoding="utf-8") + return target + + +# -------------------------------------------------------------------------- +# last run +# -------------------------------------------------------------------------- + + +def _parse_timestamp(text: str) -> Optional[datetime]: + try: + return datetime.strptime(text, TIMESTAMP_FORMAT) + except (ValueError, TypeError): + return None + + +def _row_bytes(row: Mapping[str, str]) -> int: + try: + return int(str(row.get("size_bytes", "") or 0)) + except ValueError: + return 0 + + +def run_info(path: Path) -> Optional[RunInfo]: + """Read one manifest file into a :class:`RunInfo`, or ``None`` if it is not one.""" + + target = Path(path) + match = _MANIFEST_RE.match(target.name) + if match is None: + return None + try: + rows = read_manifest(target) + except (OSError, ValueError): + rows = [] + statuses: Dict[str, int] = {} + moved_at = "" + total = 0 + done = 0 + for row in rows: + status = str(row.get("status", "")).strip().upper() + statuses[status] = statuses.get(status, 0) + 1 + stamp = str(row.get("moved_at", "")).strip() + if stamp > moved_at: + moved_at = stamp + if status in DONE_STATUSES: + done += 1 + total += _row_bytes(row) + return RunInfo( + path=target, + series=match.group("series"), + timestamp=_parse_timestamp(match.group("ts")), + moved_at=moved_at or match.group("ts").replace("_", " ").replace("-", "-"), + rows=len(rows), + done=done, + bytes=total, + statuses=statuses, + ) + + +def list_runs(managed: Path, *, series: Sequence[str] = TIDY_SERIES) -> List[RunInfo]: + """Every manifest in ``managed``, newest first, judged only by file names.""" + + folder = Path(managed) + if not folder.is_dir(): + return [] + found: List[RunInfo] = [] + try: + names = sorted(entry.name for entry in folder.iterdir() if entry.is_file()) + except OSError: + return [] + for name in names: + match = _MANIFEST_RE.match(name) + if match is None or match.group("series") not in series: + continue + info = run_info(folder / name) + if info is not None: + found.append(info) + found.sort( + key=lambda item: (item.timestamp or datetime.min, item.path.name), reverse=True + ) + return found + + +def latest_run(managed: Path, *, series: Sequence[str] = TIDY_SERIES) -> Optional[RunInfo]: + """The most recent run, read from manifest names only; folder mtimes lie.""" + + runs = list_runs(managed, series=series) + return runs[0] if runs else None diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py new file mode 100644 index 0000000..6ddaeb4 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py @@ -0,0 +1,283 @@ +"""The agent explanation layer: prose laid on top of a plan, changing nothing. + +The rules decide, and they say why in one fixed sentence per rule. That reads +like a form letter, because it is one. An agent that has looked at the folder +can say what a thing actually is, why it is there and what breaks if it goes +away, and that is what this module carries. + +The one rule that makes it safe: notes are text. :func:`merge_notes` copies +nothing but strings, and it copies them into ``plan["notes"]``, never into an +action. No note can add an action, approve one, change a destination, change a +colour on an action, or turn a hold into anything else. The executor still only +knows the ids the planner computed, so a wrong note is a wrong sentence and +nothing worse. + +The single exception is the folder colour: thresholds are a guess about a folder +an agent has actually looked at, so ``notes["mess"]["color"]`` may override +``plan["mess"]["color"]``. The original stays in ``rule_color`` and the plan +records who decided, so nobody has to wonder later. + + plan = merge_notes(plan, json.loads(Path("notes.json").read_text())) + +Unknown ids only warn. A note left over from a previous scan is a stale +sentence, not a reason to refuse to write a report. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from . import i18n +from .planner import COLORS + +#: Version of the notes.json shape. Bumped only when a field is removed. +NOTES_SCHEMA = 1 + +#: The three sentences an agent may write about one action or one group. +ITEM_FIELDS = ("what", "why", "if_removed") + +#: Top level keys notes.json is allowed to carry. +TOP_FIELDS = ("mess", "actions", "groups", "folder_line") + +#: Longest a single sentence may be; longer is truncated, never dropped. +MAX_TEXT = 600 + + +class NotesError(ValueError): + """The notes file is not a JSON object at all. Everything else only warns.""" + + +# --------------------------------------------------------------------------- +# reading and cleaning +# --------------------------------------------------------------------------- + + +def load_notes(path: Path) -> Dict[str, Any]: + """Read notes.json, or raise :class:`NotesError` with a readable line.""" + + try: + raw = Path(path).read_text(encoding="utf-8") + except OSError as error: + raise NotesError("cannot read notes file {0}: {1}".format(path, error)) from error + try: + data = json.loads(raw) + except ValueError as error: + raise NotesError("notes file {0} is not valid JSON: {1}".format(path, error)) from error + if not isinstance(data, dict): + raise NotesError("notes file {0} must hold a JSON object".format(path)) + return data + + +def _text(value: Any) -> Optional[str]: + if not isinstance(value, str): + return None + cleaned = value.strip() + if not cleaned: + return None + return cleaned[:MAX_TEXT] + + +def _clean_item( + raw: Any, where: str, warnings: List[str] +) -> Optional[Dict[str, str]]: + """Keep the three known sentences, drop everything else with a warning.""" + + if not isinstance(raw, dict): + warnings.append("{0}: expected an object, ignored".format(where)) + return None + kept: Dict[str, str] = {} + for key, value in raw.items(): + if key not in ITEM_FIELDS: + warnings.append("{0}: unknown field {1!r}, ignored".format(where, key)) + continue + text = _text(value) + if text is None: + warnings.append("{0}.{1}: not a non-empty string, ignored".format(where, key)) + continue + kept[key] = text + if not kept: + return None + return kept + + +def _clean_map( + raw: Any, + known_ids: Mapping[str, Any], + label: str, + warnings: List[str], +) -> Dict[str, Dict[str, str]]: + if raw is None: + return {} + if not isinstance(raw, dict): + warnings.append("{0}: expected an object keyed by id, ignored".format(label)) + return {} + cleaned: Dict[str, Dict[str, str]] = {} + for key, value in raw.items(): + if not isinstance(key, str): + warnings.append("{0}: non-string id ignored".format(label)) + continue + if key not in known_ids: + warnings.append("{0}: no such id in this plan: {1}".format(label, key)) + continue + item = _clean_item(value, "{0}[{1}]".format(label, key), warnings) + if item is not None: + cleaned[key] = item + return cleaned + + +def _clean_mess(raw: Any, warnings: List[str]) -> Dict[str, Any]: + if raw is None: + return {} + if not isinstance(raw, dict): + warnings.append("mess: expected an object, ignored") + return {} + cleaned: Dict[str, Any] = {} + for key, value in raw.items(): + if key == "color": + if value in COLORS: + cleaned["color"] = value + else: + warnings.append("mess.color: not one of {0}, ignored".format(", ".join(COLORS))) + elif key == "line": + text = _text(value) + if text is None: + warnings.append("mess.line: not a non-empty string, ignored") + else: + cleaned["line"] = text + else: + warnings.append("mess: unknown field {0!r}, ignored".format(key)) + return cleaned + + +# --------------------------------------------------------------------------- +# merging +# --------------------------------------------------------------------------- + + +def validate_notes( + plan: Mapping[str, Any], notes: Mapping[str, Any] +) -> Tuple[Dict[str, Any], List[str]]: + """``(cleaned notes, warnings)``. Nothing here reads the filesystem.""" + + warnings: List[str] = [] + if not isinstance(notes, dict): + raise NotesError("notes must be a JSON object") + + action_ids = { + str(action.get("id")): action for action in (plan.get("actions") or []) if isinstance(action, dict) + } + group_ids = { + str(group.get("group_id")): group for group in (plan.get("groups") or []) if isinstance(group, dict) + } + + for key in notes: + if key not in TOP_FIELDS: + warnings.append("unknown top level field {0!r}, ignored".format(key)) + + cleaned: Dict[str, Any] = { + "schema": NOTES_SCHEMA, + "mess": _clean_mess(notes.get("mess"), warnings), + "actions": _clean_map(notes.get("actions"), action_ids, "actions", warnings), + "groups": _clean_map(notes.get("groups"), group_ids, "groups", warnings), + "folder_line": _text(notes.get("folder_line")), + } + if notes.get("folder_line") is not None and cleaned["folder_line"] is None: + warnings.append("folder_line: not a non-empty string, ignored") + return cleaned, warnings + + +def merge_notes(plan: Mapping[str, Any], notes: Mapping[str, Any]) -> Dict[str, Any]: + """A copy of ``plan`` with ``notes`` merged into ``plan["notes"]``. + + The input plan is not touched. Merging twice is the same as merging once + with the second set of notes: the previous block is replaced, not appended, + so a re-run after fixing a sentence does not leave the old one behind. + """ + + if not isinstance(plan, dict): + raise NotesError("plan must be a JSON object") + cleaned, warnings = validate_notes(plan, notes) + merged = copy.deepcopy(dict(plan)) + cleaned["warnings"] = warnings + merged["notes"] = cleaned + + mess = merged.get("mess") + if isinstance(mess, dict): + # Put the rule's own answer somewhere permanent before anything can + # overwrite it, whether or not the notes carry a colour. + mess.setdefault("rule_color", mess.get("color")) + chosen = cleaned["mess"].get("color") + if chosen: + mess["color"] = chosen + mess["color_source"] = "notes" + else: + mess["color_source"] = mess.get("color_source") or "rule" + line = cleaned["mess"].get("line") + if line: + mess["line"] = line + elif cleaned["mess"]: + warnings.append("mess: this plan has no mess block, the folder colour was not changed") + return merged + + +def merge_notes_file(plan: Mapping[str, Any], path: Path) -> Dict[str, Any]: + """:func:`merge_notes` with the reading done for you.""" + + return merge_notes(plan, load_notes(path)) + + +# --------------------------------------------------------------------------- +# CLI entry point: carl-file-organizer build +# --------------------------------------------------------------------------- + + +def run(args: Any) -> int: + """``build``: lay notes.json over a plan.json and write the plan back.""" + + plan_path = Path(getattr(args, "plan")) + lang = getattr(args, "lang", None) or "en" + try: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + except OSError as error: + raise NotesError("cannot read plan {0}: {1}".format(plan_path, error)) from error + except ValueError as error: + raise NotesError("plan {0} is not valid JSON: {1}".format(plan_path, error)) from error + if not isinstance(plan, dict): + raise NotesError("plan {0} must hold a JSON object".format(plan_path)) + lang = plan.get("lang") or lang + + notes_path = getattr(args, "notes", None) + if notes_path is None: + merged = plan + warnings: List[str] = [] + else: + merged = merge_notes(plan, load_notes(Path(notes_path))) + warnings = list((merged.get("notes") or {}).get("warnings") or ()) + + output = Path(getattr(args, "output", None) or plan_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(merged, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + for line in warnings: + print(i18n.t("warning_prefix", lang) + line) + print(i18n.t("cli_notes_merged", lang, path=output, count=len(warnings))) + + if bool(getattr(args, "report", False)): + try: + from .report import write_report + + report_path = output.parent / "report.html" + write_report(merged, report_path) + except Exception as error: # noqa: BLE001 - the plan is the deliverable + print( + i18n.t("cli_report_skipped", lang, module="carl_file_organizer.report") + + " ({0})".format(error) + ) + else: + print(i18n.t("cli_report_written", lang, path=report_path)) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py new file mode 100644 index 0000000..e46de3b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py @@ -0,0 +1,82 @@ +"""Show a path to the person sitting in front of the machine. + +This is the only thing offered for the items the tool refuses to touch: it +cannot move them, it cannot dispose of them, but it can put the folder on +screen with the item already selected, so the decision is made by a person +looking at the real thing. + +Nothing here writes, deletes or renames. Every call goes through +:func:`subprocess.Popen` with an argument list and never through a shell, so a +file called ``$(rm -rf ~).pdf`` is a file with a silly name and nothing else. + + reveal(Path("~/Downloads/report.pdf").expanduser()) + +Returns True when the platform's file manager was launched, False when there is +nothing to launch it with; it never raises for a missing file manager, because +"I could not show you this" is not worth ending a review session over. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable, List, Optional + +DARWIN = "darwin" +WINDOWS = "win32" + +#: Handed to Popen so a file manager that lives a long time does not keep the +#: review server alive with it. +_POPEN_KW = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL} + + +def supported(platform: Optional[str] = None) -> bool: + """True when this machine has a file manager we know how to call.""" + + return reveal_command(Path.home(), platform=platform) is not None + + +def reveal_command(path: Path, *, platform: Optional[str] = None) -> Optional[List[str]]: + """The exact argv used to reveal ``path``, or None when there is none. + + Split out from :func:`reveal` so the shape of each platform's command can be + asserted from any other platform. macOS and Windows both select the item + itself; ``xdg-open`` has no such option anywhere, so on Linux the parent + folder is opened and the item is merely visible in it. + """ + + system = platform or sys.platform + absolute = os.path.abspath(str(path)) + + if system == DARWIN: + return ["open", "-R", absolute] + if system == WINDOWS: + # explorer wants exactly this comma with no space after it, and it is + # one argument, not two. + return ["explorer", "/select,{0}".format(absolute)] + opener = shutil.which("xdg-open") + if opener is None: + return None + return [opener, os.path.dirname(absolute) or absolute] + + +def reveal( + path: Path, + *, + platform: Optional[str] = None, + runner: Optional[Callable[..., Any]] = None, +) -> bool: + """Open the file manager with ``path`` shown. Never raises for a missing one.""" + + argv = reveal_command(Path(path), platform=platform) + if argv is None: + return False + launch = runner if runner is not None else subprocess.Popen + try: + launch(argv, **_POPEN_KW) + except OSError: + return False + return True diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py new file mode 100644 index 0000000..d2c74cb --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py @@ -0,0 +1,413 @@ +"""Path helpers: portable $HOME form, boundary checks, safe names, no-go zones. + +This module never imports other Carl File Organizer modules, so every work package can +depend on it. ``config`` objects are accepted by duck typing (anything exposing +``.profile`` as a mapping is fine). + +It also owns the two halves of the privacy rule, because both are pure path +arithmetic: :func:`strip_absolute` drops every absolute path out of a plan +before it is embedded in a shareable page, and +:func:`hydrate_absolute` puts them back from the ``$HOME`` twins at apply time. +""" + +from __future__ import annotations + +import copy +import fnmatch +import os +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +#: The one token written to disk, on every platform. plan.json, the manifests, +#: the audit log and report.html all use this and only this, because those files +#: travel: a plan made on a Mac gets opened on a Windows machine, and a report is +#: a page somebody forwards. One stored token means one parser and one privacy +#: check, and that check is a single ``startswith`` over the whole document. +HOME_TOKEN = "$HOME" + +#: The same idea spelled the way Windows spells it. Understood on input, and +#: shown on screen by :func:`display_portable`, but never written into a file. +WINDOWS_HOME_TOKEN = "%USERPROFILE%" + +#: Both spellings, for everything that reads rather than writes. +HOME_TOKENS = (HOME_TOKEN, WINDOWS_HOME_TOKEN) + +#: Path separators a stored portable string may arrive with. +SEPARATORS = ("/", "\\") + + +def home_token(platform: Optional[str] = None) -> str: + """The token a person on this platform recognises at a glance.""" + + system = platform if platform is not None else sys.platform + return WINDOWS_HOME_TOKEN if system == "win32" else HOME_TOKEN + + +def split_home_token(text: str) -> Optional[Tuple[str, str]]: + """``(token, remainder)`` when ``text`` starts with either home token. + + The remainder has no leading separator and always comes back with forward + slashes, so callers can hand it straight to :class:`~pathlib.Path`. + """ + + if not isinstance(text, str): + return None + for token in HOME_TOKENS: + if text == token: + return token, "" + for separator in SEPARATORS: + prefix = token + separator + if text.startswith(prefix): + rest = text[len(prefix) :] + return token, rest.replace("\\", "/") + return None + + +def display_portable(text: str, platform: Optional[str] = None) -> str: + """Rewrite a stored ``$HOME/...`` string for the eyes of this platform. + + Storage never changes; only the rendering does. Everywhere but Windows + this hands back exactly what it was given. + """ + + token = home_token(platform) + if token == HOME_TOKEN: + return text + split = split_home_token(text) + if split is None: + return text + _found, rest = split + if not rest: + return token + return token + "\\" + rest.replace("/", "\\") + +#: Compound suffixes that must survive as one unit, so ``a.tar.gz`` is an archive. +#: :mod:`carl_file_organizer.scanner` re-exports this pair; there is only one spelling. +DOUBLE_SUFFIXES = (".tar.gz", ".tar.bz2", ".tar.xz", ".tar.zst", ".tar.lz") + +#: Envelope fields that hold an absolute path and have a ``_portable`` twin. +PLAN_ABSOLUTE_FIELDS = ("source_root", "managed_dir") + +#: The same, per action. +ACTION_ABSOLUTE_FIELDS = ("source", "destination") + +#: Roots that are refused outright, relative to ``$HOME``. +REFUSED_HOME_SUBDIRS = ("Library", "Applications") + +#: Absolute roots that are refused outright regardless of ``$HOME``. +REFUSED_ABSOLUTE = ("/", "/System", "/Library", "/Applications", "/private", "/usr", "/bin", "/etc", "/var") + + +def _home(home: Optional[Path] = None) -> Path: + return Path(os.path.realpath(str(home))) if home is not None else Path(os.path.realpath(str(Path.home()))) + + +def realpath(path: Path) -> Path: + """``os.path.realpath`` as a :class:`Path`, without touching the filesystem twice.""" + + return Path(os.path.realpath(str(path))) + + +def portable(path: Path, home: Optional[Path] = None) -> str: + """Render ``path`` as ``$HOME/...`` when it lives under the home directory. + + Forward slashes on every platform, because this string is stored, compared + and shipped rather than opened. :func:`display_portable` is what turns it + into something a Windows user reads. + """ + + base = home if home is not None else Path.home() + text = str(path) + try: + return HOME_TOKEN + "/" + Path(text).relative_to(base).as_posix() + except ValueError: + return text + + +def expand_portable(text: str, home: Optional[Path] = None) -> Path: + """Inverse of :func:`portable`, reading either home token. + + A plan written on Windows and applied on a Mac, or a person who typed + ``%USERPROFILE%\\Downloads`` into a settings file by hand, both land here + and both come out as a real path on this machine. + """ + + base = home if home is not None else Path.home() + split = split_home_token(text) + if split is not None: + _token, rest = split + return Path(base) / rest if rest else Path(base) + return Path(text).expanduser() + + +def inside_root(path: Path, root: Path) -> bool: + """True when ``path`` resolves inside ``root`` (both sides realpath'ed).""" + + resolved = realpath(path) + base = realpath(root) + if resolved == base: + return True + try: + resolved.relative_to(base) + return True + except ValueError: + return False + + +def has_symlink_component(path: Path, root: Path) -> bool: + """True when any path segment below ``root`` is a symlink.""" + + base = realpath(root) + try: + relative = Path(path).relative_to(root) + except ValueError: + try: + relative = realpath(path).relative_to(base) + except ValueError: + return True + current = Path(root) + for part in relative.parts: + current = current / part + if current.is_symlink(): + return True + return False + + +def safe_dir_name(name: str) -> str: + """Turn an arbitrary entry name into something safe to use as a directory name.""" + + cleaned = name.replace("\x00", "").replace("/", "-").replace(os.sep, "-") + cleaned = cleaned.lstrip("$").strip() + had_quote = '"' in cleaned or "'" in cleaned + cleaned = cleaned.replace('"', "").replace("'", "").strip() + cleaned = cleaned.strip(".").strip() + if not cleaned: + return "quote-dir" if had_quote else "unnamed" + return cleaned + + +def split_suffix(name: str) -> Tuple[str, str]: + """``("archive", ".tar.gz")``; a leading dot never counts as a suffix. + + The stem keeps its original case, the suffix is folded, so callers comparing + against a rule table do not have to fold again. ``name[len(stem):]`` gives + the suffix back exactly as it was written. + """ + + folded = name.casefold() + for double in DOUBLE_SUFFIXES: + if folded.endswith(double) and len(name) > len(double): + return name[: -len(double)], double + stem, dot, ext = name.rpartition(".") + if not dot or not stem: + return name, "" + return stem, ("." + ext).casefold() + + +def conflict_name(name: str, ts: datetime, *, is_dir: bool = False) -> str: + """Name used when a destination is taken: ``quarterly-report-2026-09-02_10-20-11.pdf``. + + The stamp goes before the suffix so the copy still opens with a double click. + Directories and names without a suffix keep the plain ``-`` + shape, and ``a.tar.gz`` stays one unit because the split is the scanner's. + """ + + safe = safe_dir_name(name) + stamp = ts.strftime("%Y-%m-%d_%H-%M-%S") + if is_dir: + return "{0}-{1}".format(safe, stamp) + stem, folded = split_suffix(safe) + if not folded: + return "{0}-{1}".format(safe, stamp) + return "{0}-{1}{2}".format(stem, stamp, safe[len(stem) :]) + + +def _forbidden_section(config: Any) -> dict: + profile = getattr(config, "profile", config) + section = profile.get("forbidden") if hasattr(profile, "get") else None + return section if isinstance(section, dict) else {} + + +def is_forbidden(path: Path, config: Any) -> Optional[str]: + """Return a rule label when ``path`` sits in a no-go zone, otherwise ``None``.""" + + section = _forbidden_section(config) + name = Path(path).name + folded = name.casefold() + + for basename in section.get("basenames", []) or []: + if name == basename or folded == str(basename).casefold(): + return "forbidden:basename:{0}".format(basename) + + for suffix in section.get("suffixes", []) or []: + if folded.endswith(str(suffix).casefold()): + return "forbidden:suffix:{0}".format(suffix) + + for pattern in section.get("glob", []) or []: + if fnmatch.fnmatch(name, str(pattern)): + return "forbidden:glob:{0}".format(pattern) + + try: + is_dir = Path(path).is_dir() + except OSError: + is_dir = False + if is_dir: + for marker in section.get("markers", []) or []: + try: + if (Path(path) / str(marker)).exists(): + return "forbidden:marker:{0}".format(marker) + except OSError: + continue + return None + + +def refuse_root( + root: Path, + *, + allow_outside_home: bool = False, + home: Optional[Path] = None, +) -> Path: + """Validate a scan root and return its realpath, or raise ``ValueError``.""" + + candidate = Path(root).expanduser() + if candidate.is_symlink(): + raise ValueError("refused root: {0} is a symlink".format(candidate)) + resolved = realpath(candidate) + if not resolved.is_dir(): + raise ValueError("refused root: {0} is not a directory".format(resolved)) + + home_dir = _home(home) + if str(resolved) in REFUSED_ABSOLUTE: + raise ValueError("refused root: {0} is a system directory".format(resolved)) + if resolved == home_dir: + raise ValueError("refused root: the home directory itself is out of scope") + for sub in REFUSED_HOME_SUBDIRS: + blocked = home_dir / sub + if resolved == blocked or inside_root(resolved, blocked): + raise ValueError("refused root: {0} is out of scope".format(blocked)) + if not allow_outside_home and not inside_root(resolved, home_dir): + raise ValueError( + "refused root: {0} is outside {1}; pass --allow-outside-home to override".format( + resolved, home_dir + ) + ) + return resolved + + +# --------------------------------------------------------------------------- +# the privacy rule: absolute paths never leave the managed folder +# --------------------------------------------------------------------------- + + +def plan_home(plan: Any, home: Optional[Path] = None) -> Path: + """Recover the home directory a plan was written against. + + ``source_root`` and ``source_root_portable`` are two spellings of the same + path, so subtracting one from the other gives back ``$HOME`` and nobody has + to store it anywhere. A stripped plan has lost the absolute half, and then + the answer is this machine's home, which is the only one apply can act on. + """ + + if home is not None: + return Path(home) + envelope = plan if isinstance(plan, dict) else {} + root = str(envelope.get("source_root") or "") + portable_text = str(envelope.get("source_root_portable") or "") + if not root: + return Path.home() + split = split_home_token(portable_text) + if split is None: + return Path.home() + _token, rest = split + if not rest: + return Path(root) + relative = Path(rest).parts + whole = Path(root).parts + if len(whole) > len(relative) and whole[-len(relative) :] == relative: + return Path(*whole[: -len(relative)]) + return Path.home() + + +def _portable_text(text: str, base: str) -> str: + """Rewrite every mention of the home directory, in either separator style.""" + + if not base or base in ("/", os.sep): + return text + if text == base: + return HOME_TOKEN + for separator in dict.fromkeys(("/", os.sep)): + text = text.replace(base + separator, HOME_TOKEN + "/") + return text + + +def _scrub(value: Any, base: str) -> Any: + """Every remaining string rendered in ``$HOME`` form, no exemptions.""" + + if isinstance(value, dict): + return {k: _scrub(v, base) for k, v in value.items()} + if isinstance(value, list): + return [_scrub(item, base) for item in value] + if isinstance(value, str): + return _portable_text(value, base) + return value + + +def strip_absolute(plan: Any, home: Optional[Path] = None) -> Any: + """A copy of ``plan`` carrying no absolute path, only the ``_portable`` twins. + + This is what the review page embeds and what ``/api/plan`` serves, so a + report.html handed to somebody else never spells out the account it came + from. The four fields with a twin are dropped outright; anything that names + the home directory is rewritten rather than deleted, so a field added later + cannot quietly reopen the leak. The plan.json in the managed folder is not + touched, and :func:`hydrate_absolute` reverses this at apply time. + """ + + if not isinstance(plan, dict): + return plan + base = str(plan_home(plan, home)) + data = copy.deepcopy(plan) + for field in PLAN_ABSOLUTE_FIELDS: + data.pop(field, None) + actions = data.get("actions") + if isinstance(actions, list): + for action in actions: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + action.pop(field, None) + return _scrub(data, base) + + +def _rehydrate(holder: Dict[str, Any], field: str, home: Optional[Path]) -> None: + if holder.get(field): + return + twin = holder.get(field + "_portable") + if isinstance(twin, str) and twin: + holder[field] = str(expand_portable(twin, home)) + else: + holder[field] = holder.get(field) + + +def hydrate_absolute(plan: Any, home: Optional[Path] = None) -> Any: + """A copy of ``plan`` with the absolute paths rebuilt from their twins. + + Idempotent: a plan that still has them is returned unchanged. A plan + exported from the review page has only ``$HOME`` forms, and they are expanded + against this machine's home unless one is injected. + """ + + if not isinstance(plan, dict): + return plan + base = plan_home(plan, home) + data = copy.deepcopy(plan) + for field in PLAN_ABSOLUTE_FIELDS: + _rehydrate(data, field, base) + actions = data.get("actions") + if isinstance(actions, list): + for action in actions: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + _rehydrate(action, field, base) + return data diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py new file mode 100644 index 0000000..f633d78 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py @@ -0,0 +1,1435 @@ +"""The decision chain: scan, classify, group, guard, and write plan.json v2. + +Nothing in this module touches the filesystem beyond reading. It produces a +list of candidate actions and a list of grouped choices, and every one of them +starts out unapproved. The order of the chain is the whole design: + + pinned you already put it back once, so it stays + no-go zone shown, never touched + guard a process, a launch agent or a symlink still needs it + sensitive naming isolated without ever opening the file + your Finder tags optional, off by default + your name rules from the settings file + archive pairing a .zip and its extracted folder move together + top level folders whole, keeping their name + extension rules plus the settling period + unknown extension waits in Pending and can be rerouted + cold candidates large items and used-up installers get a disposal option + +Afterwards, duplicate and regenerable groups add their disposal options, and any +move whose destination already exists is redirected to a Duplicates Review +folder rather than overwriting anything. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import sys +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from . import __version__ +from . import classifier +from . import config as config_module +from . import grouping, guard, i18n, paths, trash +from .ids import action_id as _action_id +from .scanner import Entry, scan_top_level, split_suffix, user_tags + +SCHEMA_VERSION = 2 + +DEFAULT_LARGE_BYTES = 500 * 1024 * 1024 +DEFAULT_TARGET_PERCENT = 90.0 + +CONFIDENCE = { + "ext": 0.98, + "sensitive": 0.9, + "name": 0.9, + "tag": 0.85, + "pair": 0.8, + "dir": 0.6, + "unknown": 0.55, + "dup_sha256": 0.95, + "dup_name": 0.7, + "regenerable": 0.99, + "cold": 0.5, + "hold": 1.0, +} + +# --------------------------------------------------------------------------- +# the three colours +# --------------------------------------------------------------------------- + +GREEN = "green" +YELLOW = "yellow" +RED = "red" + +#: Written into ``action.color``, ``group.color`` and ``mess.color``. +COLORS = (GREEN, YELLOW, RED) + +#: Folder mess thresholds. ``references/tiers.md`` is the prose version of this +#: table, and an agent may override the resulting colour through notes.json; +#: it may never override an action's colour, because the buttons follow that. +MESS_THRESHOLDS = { + GREEN: {"loose": 3, "pending": 0}, + YELLOW: {"loose": 15, "pending": 5}, +} + +#: Weights behind ``mess.score``: how much of each thing counts as "full". The +#: score only sets the length of a bar; the colour comes from the thresholds +#: above, and neither is derived from the other. +MESS_WEIGHTS = { + "loose": 15.0, + "pending": 5.0, + "aging": 20.0, + "groups": 12.0, + "top_dirs": 20.0, +} + +#: Top level folders below this count do not add to the score at all. +MESS_TOP_DIRS_FREE = 5 + + +def action_color(action: Mapping[str, Any]) -> str: + """Green go ahead, yellow look first, red leave it alone. + + The buttons on the review page follow this one field, so it is derived from + the decision rather than written by hand: every hold is red; a move is green + unless it is one of the reroutable ones still waiting on a human; a disposal + is green only when the thing disposed of can be rebuilt from source. + """ + + if action.get("kind") == "hold": + return RED + if action.get("kind") == "move": + return YELLOW if action.get("reroutable") else GREEN + return GREEN if action.get("tier") == "regenerable" else YELLOW + + +def mess_counts( + entries: Sequence[Any], actions: Sequence[Mapping[str, Any]], groups: Sequence[Any] +) -> Dict[str, int]: + """The five numbers the folder colour is decided on.""" + + return { + "loose": len(entries), + "aging": sum( + 1 for a in actions if a.get("kind") == "hold" and a.get("tier") == "aging" + ), + "pending": len({a["subject_id"] for a in actions if a.get("reroutable")}), + "groups": len(groups), + "top_dirs": sum(1 for entry in entries if getattr(entry, "is_dir", False)), + } + + +def mess_score(counts: Mapping[str, int]) -> int: + """0 to 100, bigger is messier. Only a bar length, never a verdict. + + Two properties matter and a plain weighted sum has neither. The bar must + never sit at 100 on a folder that is merely untidy, or the one folder that + really is a landslide has nowhere left to go; and it must keep moving, so + that clearing thirty files out of two hundred visibly does something. So + the pressure is fed through ``p / (p + 1)``, which saturates instead of + clipping: it approaches 100 and never arrives. + + ``loose`` and ``pending`` are combined with max rather than added, because + they measure the same folder two ways and adding them counts one pile twice. + """ + + load = max( + counts.get("loose", 0) / MESS_WEIGHTS["loose"], + counts.get("pending", 0) / MESS_WEIGHTS["pending"], + ) + extra = ( + counts.get("aging", 0) / MESS_WEIGHTS["aging"] + + counts.get("groups", 0) / MESS_WEIGHTS["groups"] + + max(0, counts.get("top_dirs", 0) - MESS_TOP_DIRS_FREE) / MESS_WEIGHTS["top_dirs"] + ) + pressure = load + extra + return int(round(100 * pressure / (pressure + 1))) + + +def mess_color(counts: Mapping[str, int]) -> str: + """Green, yellow or red for the folder as a whole.""" + + loose = counts.get("loose", 0) + pending = counts.get("pending", 0) + if loose <= MESS_THRESHOLDS[GREEN]["loose"] and pending <= MESS_THRESHOLDS[GREEN]["pending"]: + return GREEN + if loose <= MESS_THRESHOLDS[YELLOW]["loose"] or pending <= MESS_THRESHOLDS[YELLOW]["pending"]: + return YELLOW + return RED + + +def build_mess( + entries: Sequence[Any], actions: Sequence[Mapping[str, Any]], groups: Sequence[Any] +) -> Dict[str, Any]: + """The ``mess`` block: how bad this folder is, and the one line that says why.""" + + counts = mess_counts(entries, actions, groups) + colour = mess_color(counts) + return { + "score": mess_score(counts), + "color": colour, + "color_source": "rule", + "counts": counts, + "reason": i18n.both("mess_" + colour, **counts), + } + + +def color_totals(actions: Sequence[Mapping[str, Any]]) -> Dict[str, Dict[str, int]]: + """``summary.by_color``: how many actions wear each colour, and how big they are. + + Counted per action, not per subject, exactly like ``by_tier`` and ``by_kind``: + one file can hold a green move and a yellow disposal at the same time, so the + bytes column adds up to more than the folder holds. It is a shape, not a total. + """ + + totals = {colour: {"actions": 0, "bytes": 0} for colour in COLORS} + for action in actions: + slot = totals.get(str(action.get("color"))) + if slot is None: + continue + slot["actions"] += 1 + slot["bytes"] += int(action.get("size_bytes") or 0) + return totals + + +@dataclass +class PlanPaths: + managed: Path + plan: Path + report: Path + + +def plan_paths(config: Any) -> PlanPaths: + managed = config_module.managed_dir(config) + return PlanPaths(managed=managed, plan=managed / "plan.json", report=managed / "report.html") + + +# --------------------------------------------------------------------------- +# small helpers +# --------------------------------------------------------------------------- + + +def action_id(subject: str, kind: str, destination_portable: Optional[str]) -> str: + """Thin re-export so no caller ever hand-rolls an id.""" + + return _action_id(subject, kind, destination_portable) + + +def _disk_status(root: Path, target_percent: float) -> Dict[str, Any]: + usage = shutil.disk_usage(str(root)) + percent = (usage.used / usage.total * 100) if usage.total else 0.0 + target_used = int(usage.total * target_percent / 100) + return { + "total_bytes": usage.total, + "used_bytes": usage.used, + "free_bytes": usage.free, + "used_percent": round(percent, 2), + "target_percent": target_percent, + "bytes_to_remove_for_target": max(0, usage.used - target_used), + "note": "Moves on this volume reclaim 0 bytes.", + } + + +def detect_trash_backend() -> str: + """Which trash backend this machine has, without importing the executor. + + One spelling only: :mod:`carl_file_organizer.trash` owns the answer, and a + platform added there shows up in the plan without a second edit here. + """ + + return trash.detect_backend() + + +def _finder_tags_available() -> bool: + return sys.platform == "darwin" and shutil.which("xattr") is not None + + +def _aware(moment: Optional[datetime]) -> datetime: + if moment is None: + return datetime.now().astimezone() + if moment.tzinfo is None: + return moment.astimezone() + return moment + + +def _format_bytes(value: int) -> str: + size = float(value) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024: + return "{0:.1f} {1}".format(size, unit) + size /= 1024 + return "{0:.1f} TB".format(size) + + +# --------------------------------------------------------------------------- +# planning context +# --------------------------------------------------------------------------- + + +@dataclass +class _PairRef: + group_id: str + display: str + stem: str + + +@dataclass +class PlanContext: + config: Any + root: Path + home: Path + now: datetime + large_bytes: int + offer_delete: bool + trash_backend: str + allow_referenced: bool + guard_context: Any + pairs: Dict[str, _PairRef] = field(default_factory=dict) + duplicates: Dict[str, Any] = field(default_factory=dict) + taken: set = field(default_factory=set) + + @property + def trash_available(self) -> bool: + return self.trash_backend != "none" + + +def _portable(ctx: PlanContext, path: Path) -> str: + return paths.portable(path, ctx.home) + + +def _dest_base(ctx: PlanContext, key: str) -> Path: + return config_module.dest_dir(ctx.config, key) + + +def _claim(ctx: PlanContext, entry: Entry, key: str, target: Path) -> Tuple[Path, str, bool]: + """Hand back a destination that does not exist yet; collisions get redirected. + + ``entry`` rather than its name, because a redirected file keeps its suffix + (``report-.pdf``) while a directory does not. + """ + + occupied = target.exists() or target.is_symlink() or str(target) in ctx.taken + if not occupied: + ctx.taken.add(str(target)) + return target, key, False + dup_key = "sensitive.dup" if str(key).split(".")[0] == "sensitive" else "inbox.dup" + try: + redirected = _dest_base(ctx, dup_key) / paths.conflict_name( + entry.name, ctx.now, is_dir=entry.is_dir + ) + except (KeyError, ValueError): + ctx.taken.add(str(target)) + return target, key, False + ctx.taken.add(str(redirected)) + return redirected, dup_key, True + + +# --------------------------------------------------------------------------- +# action construction +# --------------------------------------------------------------------------- + + +def make_action( + entry: Entry, + kind: str, + *, + ctx: PlanContext, + rule: str, + reason: Mapping[str, str], + tier: str, + confidence: float, + destination: Optional[Path] = None, + destination_key: Optional[str] = None, + detail: Optional[Mapping[str, str]] = None, + group_id: Optional[str] = None, + approvable: bool = True, + reroutable: bool = False, + requires: Optional[Sequence[str]] = None, + hints: Optional[Sequence[str]] = None, + hold_reason: Optional[Mapping[str, Any]] = None, + guard_block: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """One row of ``plan.actions``, with every frozen field present.""" + + source_portable = _portable(ctx, entry.path) + destination_portable = _portable(ctx, destination) if destination is not None else None + + if kind == "move" and destination is not None: + restore = i18n.both( + "restore_move", + dest='"{0}"'.format(destination_portable), + src='"{0}/"'.format(_portable(ctx, entry.path.parent)), + ) + elif kind == "trash": + restore = i18n.both("restore_trash") + elif kind == "delete": + restore = i18n.both("restore_delete") + else: + restore = None + + action = { + "id": _action_id(entry.subject_id, kind, destination_portable), + "subject_id": entry.subject_id, + "kind": kind, + "subject_kind": entry.kind, + "source": str(entry.path), + "source_portable": source_portable, + "filename": entry.name, + "destination": str(destination) if destination is not None else None, + "destination_portable": destination_portable, + "destination_key": destination_key, + "size_bytes": entry.size_bytes, + "modified_ns": entry.modified_ns, + "age_hours": entry.age_hours, + "rule": rule, + "reason": dict(reason), + "detail": dict(detail) if detail else None, + "confidence": confidence, + "tier": tier, + "group_id": group_id, + "approvable": bool(approvable), + "reroutable": bool(reroutable), + "default_selected": False, + "requires": list(requires or ()), + "reclaims_bytes": entry.size_bytes if kind in ("trash", "delete") else 0, + "restore_method": restore, + "tags": user_tags(entry.tags, config_module.transient_tags(ctx.config)), + "hints": list(hints or ()), + "hold_reason": dict(hold_reason) if hold_reason else None, + "guard": dict(guard_block) + if guard_block + else {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []}, + "dir_stats": entry.dir_stats.as_dict() if entry.dir_stats is not None else None, + "color": None, + } + # Derived last so it can never disagree with the fields above; build_plan + # recomputes it once more after the duplicate pass rewrites a few tiers. + action["color"] = action_color(action) + return action + + +def _hold( + entry: Entry, + ctx: PlanContext, + *, + rule: str, + tier: str, + reason: Mapping[str, str], + extra: Optional[Mapping[str, Any]] = None, + hints: Optional[Sequence[str]] = None, + guard_block: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + hold_reason = dict(reason) + if extra: + hold_reason.update(extra) + return make_action( + entry, + "hold", + ctx=ctx, + rule=rule, + reason=reason, + tier=tier, + confidence=CONFIDENCE["hold"], + approvable=False, + hold_reason=hold_reason, + hints=hints, + guard_block=guard_block, + ) + + +def disposal_alternatives( + entry: Entry, + ctx: PlanContext, + *, + rule: str, + reason: Mapping[str, str], + tier: str, + confidence: float, + group_id: Optional[str] = None, + guard_block: Optional[Mapping[str, Any]] = None, + allow_delete: bool = True, + detail: Optional[Mapping[str, str]] = None, +) -> List[Dict[str, Any]]: + """Trash and, when it is offered, permanent delete for the same subject.""" + + found: List[Dict[str, Any]] = [] + if ctx.trash_available: + found.append( + make_action( + entry, + "trash", + ctx=ctx, + rule=rule, + reason=reason, + detail=detail, + tier=tier, + confidence=confidence, + group_id=group_id, + guard_block=guard_block, + ) + ) + if ctx.offer_delete and allow_delete: + found.append( + make_action( + entry, + "delete", + ctx=ctx, + rule=rule, + reason=reason, + detail=detail, + tier=tier, + confidence=confidence, + group_id=group_id, + requires=["allow-permanent-delete"], + guard_block=guard_block, + ) + ) + return found + + +# --------------------------------------------------------------------------- +# the guard verdict +# --------------------------------------------------------------------------- + + +def _who(handles: Sequence[Mapping[str, Any]]) -> str: + return "、".join( + "{0}({1})".format(item.get("command") or "?", item.get("pid")) for item in handles + ) + + +def _where(references: Sequence[Mapping[str, Any]], links: Sequence[str]) -> str: + parts = ["{0}:{1}".format(item.get("file"), item.get("line")) for item in references] + parts.extend(str(link) for link in links) + return "、".join(parts) + + +def guard_verdict( + entry: Entry, block: Mapping[str, Any], ctx: PlanContext +) -> Optional[Tuple[str, str, Dict[str, str]]]: + """``(rule, tier, reason)`` when the safety net says leave this alone.""" + + handles = block.get("open_by") or [] + if handles: + return "hold:in_use", "in_use", i18n.both("hold_in_use", who=_who(handles)) + + if "git-worktree" in (block.get("shape") or []): + return "hold:referenced", "referenced", i18n.both("hold_worktree") + + references = block.get("referenced_in") or [] + links = block.get("incoming_symlinks") or [] + if (references or links) and not ctx.allow_referenced: + return ( + "hold:referenced", + "referenced", + i18n.both("hold_referenced", where=_where(references, links)), + ) + + if guard.recently_modified(entry, ctx.now, ctx.guard_context.recent_minutes): + return "hold:aging", "aging", i18n.both("hold_recent") + return None + + +def _guard_hints( + block: Mapping[str, Any], ctx: PlanContext +) -> Tuple[List[str], Optional[Dict[str, str]]]: + """Observations that never change the verdict, plus the downgrade warning.""" + + hints: List[str] = [] + shape = block.get("shape") or [] + for label, hint in ( + ("git-dir", "git-repo"), + ("venv", "venv"), + ("dotenv", "dotenv"), + ("node-project", "node-project"), + ): + if label in shape: + hints.append(hint) + detail = None + references = block.get("referenced_in") or [] + links = block.get("incoming_symlinks") or [] + if (references or links) and ctx.allow_referenced: + hints.append("referenced-warning") + detail = i18n.both("detail_referenced_warning", where=_where(references, links)) + return hints, detail + + +# --------------------------------------------------------------------------- +# the chain +# --------------------------------------------------------------------------- + + +def decide_entry(entry: Entry, ctx: PlanContext) -> List[Dict[str, Any]]: + """Every candidate action for one scanned item, in priority order.""" + + config = ctx.config + + # 1. pinned ------------------------------------------------------------- + pinned = classifier.match_pinned(entry.name, config) + if pinned: + return [ + _hold( + entry, + ctx, + rule="hold:pinned", + tier="pinned", + reason=i18n.both("hold_pinned"), + extra={"pattern": pinned}, + ) + ] + + # 2. no-go zone --------------------------------------------------------- + label = paths.is_forbidden(entry.path, config) + if label: + return [ + _hold( + entry, + ctx, + rule="hold:forbidden", + tier="forbidden", + reason=i18n.both("hold_forbidden", label=label), + extra={"label": label}, + ) + ] + + # 3. the safety net ----------------------------------------------------- + block = guard.inspect(entry.path, config, ctx.guard_context, is_dir=entry.is_dir) + verdict = guard_verdict(entry, block, ctx) + if verdict is not None: + rule, tier, reason = verdict + return [_hold(entry, ctx, rule=rule, tier=tier, reason=reason, guard_block=block)] + hints, guard_detail = _guard_hints(block, ctx) + + # 4. sensitive naming --------------------------------------------------- + hit = classifier.is_sensitive(entry.name, config) + if hit is not None: + target, key, conflicted = _claim( + ctx, + entry, + "sensitive.pending", + _dest_base(ctx, "sensitive.pending") / entry.name, + ) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "sensitive:{0}".format(hit.layer), + reason=i18n.both("conflict") if conflicted else i18n.both("sensitive_isolated"), + detail=guard_detail or i18n.both("detail_sensitive", needle=hit.needle), + tier="sensitive", + confidence=CONFIDENCE["sensitive"], + hints=hints, + guard_block=block, + ) + ] + + # 5. your own Finder tags ---------------------------------------------- + profile = getattr(config, "profile", config) + group_by_tag = bool((profile.get("tags") or {}).get("group_by_user_tags")) + mine = user_tags(entry.tags, config_module.transient_tags(config)) + if group_by_tag and mine: + tag = sorted(mine)[0] + base = _dest_base(ctx, "inbox.by_tag") / paths.safe_dir_name(tag) + target, key, conflicted = _claim(ctx, entry, "inbox.by_tag", base / entry.name) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "tag:{0}".format(tag), + reason=i18n.both("conflict") if conflicted else i18n.both("tag_group", tag=tag), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["tag"], + hints=hints, + guard_block=block, + ) + ] + + # 6. your own name rules ------------------------------------------------ + pattern = classifier.match_name_pattern(entry.name, config) + if pattern is not None: + if pattern.dest_key: + base = _dest_base(ctx, pattern.dest_key) + key = pattern.dest_key + keyed = True + else: + base = ctx.root / str(pattern.dest) + key = "inbox.pending" + keyed = False + target, key, conflicted = _claim(ctx, entry, key, base / entry.name) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key if (keyed or conflicted) else None, + rule="conflict" if conflicted else "name:{0}".format(pattern.id), + reason=i18n.both("conflict") + if conflicted + else i18n.both("name_pattern", pattern=pattern.id), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["name"], + hints=hints, + guard_block=block, + ) + ] + + # 7. archive and extracted folder -------------------------------------- + pair = ctx.pairs.get(entry.subject_id) + if pair is not None: + folder = paths.safe_dir_name(split_suffix(pair.display)[0]) + base = _dest_base(ctx, "inbox.pending") / folder + target, key, conflicted = _claim(ctx, entry, "inbox.pending", base / entry.name) + reason = i18n.both("pair") + found = [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "pair", + reason=i18n.both("conflict") if conflicted else reason, + detail=guard_detail, + tier="duplicate", + confidence=CONFIDENCE["pair"], + group_id=pair.group_id, + hints=hints, + guard_block=block, + ) + ] + found.extend( + disposal_alternatives( + entry, + ctx, + rule="pair", + reason=reason, + tier="duplicate", + confidence=CONFIDENCE["pair"], + group_id=pair.group_id, + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + return found + + in_duplicate_group = entry.subject_id in ctx.duplicates + + # 8. top level folders -------------------------------------------------- + if entry.is_dir: + hint = classifier.derivative_hint(entry.name, config) + if hint: + hints = list(hints) + ["derivative-copy:{0}".format(hint)] + target, key, conflicted = _claim( + ctx, entry, "inbox.pending", _dest_base(ctx, "inbox.pending") / entry.name + ) + rule = "dir:derivative" if hint else "dir:pending" + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else rule, + reason=i18n.both("conflict") + if conflicted + else (i18n.both("dir_derivative", hint=hint) if hint else i18n.both("dir_pending")), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["dir"], + reroutable=not conflicted, + hints=hints, + guard_block=block, + ) + ] + + # 9. extension rules and the settling period --------------------------- + rule_hit = classifier.match_rule(entry.suffix, config) + found = [] + if rule_hit is not None and rule_hit.dest_key: + if entry.age_hours < rule_hit.min_age_hours: + ready = datetime.fromtimestamp( + entry.modified_ns / 1_000_000_000, tz=ctx.now.tzinfo + ) + timedelta(hours=rule_hit.min_age_hours) + return [ + _hold( + entry, + ctx, + rule="hold:aging", + tier="aging", + reason=i18n.both( + "hold_aging", age=entry.age_hours, needed=rule_hit.min_age_hours + ), + extra={"ready_at": ready.isoformat()}, + hints=hints, + guard_block=block, + ) + ] + dest_name = config_module.dir_name(config, rule_hit.dest_key) + target, key, conflicted = _claim( + ctx, entry, rule_hit.dest_key, _dest_base(ctx, rule_hit.dest_key) / entry.name + ) + detail = guard_detail + if detail is None and rule_hit.min_age_hours > 0: + detail = i18n.both("detail_aged", age=entry.age_hours, needed=rule_hit.min_age_hours) + found.append( + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "ext:{0}".format(entry.suffix), + reason=i18n.both("conflict") + if conflicted + else i18n.both(rule_hit.reason_key, dest=dest_name, hours=rule_hit.min_age_hours), + detail=detail, + tier="routine", + confidence=CONFIDENCE["ext"], + hints=hints, + guard_block=block, + ) + ) + else: + # 10. unknown extension --------------------------------------------- + target, key, conflicted = _claim( + ctx, entry, "inbox.pending", _dest_base(ctx, "inbox.pending") / entry.name + ) + found.append( + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "unknown-ext", + reason=i18n.both("conflict") if conflicted else i18n.both("unknown_ext"), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["unknown"], + reroutable=not conflicted, + hints=hints, + guard_block=block, + ) + ) + + # 11. cold storage candidates ------------------------------------------- + if not in_duplicate_group: + if entry.size_bytes >= ctx.large_bytes: + found.extend( + disposal_alternatives( + entry, + ctx, + rule="cold:large", + reason=i18n.both("cold_large", size=_format_bytes(entry.size_bytes)), + tier="cold", + confidence=CONFIDENCE["cold"], + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + elif rule_hit is not None and ( + classifier.is_installer_suffix(entry.suffix, config) + or classifier.is_archive_suffix(entry.suffix, config) + ): + found.extend( + disposal_alternatives( + entry, + ctx, + rule="cold:installer", + reason=i18n.both("cold_installer"), + tier="cold", + confidence=CONFIDENCE["cold"], + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + return found + + +# --------------------------------------------------------------------------- +# groups +# --------------------------------------------------------------------------- + + +def _group_id(prefix: str, material: str) -> str: + return "{0}-{1}".format(prefix, hashlib.sha256(material.encode("utf-8")).hexdigest()[:8]) + + +def _member_label(entry: Entry) -> str: + return entry.name + "/" if entry.is_dir else entry.name + + +def _keep_options( + members: Sequence[Entry], + index: Mapping[Tuple[str, str], str], + *, + label_of, +) -> List[Dict[str, Any]]: + options: List[Dict[str, Any]] = [] + for keep in members: + ids = [index.get((keep.subject_id, "move"))] + dropped = [m for m in members if m.subject_id != keep.subject_id] + ids += [index.get((m.subject_id, "trash")) for m in dropped] + if not all(ids): + continue + options.append( + { + "key": "keep:{0}".format(keep.subject_id), + "label": i18n.ui_both( + "option_keep_one", + keep=label_of(keep), + drop="、".join(label_of(m) for m in dropped), + ), + "action_ids": ids, + "requires": [], + } + ) + return options + + +def _build_pair_group( + raw: Any, group_id: str, index: Mapping[Tuple[str, str], str], ctx: PlanContext +) -> Optional[Dict[str, Any]]: + members = list(raw.members) + options: List[Dict[str, Any]] = [] + + move_ids = [index.get((m.subject_id, "move")) for m in members] + if all(move_ids): + options.append( + { + "key": "keep_all", + "label": i18n.ui_both("option_keep_all"), + "action_ids": list(move_ids), + "requires": [], + } + ) + options.extend(_keep_options(members, index, label_of=_member_label)) + + default_member = next((m for m in members if m.is_dir), members[0]) + dropped = [m for m in members if m.subject_id != default_member.subject_id] + delete_ids = [index.get((default_member.subject_id, "move"))] + [ + index.get((m.subject_id, "delete")) for m in dropped + ] + if ctx.offer_delete and dropped and all(delete_ids): + options.append( + { + "key": "keep:{0}:delete".format(default_member.subject_id), + "label": i18n.ui_both( + "option_keep_one_delete", + keep=_member_label(default_member), + drop="、".join(_member_label(m) for m in dropped), + ), + "action_ids": delete_ids, + "requires": ["allow-permanent-delete"], + } + ) + + if not options: + return None + keys = [option["key"] for option in options] + preferred = "keep:{0}".format(default_member.subject_id) + default_choice = preferred if preferred in keys else keys[0] + return { + "group_id": group_id, + "kind": "pair", + "tier": "duplicate", + "stem": raw.stem, + "evidence": raw.evidence, + "sha256": raw.sha256, + "members": [m.subject_id for m in members], + "member_labels": {m.subject_id: _member_label(m) for m in members}, + "reason": i18n.both("pair"), + "options": options, + "default_choice": default_choice, + "potential_bytes": sum(m.size_bytes for m in dropped) + if default_choice == preferred + else 0, + } + + +def _build_duplicate_group( + raw: Any, group_id: str, index: Mapping[Tuple[str, str], str], ctx: PlanContext +) -> Optional[Dict[str, Any]]: + members = list(raw.members) + verified = raw.evidence == "sha256" + reason = i18n.both("dup_sha256") if verified else i18n.both("dup_name") + options: List[Dict[str, Any]] = [] + + move_ids = [index.get((m.subject_id, "move")) for m in members] + if all(move_ids): + options.append( + { + "key": "keep_all", + "label": i18n.ui_both("option_keep_all_plain"), + "action_ids": list(move_ids), + "requires": [], + } + ) + options.extend(_keep_options(members, index, label_of=lambda item: item.name)) + + if verified and ctx.offer_delete: + keep = members[0] + dropped = [m for m in members if m.subject_id != keep.subject_id] + delete_ids = [index.get((keep.subject_id, "move"))] + [ + index.get((m.subject_id, "delete")) for m in dropped + ] + if dropped and all(delete_ids): + options.append( + { + "key": "keep:{0}:delete".format(keep.subject_id), + "label": i18n.ui_both( + "option_keep_one_delete", + keep=keep.name, + drop="、".join(m.name for m in dropped), + ), + "action_ids": delete_ids, + "requires": ["allow-permanent-delete"], + } + ) + + if not options: + return None + keys = [option["key"] for option in options] + if verified: + preferred = "keep:{0}".format(members[0].subject_id) + default_choice = preferred if preferred in keys else keys[0] + potential = sum(m.size_bytes for m in members[1:]) if default_choice == preferred else 0 + else: + default_choice = "keep_all" if "keep_all" in keys else keys[0] + potential = 0 + return { + "group_id": group_id, + "kind": "duplicate", + "tier": "duplicate", + "stem": raw.stem, + "evidence": raw.evidence, + "sha256": raw.sha256, + "members": [m.subject_id for m in members], + "member_labels": {m.subject_id: m.name for m in members}, + "reason": reason, + "options": options, + "default_choice": default_choice, + "potential_bytes": potential, + } + + +# --------------------------------------------------------------------------- +# build_plan +# --------------------------------------------------------------------------- + + +def build_plan( + source_root: Path, + *, + config: Any = None, + now: Optional[datetime] = None, + hash_duplicates: bool = False, + large_bytes: int = DEFAULT_LARGE_BYTES, + target_percent: float = DEFAULT_TARGET_PERCENT, + dir_sizes: bool = True, + offer_permanent_delete: bool = True, + allow_referenced: bool = False, + home: Optional[Path] = None, + guard_options: Optional[Mapping[str, Any]] = None, + trash_backend: Optional[str] = None, +) -> Dict[str, Any]: + """Scan ``source_root`` read-only and return a schema v2 plan.""" + + requested = Path(source_root).expanduser() + cfg = config if config is not None else config_module.resolve_config(requested) + # The config owns the root: destinations are built from it, so anything else + # here would make every destination fall outside the root it is compared to. + root = Path(cfg.root) + if not root.is_dir(): + raise ValueError("Source is not a directory: {0}".format(root)) + moment = _aware(now) + home_dir = Path(home) if home is not None else Path.home() + backend = trash_backend if trash_backend is not None else detect_trash_backend() + + options = dict(guard_options or {}) + guard_context = guard.build_context( + cfg, + home=home_dir, + enabled=bool(options.get("enabled", True)), + reference_files=options.get("reference_files"), + scan_roots=options.get("scan_roots"), + lsof_runner=options.get("lsof_runner"), + lsof_enabled=bool(options.get("lsof_enabled", True)), + crontab_runner=options.get("crontab_runner"), + git_status_runner=options.get("git_status_runner"), + ) + + entries, notes = scan_top_level(root, cfg, now=moment, dir_sizes=dir_sizes) + guard_context.prime( + [entry.path for entry in entries if not entry.is_dir], + [entry.path for entry in entries if entry.is_dir], + ) + + profile = cfg.profile + dedupe = profile.get("dedupe") or {} + hash_max_bytes = int(dedupe.get("hash_max_bytes", grouping.DEFAULT_HASH_MAX_BYTES)) + regen_depth = int((profile.get("regenerable") or {}).get("scan_depth", 3)) + + ctx = PlanContext( + config=cfg, + root=root, + home=home_dir, + now=moment, + large_bytes=int(large_bytes), + offer_delete=bool(offer_permanent_delete), + trash_backend=backend, + allow_referenced=bool(allow_referenced), + guard_context=guard_context, + ) + + raw_pairs = grouping.pair_archives(entries, cfg) + paired_subjects = set() + for raw in raw_pairs: + raw.group_id = _group_id("pair", raw.stem) + for member in raw.members: + paired_subjects.add(member.subject_id) + ctx.pairs[member.subject_id] = _PairRef(raw.group_id, raw.display, raw.stem) + + raw_dups = grouping.find_duplicates( + entries, + hash_duplicates=hash_duplicates, + hash_max_bytes=hash_max_bytes, + exclude=paired_subjects, + ) + for raw in raw_dups: + raw.group_id = _group_id("dup", (raw.sha256 or raw.stem) + raw.evidence) + for member in raw.members: + ctx.duplicates[member.subject_id] = raw + + actions: List[Dict[str, Any]] = [] + for entry in entries: + actions.extend(decide_entry(entry, ctx)) + + by_subject: Dict[str, List[Dict[str, Any]]] = {} + for action in actions: + by_subject.setdefault(action["subject_id"], []).append(action) + + # duplicate groups relabel the primary move and add disposal options + for raw in raw_dups: + verified = raw.evidence == "sha256" + reason = i18n.both("dup_sha256") if verified else i18n.both("dup_name") + rule = "dup:sha256" if verified else "dup:name" + confidence = CONFIDENCE["dup_sha256"] if verified else CONFIDENCE["dup_name"] + for member in raw.members: + primary = [ + item for item in by_subject.get(member.subject_id, []) if item["kind"] == "move" + ] + if not primary: + continue + head = primary[0] + head["group_id"] = raw.group_id + if head["rule"] != "conflict": + head["rule"] = rule + head["reason"] = dict(reason) + head["confidence"] = confidence + head["tier"] = "duplicate" + head["reroutable"] = False + actions.extend( + disposal_alternatives( + member, + ctx, + rule=rule, + reason=reason, + tier="duplicate", + confidence=confidence, + group_id=raw.group_id, + guard_block=head["guard"], + allow_delete=verified and not member.is_dir, + detail=None if verified else i18n.both("detail_dup_unverified"), + ) + ) + + # regenerable build output inside top level folders + regen_groups: List[Dict[str, Any]] = [] + for hit in grouping.find_regenerable(entries, cfg, max_depth=regen_depth, now=moment): + subject = Entry( + path=hit.path, + name=hit.name, + kind="dir", + size_bytes=hit.dir_stats.bytes, + modified_ns=hit.modified_ns, + age_hours=hit.age_hours, + suffix="", + tags=[], + dir_stats=hit.dir_stats, + subject_id=hit.subject_id, + ) + group_id = _group_id("regen", hit.relative) + restore = classifier.regenerable_restore(hit.name, cfg) + reason = { + "zh": i18n.reason("regenerable", "zh", restore=restore.get("zh", "")), + "en": i18n.reason("regenerable", "en", restore=restore.get("en", "")), + } + made = disposal_alternatives( + subject, + ctx, + rule="regenerable:{0}".format(hit.name), + reason=reason, + tier="regenerable", + confidence=CONFIDENCE["regenerable"], + group_id=group_id, + ) + if not made: + continue + actions.extend(made) + by_kind = {item["kind"]: item["id"] for item in made} + options = [] + if "trash" in by_kind: + options.append( + { + "key": "trash_all", + "label": i18n.ui_both("option_trash_all"), + "action_ids": [by_kind["trash"]], + "requires": [], + } + ) + if "delete" in by_kind: + options.append( + { + "key": "delete_all", + "label": i18n.ui_both("option_delete_all"), + "action_ids": [by_kind["delete"]], + "requires": ["allow-permanent-delete"], + } + ) + regen_groups.append( + { + "group_id": group_id, + "kind": "regenerable", + "tier": "regenerable", + "stem": hit.name, + "evidence": "basename", + "sha256": None, + "members": [hit.subject_id], + "member_labels": {hit.subject_id: hit.relative}, + "reason": reason, + "options": options, + "default_choice": options[0]["key"], + "potential_bytes": hit.dir_stats.bytes, + } + ) + + index: Dict[Tuple[str, str], str] = {} + for action in actions: + index[(action["subject_id"], action["kind"])] = action["id"] + + groups: List[Dict[str, Any]] = [] + for raw in raw_pairs: + built = _build_pair_group(raw, raw.group_id, index, ctx) + if built is not None: + groups.append(built) + for raw in raw_dups: + built = _build_duplicate_group(raw, raw.group_id, index, ctx) + if built is not None: + groups.append(built) + groups.extend(regen_groups) + + known_groups = {group["group_id"] for group in groups} + for action in actions: + if action["group_id"] is not None and action["group_id"] not in known_groups: + action["group_id"] = None + + # The duplicate pass rewrites a few tiers and clears a reroutable flag, so + # every colour is settled here, once, on the finished list. + for action in actions: + action["color"] = action_color(action) + # A group is a question, and a question is always yellow: whichever option + # you pick, something gets disposed of that a rule alone would not touch. + for group in groups: + group["color"] = YELLOW + + names: Dict[str, str] = {} + for key in profile.get("names") or {}: + try: + names[key] = config_module.dir_name(cfg, key) + except (KeyError, ValueError): + continue + + by_tier: Dict[str, int] = {} + by_kind_count: Dict[str, int] = {} + by_destination: Dict[str, int] = {} + reclaimable: Dict[str, int] = {} + for action in actions: + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + by_kind_count[action["kind"]] = by_kind_count.get(action["kind"], 0) + 1 + if action["kind"] == "move" and action["destination"]: + try: + relative = Path(action["destination"]).parent.relative_to(root).as_posix() + except ValueError: + relative = str(Path(action["destination"]).parent) + by_destination[relative] = by_destination.get(relative, 0) + 1 + if action["kind"] in ("trash", "delete"): + reclaimable[action["subject_id"]] = action["size_bytes"] + + managed = config_module.managed_dir(cfg) + return { + "schema_version": SCHEMA_VERSION, + "generator": {"name": "carl-file-organizer", "version": __version__}, + # created_at follows the plan's clock, so --now gives a byte-identical + # plan twice in a row; generated_at keeps the real one for support. + "created_at": moment.isoformat(), + "generated_at": datetime.now().astimezone().isoformat(), + "now": moment.isoformat(), + "lang": cfg.lang, + "lang_source": cfg.lang_source, + "profile": { + "name": cfg.profile_name, + "source": cfg.profile_source, + "dotfile": paths.portable(cfg.dotfile_path, home_dir) if cfg.dotfile_path else None, + "hash": cfg.profile_hash, + }, + "source_root": str(root), + "source_root_portable": paths.portable(root, home_dir), + "managed_dir": str(managed), + "managed_dir_portable": paths.portable(managed, home_dir), + "names": names, + "capabilities": { + "trash_backend": backend, + "finder_tags": _finder_tags_available(), + "permanent_delete_offered": bool(offer_permanent_delete) + and any(action["kind"] == "delete" for action in actions), + }, + "scan_policy": { + "depth": 1, + "hidden_files": "ignored", + "symlinks": "ignored", + "managed_partitions": "skipped", + "regenerable_scan_depth": regen_depth, + "dir_size_limit": {"max_entries": 50000, "time_budget_s": 1.5} + if dir_sizes + else {"max_entries": 0, "time_budget_s": 0}, + "hash_duplicates": bool(hash_duplicates), + "hash_max_bytes": hash_max_bytes, + "allow_referenced": bool(allow_referenced), + "skipped": { + "hidden": notes.skipped_hidden, + "symlinks": notes.skipped_symlinks, + "partitions": notes.skipped_partitions, + "artifacts": notes.skipped_artifacts, + }, + }, + "disk": _disk_status(root, target_percent), + "mess": build_mess(entries, actions, groups), + "summary": { + "entries": len(entries), + "files": sum(1 for entry in entries if not entry.is_dir), + "dirs": sum(1 for entry in entries if entry.is_dir), + "bytes": sum(entry.size_bytes for entry in entries), + "by_tier": dict(sorted(by_tier.items())), + "by_kind": dict(sorted(by_kind_count.items())), + "by_color": color_totals(actions), + "by_destination": dict(sorted(by_destination.items())), + "groups": { + "pair": sum(1 for group in groups if group["kind"] == "pair"), + "duplicate": sum(1 for group in groups if group["kind"] == "duplicate"), + "regenerable": sum(1 for group in groups if group["kind"] == "regenerable"), + }, + "potential_reclaimable_bytes": sum(reclaimable.values()), + "proposed_reclaimed_bytes": 0, + }, + "pinned": [str(item) for item in (profile.get("pinned") or ())], + "actions": actions, + "groups": groups, + # Prose written by an agent on top of the rules, filled in by + # ``carl_file_organizer.notes``. None means nobody wrote any, and the + # report falls back to the rule text. + "notes": None, + "approved_action_ids": [], + "overrides": [], + "approved_at": None, + "approved_by": None, + } + + +def write_plan(plan: Dict[str, Any], output: Path) -> None: + Path(output).parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text( + json.dumps(plan, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def run(args: Any) -> int: + """``carl-file-organizer plan``: a read-only scan, then a plan and a review page.""" + + cfg = getattr(args, "config", None) + if cfg is None: + cfg = config_module.resolve_config( + Path(args.source), + lang=getattr(args, "lang", None), + profile=getattr(args, "profile", None), + ) + + large_mb = int(getattr(args, "large_mb", 500)) + target_percent = float(getattr(args, "target_percent", DEFAULT_TARGET_PERCENT)) + if large_mb < 1: + raise ValueError("--large-mb must be at least 1") + if not 1 <= target_percent <= 99: + raise ValueError("--target-percent must be between 1 and 99") + + where = plan_paths(cfg) + where.managed.mkdir(parents=True, exist_ok=True) + + plan = build_plan( + cfg.root, + config=cfg, + now=getattr(args, "now", None), + hash_duplicates=bool(getattr(args, "hash_duplicates", False)), + large_bytes=large_mb * 1024 * 1024, + target_percent=target_percent, + dir_sizes=bool(getattr(args, "dir_sizes", True)), + offer_permanent_delete=bool(getattr(args, "permanent_delete_options", True)), + allow_referenced=bool(getattr(args, "allow_referenced", False)), + ) + write_plan(plan, where.plan) + print(i18n.t("cli_plan_written", cfg.lang, path=where.plan)) + + try: + from .report import write_report + + write_report(plan, where.report) + except Exception as error: # noqa: BLE001 - the plan itself is the deliverable + print( + i18n.t("cli_report_skipped", cfg.lang, module="carl_file_organizer.report") + + " ({0})".format(error) + ) + else: + print(i18n.t("cli_report_written", cfg.lang, path=where.report)) + + approvable = sum(1 for action in plan["actions"] if action["approvable"]) + print( + i18n.t( + "cli_scan_summary", + cfg.lang, + entries=plan["summary"]["entries"], + actions=len(plan["actions"]), + approvable=approvable, + ) + ) + print(i18n.t("cli_nothing_moved", cfg.lang)) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py new file mode 100644 index 0000000..0c1b881 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py @@ -0,0 +1,10 @@ +"""Built-in profiles. + +Profiles are data, not code. Load them with +``importlib.resources.files("carl_file_organizer.profiles").joinpath("tiered.json")`` +so they keep working from a wheel; never build a path from ``__file__``. +""" + +from __future__ import annotations + +BUILTIN_PROFILES = ("tiered", "simple") diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json new file mode 100644 index 0000000..bee5a7f --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json @@ -0,0 +1,500 @@ +{ + "profile": "simple", + "profile_version": 1, + "lang": "en", + "managed_dir": { + "zh": "00_File_Organizer", + "en": "00_File_Organizer" + }, + "transient_tag": { + "zh": "文件移动", + "en": "Moved by File Organizer" + }, + "review_log": { + "zh": "review-log.md", + "en": "review-log.md" + }, + "names": { + "documents": { + "zh": "Documents", + "en": "Documents" + }, + "images": { + "zh": "Images", + "en": "Images" + }, + "video": { + "zh": "Video", + "en": "Video" + }, + "audio": { + "zh": "Audio", + "en": "Audio" + }, + "archives": { + "zh": "Archives", + "en": "Archives" + }, + "installers": { + "zh": "Installers", + "en": "Installers" + }, + "code": { + "zh": "Code & Data", + "en": "Code & Data" + }, + "other": { + "zh": "Other", + "en": "Other" + }, + "inbox": { + "zh": "Other", + "en": "Other" + }, + "inbox.pending": { + "zh": "Pending", + "en": "Pending" + }, + "inbox.dup": { + "zh": "Duplicates_Review", + "en": "Duplicates_Review" + }, + "sensitive": { + "zh": "Sensitive", + "en": "Sensitive" + }, + "sensitive.pending": { + "zh": "Pending", + "en": "Pending" + }, + "sensitive.dup": { + "zh": "Duplicates_Review", + "en": "Duplicates_Review" + } + }, + "aging": { + "default_hours": 0, + "archive_hours": 0 + }, + "rules": { + ".doc": { + "dest": "documents", + "reason": "ext_doc" + }, + ".docx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".epub": { + "dest": "documents", + "reason": "ext_doc" + }, + ".key": { + "dest": "documents", + "reason": "ext_doc" + }, + ".md": { + "dest": "documents", + "reason": "ext_doc" + }, + ".numbers": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pages": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pdf": { + "dest": "documents", + "reason": "ext_doc" + }, + ".ppt": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pptx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".rtf": { + "dest": "documents", + "reason": "ext_doc" + }, + ".txt": { + "dest": "documents", + "reason": "ext_doc" + }, + ".xls": { + "dest": "documents", + "reason": "ext_doc" + }, + ".xlsx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".avif": { + "dest": "images", + "reason": "ext_image" + }, + ".gif": { + "dest": "images", + "reason": "ext_image" + }, + ".heic": { + "dest": "images", + "reason": "ext_image" + }, + ".jpeg": { + "dest": "images", + "reason": "ext_image" + }, + ".jpg": { + "dest": "images", + "reason": "ext_image" + }, + ".png": { + "dest": "images", + "reason": "ext_image" + }, + ".svg": { + "dest": "images", + "reason": "ext_image" + }, + ".tif": { + "dest": "images", + "reason": "ext_image" + }, + ".tiff": { + "dest": "images", + "reason": "ext_image" + }, + ".webp": { + "dest": "images", + "reason": "ext_image" + }, + ".avi": { + "dest": "video", + "reason": "ext_video" + }, + ".m4v": { + "dest": "video", + "reason": "ext_video" + }, + ".mkv": { + "dest": "video", + "reason": "ext_video" + }, + ".mov": { + "dest": "video", + "reason": "ext_video" + }, + ".mp4": { + "dest": "video", + "reason": "ext_video" + }, + ".webm": { + "dest": "video", + "reason": "ext_video" + }, + ".aac": { + "dest": "audio", + "reason": "ext_audio" + }, + ".flac": { + "dest": "audio", + "reason": "ext_audio" + }, + ".m4a": { + "dest": "audio", + "reason": "ext_audio" + }, + ".mp3": { + "dest": "audio", + "reason": "ext_audio" + }, + ".ogg": { + "dest": "audio", + "reason": "ext_audio" + }, + ".wav": { + "dest": "audio", + "reason": "ext_audio" + }, + ".7z": { + "dest": "archives", + "reason": "ext_archive" + }, + ".bz2": { + "dest": "archives", + "reason": "ext_archive" + }, + ".gz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".rar": { + "dest": "archives", + "reason": "ext_archive" + }, + ".tar": { + "dest": "archives", + "reason": "ext_archive" + }, + ".tgz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".xz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".zip": { + "dest": "archives", + "reason": "ext_archive" + }, + ".apk": { + "dest": "installers", + "reason": "ext_installer" + }, + ".dmg": { + "dest": "installers", + "reason": "ext_installer" + }, + ".exe": { + "dest": "installers", + "reason": "ext_installer" + }, + ".iso": { + "dest": "installers", + "reason": "ext_installer" + }, + ".msi": { + "dest": "installers", + "reason": "ext_installer" + }, + ".pkg": { + "dest": "installers", + "reason": "ext_installer" + }, + ".c": { + "dest": "code", + "reason": "ext_data" + }, + ".cpp": { + "dest": "code", + "reason": "ext_data" + }, + ".css": { + "dest": "code", + "reason": "ext_data" + }, + ".csv": { + "dest": "code", + "reason": "ext_data" + }, + ".go": { + "dest": "code", + "reason": "ext_data" + }, + ".html": { + "dest": "code", + "reason": "ext_data" + }, + ".ipynb": { + "dest": "code", + "reason": "ext_data" + }, + ".java": { + "dest": "code", + "reason": "ext_data" + }, + ".js": { + "dest": "code", + "reason": "ext_data" + }, + ".json": { + "dest": "code", + "reason": "ext_data" + }, + ".jsx": { + "dest": "code", + "reason": "ext_data" + }, + ".py": { + "dest": "code", + "reason": "ext_data" + }, + ".rb": { + "dest": "code", + "reason": "ext_data" + }, + ".rs": { + "dest": "code", + "reason": "ext_data" + }, + ".sql": { + "dest": "code", + "reason": "ext_data" + }, + ".swift": { + "dest": "code", + "reason": "ext_data" + }, + ".toml": { + "dest": "code", + "reason": "ext_data" + }, + ".ts": { + "dest": "code", + "reason": "ext_data" + }, + ".tsx": { + "dest": "code", + "reason": "ext_data" + }, + ".xml": { + "dest": "code", + "reason": "ext_data" + }, + ".yaml": { + "dest": "code", + "reason": "ext_data" + }, + ".yml": { + "dest": "code", + "reason": "ext_data" + } + }, + "name_patterns": [], + "sensitive": { + "exact_names": [ + ".env", + ".netrc", + "credentials.json", + "id_rsa", + "id_ed25519", + "secrets.json" + ], + "exact_stems": [ + "key", + "keys", + "env", + "pass", + "token", + "secret", + "auth" + ], + "suffixes": [ + ".cer", + ".crt", + ".pem", + ".p12", + ".pfx", + ".mobileprovision", + ".env" + ], + "markers": [ + "credential", + "password", + "passwd", + "apikey", + "api-key", + "api_key", + "private-key", + "recovery", + "secret" + ], + "marker_min_len": 5 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "forbidden": { + "basenames": [ + ".git", + "Library", + "Keychains", + "Messages", + "Mail" + ], + "suffixes": [ + ".photoslibrary", + ".aplibrary", + ".sparsebundle", + ".vmwarevm", + ".pvm" + ], + "markers": [ + ".obsidian" + ], + "glob": [ + "Profile *", + "Photos Library*" + ] + }, + "regenerable": { + "basenames": [ + "node_modules", + ".next", + "dist", + "build", + "coverage", + "__pycache__" + ], + "scan_depth": 3, + "restore": { + "_default": { + "zh": "由构建再生", + "en": "regenerated by the build" + } + } + }, + "derivative_hints": [ + "-copy", + " copy", + "-backup" + ], + "archive_suffixes": [ + ".zip", + ".tar.gz", + ".tgz", + ".tar", + ".7z", + ".rar", + ".gz", + ".xz", + ".bz2" + ], + "installer_suffixes": [ + ".dmg", + ".pkg", + ".iso", + ".exe", + ".msi", + ".apk" + ], + "dedupe": { + "hash_max_bytes": 2147483648, + "copy_patterns": [ + "^(?P.+?) \\((?P\\d+)\\)(?P\\.[^.]+)?$", + "^(?P.+?)_(?P\\d+)(?P\\.[^.]+)?$" + ] + }, + "tags": { + "group_by_user_tags": false + }, + "guard": { + "reference_files": [], + "reference_scan_roots": [ + "$HOME", + "$HOME/bin", + "$HOME/.local/bin", + "/usr/local/bin", + "/opt/homebrew/bin" + ], + "recent_minutes": 10, + "lsof_timeout_s": 5 + }, + "large_bytes": 524288000 +} diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json new file mode 100644 index 0000000..aafbb85 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json @@ -0,0 +1,501 @@ +{ + "profile": "tiered", + "profile_version": 1, + "lang": null, + "managed_dir": { + "zh": "00_下载目录管理", + "en": "00_File_Organizer" + }, + "transient_tag": { + "zh": "文件移动", + "en": "Moved by File Organizer" + }, + "review_log": { + "zh": "人工调整日志.md", + "en": "review-log.md" + }, + "names": { + "inbox": { + "zh": "00_收件箱", + "en": "00_Inbox" + }, + "inbox.pending": { + "zh": "待判断", + "en": "Pending" + }, + "inbox.by_tag": { + "zh": "按标签", + "en": "By_Tag" + }, + "inbox.dup": { + "zh": "重复待确认", + "en": "Duplicates_Review" + }, + "work": { + "zh": "10_工作区", + "en": "10_Workspace" + }, + "work.code": { + "zh": "项目代码", + "en": "Projects" + }, + "work.tools": { + "zh": "工具与导出", + "en": "Tools_Exports" + }, + "work.tools.scripts": { + "zh": "脚本", + "en": "Scripts" + }, + "work.db": { + "zh": "数据库与查询", + "en": "Database" + }, + "work.db.sql": { + "zh": "SQL", + "en": "SQL" + }, + "library": { + "zh": "20_知识库", + "en": "20_Library" + }, + "library.docs": { + "zh": "文档资料", + "en": "Documents" + }, + "library.docs.md": { + "zh": "Markdown", + "en": "Markdown" + }, + "library.docs.pdf": { + "zh": "PDF", + "en": "PDF" + }, + "library.docs.ppt": { + "zh": "PPT", + "en": "PPT" + }, + "library.docs.csv": { + "zh": "CSV", + "en": "CSV" + }, + "library.docs.txt": { + "zh": "TXT", + "en": "TXT" + }, + "library.docs.office": { + "zh": "办公文档", + "en": "Office" + }, + "library.images": { + "zh": "图片素材", + "en": "Images" + }, + "library.images.png": { + "zh": "PNG", + "en": "PNG" + }, + "library.images.jpeg": { + "zh": "JPEG", + "en": "JPEG" + }, + "library.images.gif": { + "zh": "GIF", + "en": "GIF" + }, + "library.images.other": { + "zh": "其他图片", + "en": "Other" + }, + "library.video": { + "zh": "视频素材", + "en": "Video" + }, + "library.video.mp4": { + "zh": "MP4", + "en": "MP4" + }, + "library.video.mov": { + "zh": "MOV", + "en": "MOV" + }, + "library.audio": { + "zh": "音频素材", + "en": "Audio" + }, + "library.web": { + "zh": "网页资料", + "en": "Web" + }, + "library.web.html": { + "zh": "HTML", + "en": "HTML" + }, + "sensitive": { + "zh": "60_敏感信息", + "en": "60_Sensitive" + }, + "sensitive.pending": { + "zh": "待转移", + "en": "Pending" + }, + "sensitive.dup": { + "zh": "重复待确认", + "en": "Duplicates_Review" + }, + "archive": { + "zh": "90_归档", + "en": "90_Archive" + }, + "archive.zip": { + "zh": "压缩包", + "en": "Compressed" + }, + "archive.zip.zip": { + "zh": "ZIP", + "en": "ZIP" + }, + "archive.zip.other": { + "zh": "其他压缩包", + "en": "Other" + }, + "archive.installers": { + "zh": "安装包", + "en": "Installers" + }, + "archive.installers.dmg": { + "zh": "DMG", + "en": "DMG" + }, + "archive.installers.other": { + "zh": "其他安装包", + "en": "Other" + } + }, + "aging": { + "default_hours": 48, + "archive_hours": 168 + }, + "rules": { + ".md": { + "dest": "library.docs.md", + "reason": "ext_doc" + }, + ".pdf": { + "dest": "library.docs.pdf", + "reason": "ext_doc" + }, + ".ppt": { + "dest": "library.docs.ppt", + "reason": "ext_doc" + }, + ".pptx": { + "dest": "library.docs.ppt", + "reason": "ext_doc" + }, + ".doc": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".docx": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".xls": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".xlsx": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".csv": { + "dest": "library.docs.csv", + "reason": "ext_doc" + }, + ".txt": { + "dest": "library.docs.txt", + "reason": "ext_doc" + }, + ".png": { + "dest": "library.images.png", + "reason": "ext_image" + }, + ".jpg": { + "dest": "library.images.jpeg", + "reason": "ext_image" + }, + ".jpeg": { + "dest": "library.images.jpeg", + "reason": "ext_image" + }, + ".gif": { + "dest": "library.images.gif", + "reason": "ext_image" + }, + ".webp": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".heic": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".svg": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".mp4": { + "dest": "library.video.mp4", + "reason": "ext_video" + }, + ".mov": { + "dest": "library.video.mov", + "reason": "ext_video" + }, + ".mp3": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".m4a": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".wav": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".html": { + "dest": "library.web.html", + "reason": "ext_web" + }, + ".js": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".sh": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".py": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".sql": { + "dest": "work.db.sql", + "reason": "ext_data" + }, + ".zip": { + "dest": "archive.zip.zip", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".tar.gz": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".7z": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".rar": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".dmg": { + "dest": "archive.installers.dmg", + "min_age_hours": 168, + "reason": "ext_installer" + }, + ".pkg": { + "dest": "archive.installers.other", + "min_age_hours": 168, + "reason": "ext_installer" + }, + ".iso": { + "dest": "archive.installers.other", + "min_age_hours": 168, + "reason": "ext_installer" + } + }, + "name_patterns": [], + "sensitive": { + "exact_names": [ + ".env", + ".netrc", + "credentials.json", + "id_rsa", + "id_ed25519", + "secrets.json", + "service-account.json" + ], + "exact_stems": [ + "key", + "keys", + "env", + "pass", + "token", + "tokens", + "secret", + "auth", + "cred", + "creds" + ], + "suffixes": [ + ".cer", + ".crt", + ".key", + ".pem", + ".p12", + ".pfx", + ".mobileprovision", + ".env", + ".keystore", + ".jks" + ], + "markers": [ + "credential", + "password", + "passwd", + "apikey", + "api-key", + "api_key", + "access-token", + "access_token", + "private-key", + "privatekey", + "recovery", + "secret", + "mnemonic", + "seed-phrase", + "密码", + "密钥", + "凭证", + "恢复码", + "身份证", + "银行卡", + "营业执照" + ], + "marker_min_len": 5 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "forbidden": { + "basenames": [ + ".git", + "Library", + "Keychains", + "Messages", + "Mail", + "vm_bundles", + "Time Machine Backups" + ], + "suffixes": [ + ".photoslibrary", + ".aplibrary", + ".musiclibrary", + ".tvlibrary", + ".sparsebundle", + ".vmwarevm", + ".pvm", + ".utm" + ], + "markers": [ + ".obsidian", + "Default/History" + ], + "glob": [ + "Profile *", + "Photos Library*" + ] + }, + "regenerable": { + "basenames": [ + "node_modules", + ".next", + "dist", + "build", + "coverage", + "__pycache__", + ".turbo", + ".cache", + ".pytest_cache", + "target" + ], + "scan_depth": 3, + "restore": { + "node_modules": { + "zh": "重新 npm/pnpm/bun install", + "en": "re-run npm/pnpm/bun install" + }, + ".next": { + "zh": "重新 build 或 dev", + "en": "re-run build or dev" + }, + "__pycache__": { + "zh": "下次运行自动生成", + "en": "regenerated on next run" + }, + "_default": { + "zh": "由构建再生", + "en": "regenerated by the build" + } + } + }, + "derivative_hints": [ + "-fix-", + "-worktree-", + "-scratch", + "-backup", + "-copy", + " copy" + ], + "archive_suffixes": [ + ".zip", + ".tar.gz", + ".tgz", + ".tar", + ".7z", + ".rar", + ".gz", + ".xz", + ".bz2" + ], + "installer_suffixes": [ + ".dmg", + ".pkg", + ".iso", + ".exe", + ".msi", + ".apk" + ], + "dedupe": { + "hash_max_bytes": 2147483648, + "copy_patterns": [ + "^(?P.+?) \\((?P\\d+)\\)(?P\\.[^.]+)?$", + "^(?P.+?)_(?P\\d+)(?P\\.[^.]+)?$", + "^(?P.+?) (?P\\d+)$" + ] + }, + "tags": { + "group_by_user_tags": false + }, + "guard": { + "reference_files": [], + "reference_scan_roots": [ + "$HOME", + "$HOME/projects", + "$HOME/bin", + "$HOME/.local/bin", + "/usr/local/bin", + "/opt/homebrew/bin" + ], + "recent_minutes": 10, + "lsof_timeout_s": 5 + }, + "large_bytes": 524288000 +} diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py new file mode 100644 index 0000000..bf47d5b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py @@ -0,0 +1,1266 @@ +"""Self-contained HTML review page: one template, two modes. + +``render_report(plan, mode="static"|"serve", token="", lang=None)`` renders the +same controls either way. Only the bottom bar and the submit function differ: +static exports ``carl-file-organizer-approved.json`` through a Blob download, serve +POSTs to ``/api/apply`` on the local review server. + +The page reads nothing but plan.json (schema v2). It never loads an external +resource, so it works from ``file://`` with zero requests. Copy lives in the +``TEXT`` table below; ``i18n.py`` is not touched by this module. + +One thing the page never sees is an absolute path. ``render_report`` runs the +plan through :func:`carl_file_organizer.paths.strip_absolute` first, so both the body +and the embedded ``gn-plan`` JSON carry only ``$HOME/...`` spellings and a page +mailed to a colleague does not spell out the user's account name. The +approved.json exported from the page inherits that, and the executor expands the +twins back at apply time. +""" + +from __future__ import annotations + +import json +from html import escape +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from . import paths + +LANGS = ("zh", "en") +MODES = ("static", "serve") +PERMANENT_FLAG = "allow-permanent-delete" +HOLD_TIER_PROTECTED = "forbidden" + +TEXT: Dict[str, Dict[str, str]] = { + "zh": { + "title": "Carl File Organizer 审核页", + "heading": "整理方案，等你批准", + "lede": "扫描只读，页面上的每一项都要你亲手勾选才会执行。", + "bar_path": "目录", + "bar_time": "扫描时间", + "bar_pending": "待批准", + "bar_disk": "卷使用率", + "bar_reclaim": "可释放估算", + "bar_mode": "模式", + "mode_static": "静态页面", + "mode_serve": "本地服务", + "serve_note": "本页只在这台机器可见，关掉终端即失效。", + "pledge": "移动只改路径不删内容，任何软件找不到文件时，用 undo 或清单里的 mv 命令原路放回。", + "pledge_more": "计划生成后先别改动这些文件和目录，改过的条目执行时会被跳过，重新 plan 即可。", + "disk_note": "同一卷上移动释放 0 字节，只有进废纸篓和永久删除的行会腾出空间。", + "items": "项", + "sec_move": "待批准移动", + "sec_move_desc": "按扩展名和命名规则归到分区，目录整个搬且保留原名。", + "sec_pair": "压缩包与解压目录", + "sec_pair_desc": "成对出现的压缩包和目录，选留谁。默认留目录。", + "sec_duplicate": "重复文件", + "sec_duplicate_desc": "内容一致的副本，选留哪一份。名字像副本但没比对过内容的，默认都保留。", + "sec_regenerable": "可再生产物", + "sec_regenerable_desc": "构建产物和依赖目录，删掉能重新生成。默认进废纸篓。", + "sec_cold": "冷存候选", + "sec_cold_desc": "过了静置期的大文件和安装包。默认进废纸篓，不选就是保留。", + "sec_hold": "静置中、常驻与被占用", + "sec_hold_desc": "这一轮不动。静置期没满、常驻白名单、有进程开着、有配置引用着的都在这里。", + "sec_protected": "禁刀区", + "sec_protected_desc": "系统库和应用数据，只展示，任何情况下都不处理。", + "sec_pending": "待判断", + "sec_pending_desc": "规则表认不出的条目。可以在这里改去向，改动会写进 overrides；拿不准就留在收件箱。", + "th_pick": "批准", + "th_file": "文件", + "th_size": "大小", + "th_dest": "去向", + "th_reason": "理由", + "th_action": "处置", + "th_restore": "恢复方式", + "th_status": "状态", + "th_why": "原因", + "adopt_group": "采纳这组", + "opt_trash": "进废纸篓", + "opt_delete": "永久删除", + "toggle_permanent": "显示永久删除选项", + "permanent_not_enabled": "服务启动时没加 --allow-permanent-delete，永久删除在本页不可用。", + "permanent_static_note": "导出后执行时也要加 --allow-permanent-delete，否则整批拒绝。", + "no_trash_backend": "本机没有废纸篓后端，进废纸篓的选项已隐藏。", + "confirm_delete": "永久删除 {path}（{size}）？删掉就没了，废纸篓里也找不到。", + "confirm_apply": "将执行 {moves} 项移动、{trash} 项进废纸篓、{delete} 项永久删除，预计释放 {bytes}。继续？", + "confirm_dry": "只试运行，不动任何文件。检查这 {n} 项？", + "confirm_shutdown": "停掉本地服务？之后要重新运行 review --serve。", + "nothing_selected": "还没勾选任何项。", + "exported_note": "批准文件已下载，把它交给 carl-file-organizer apply。", + "empty": "本组没有条目。", + "footer_count": "已选 {moves} 项移动 · {trash} 项进废纸篓 · {delete} 项永久删除 · 预计释放 {bytes}", + "btn_export": "导出批准文件", + "btn_apply": "执行全部已选", + "btn_dry": "先试运行", + "btn_section": "执行本组已选", + "btn_select_all": "全选本组", + "btn_clear_all": "清空本组", + "btn_shutdown": "停止服务", + "status_working": "处理中", + "status_moved": "已移动", + "status_trashed": "已进废纸篓", + "status_deleted": "已删除", + "status_dry": "试运行通过", + "status_failed": "失败，看终端", + "status_skipped": "已跳过", + "status_done": "已完成", + "request_failed": "请求失败：{status} {detail}", + "network_failed": "连不上本地服务，终端可能已经关掉。", + "hint": "提示", + "tags": "标签", + "dir_files": "{n} 个文件", + "dir_files_trunc": "至少 {n} 个文件", + "dir_label": "目录", + "sensitive_label": "敏感命名", + "ready_at": "静置到 {time}", + "open_by": "被进程占用", + "referenced_in": "被引用", + "symlinks": "被软链指向", + "shape": "目录形态", + "keep_here": "留在原处", + "potential": "可腾出 {bytes}", + "restore_trash": "废纸篓里选放回原处", + "restore_delete": "不可恢复", + "footer_static_hint": "导出的文件会落到浏览器的下载目录，下次扫描前请交给 apply 或收走。", + "print_hint": "打印版隐藏按钮，只留清单。", + }, + "en": { + "title": "Carl File Organizer review", + "heading": "A tidy-up proposal, waiting for your approval", + "lede": "The scan was read-only. Nothing on this page runs until you tick it yourself.", + "bar_path": "Folder", + "bar_time": "Scanned", + "bar_pending": "Awaiting approval", + "bar_disk": "Volume used", + "bar_reclaim": "Reclaimable estimate", + "bar_mode": "Mode", + "mode_static": "static page", + "mode_serve": "local server", + "serve_note": "This page is only reachable from this machine and stops working when the terminal closes.", + "pledge": "Moves only change the path, never the contents. If any software cannot find a file, run undo or the mv command in the manifest to put it back.", + "pledge_more": "Leave these files and folders alone until you apply the plan; anything changed in between is skipped, and a fresh plan fixes that.", + "disk_note": "Moving files between folders on this volume reclaims 0 bytes; only the trash and permanent-delete rows free up space.", + "items": "items", + "sec_move": "Moves awaiting approval", + "sec_move_desc": "Filed by extension and naming rules. Folders move whole and keep their names.", + "sec_pair": "Archives and their extracted folders", + "sec_pair_desc": "An archive and its folder found together; pick which one stays. The folder is the default.", + "sec_duplicate": "Duplicate files", + "sec_duplicate_desc": "Copies with identical contents; pick the one to keep. Name-only matches default to keeping all.", + "sec_regenerable": "Regenerable output", + "sec_regenerable_desc": "Build output and dependency folders that can be rebuilt. Trash is the default.", + "sec_cold": "Cold storage candidates", + "sec_cold_desc": "Large files and installers past their settling period. Trash is the default; leave unticked to keep.", + "sec_hold": "Settling, pinned and in use", + "sec_hold_desc": "Untouched this round: still settling, pinned by you, held open by a process, or referenced from a config.", + "sec_protected": "No-go zone", + "sec_protected_desc": "System libraries and app data. Shown for context, never handled.", + "sec_pending": "Needs a decision", + "sec_pending_desc": "Entries the rule table does not recognise. Change the destination here and it is recorded in overrides; when unsure, leave it in the inbox.", + "th_pick": "Approve", + "th_file": "File", + "th_size": "Size", + "th_dest": "Destination", + "th_reason": "Why", + "th_action": "Action", + "th_restore": "How to restore", + "th_status": "Status", + "th_why": "Reason", + "adopt_group": "Adopt this group", + "opt_trash": "Move to Trash", + "opt_delete": "Delete permanently", + "toggle_permanent": "Show permanent delete options", + "permanent_not_enabled": "The server was started without --allow-permanent-delete, so permanent deletion is unavailable on this page.", + "permanent_static_note": "Apply the exported file with --allow-permanent-delete, otherwise the whole batch is refused.", + "no_trash_backend": "No trash backend on this machine; the trash options are hidden.", + "confirm_delete": "Permanently delete {path} ({size})? It will not be in the Trash afterwards.", + "confirm_apply": "Run {moves} moves, {trash} to Trash and {delete} permanent deletes, reclaiming about {bytes}. Continue?", + "confirm_dry": "Dry run only, nothing is touched. Check these {n} items?", + "confirm_shutdown": "Stop the local server? You will need to run review --serve again.", + "nothing_selected": "Nothing is ticked yet.", + "exported_note": "The approval file was downloaded. Hand it to carl-file-organizer apply.", + "empty": "Nothing in this group.", + "footer_count": "{moves} moves · {trash} to Trash · {delete} permanent deletes · about {bytes} reclaimed", + "btn_export": "Export approved plan", + "btn_apply": "Apply everything ticked", + "btn_dry": "Dry run first", + "btn_section": "Apply this group", + "btn_select_all": "Select all here", + "btn_clear_all": "Clear all here", + "btn_shutdown": "Stop server", + "status_working": "Working", + "status_moved": "Moved", + "status_trashed": "In Trash", + "status_deleted": "Deleted", + "status_dry": "Dry run passed", + "status_failed": "Failed, see terminal", + "status_skipped": "Skipped", + "status_done": "Done", + "request_failed": "Request failed: {status} {detail}", + "network_failed": "Cannot reach the local server; the terminal may have been closed.", + "hint": "Hint", + "tags": "Tags", + "dir_files": "{n} files", + "dir_files_trunc": "at least {n} files", + "dir_label": "folder", + "sensitive_label": "sensitive name", + "ready_at": "settles at {time}", + "open_by": "Held open by", + "referenced_in": "Referenced in", + "symlinks": "Symlinked from", + "shape": "Folder shape", + "keep_here": "Keep in place", + "potential": "frees {bytes}", + "restore_trash": "Put Back from the Trash", + "restore_delete": "Not recoverable", + "footer_static_hint": "The export lands in the browser download folder; hand it to apply or move it away before the next scan.", + "print_hint": "The print version hides the buttons and keeps the list.", + }, +} + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + + +def _format_bytes(value: Union[int, float, None]) -> str: + size = float(value or 0) + for unit in ("B", "KB", "MB", "GB", "TB"): + if size < 1024 or unit == "TB": + return "{0:.1f} {1}".format(size, unit) if unit != "B" else "{0} B".format(int(size)) + size /= 1024 + return "{0:.1f} TB".format(size) + + +def _lang(plan: Dict[str, Any], lang: Optional[str]) -> str: + chosen = lang or plan.get("lang") or "en" + return chosen if chosen in LANGS else "en" + + +def _t(lang: str, key: str, **kw: Any) -> str: + template = TEXT[lang].get(key) or TEXT["en"].get(key) or key + if kw: + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + return template + + +def _pick(value: Any, lang: str) -> str: + """A ``{zh, en}`` dict, a plain string or None -> display string.""" + + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or value.get("zh") or "") + if value is None: + return "" + return str(value) + + +def _e(value: Any) -> str: + return escape("" if value is None else str(value), quote=True) + + +def _embed_json(data: Any) -> str: + """JSON safe inside a '.format(_embed_json(plan)), + "".format( + _embed_json(config) + ), + "".format(JS_COMMON), + "".format(JS_SERVE if mode == "serve" else JS_STATIC), + "", + ] + return "\n".join(parts) + + +def write_report(plan: Dict[str, Any], output: Union[str, Path], *, lang: Optional[str] = None) -> None: + """Write the static page (the serve page never touches disk).""" + + Path(output).write_text(render_report(plan, mode="static", lang=lang), encoding="utf-8") diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py new file mode 100644 index 0000000..768a314 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py @@ -0,0 +1,320 @@ +"""Read-only top level scan: one Entry per visible item, nothing is opened. + +The scan is deliberately shallow. Partition folders and the managed folder are +boundaries, hidden files and symlinks are ignored, and directory sizes are +measured with both an entry cap and a time budget so a runaway folder can never +stall a plan. + +Frozen for every work package:: + + scan_top_level(root, config, *, now, dir_sizes=True) -> (entries, notes) + dir_stats_bounded(path, max_entries=50000, time_budget_s=1.5) -> DirStats + Entry.subject_id # from carl_file_organizer.ids.subject_id + +One rule worth stating loudly: ``Entry.size_bytes`` for a directory is the +walked total, but its ``subject_id`` is computed from the raw ``stat`` size. +Walked totals are capped and would change under ``--no-dir-sizes``, so mixing +them into the identity would invalidate every directory action; the executor +recomputes the same raw-stat identity before it touches anything. +""" + +from __future__ import annotations + +import fnmatch +import os +import plistlib +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, List, Optional, Sequence, Tuple + +from . import config as config_module +from .ids import subject_id as _subject_id +from .paths import DOUBLE_SUFFIXES, split_suffix + +#: Compound suffixes and the split that honours them live in :mod:`paths`, which +#: needs them to build conflict names; they are re-exported here because this is +#: where every caller already looks for them. +__all_reexported__ = ("DOUBLE_SUFFIXES", "split_suffix") + +#: Carl File Organizer's own artefacts, skipped so a downloaded approval file is not +#: treated as a loose document on the next run. +ARTIFACT_GLOBS = ("carl-file-organizer-*.json",) + +TAG_XATTR = "com.apple.metadata:_kMDItemUserTags" + +DEFAULT_MAX_ENTRIES = 50000 +DEFAULT_TIME_BUDGET_S = 1.5 + + +@dataclass +class DirStats: + files: int + bytes: int + truncated: bool + + def as_dict(self) -> dict: + return {"files": self.files, "bytes": self.bytes, "truncated": self.truncated} + + +@dataclass +class Entry: + path: Path + name: str + kind: str # "file" | "dir" + size_bytes: int + modified_ns: int + age_hours: int + suffix: str + tags: List[str] = field(default_factory=list) + dir_stats: Optional[DirStats] = None + subject_id: str = "" + + @property + def is_dir(self) -> bool: + return self.kind == "dir" + + +@dataclass +class ScanNotes: + """What the scan deliberately did not look at.""" + + skipped_hidden: int = 0 + skipped_symlinks: int = 0 + skipped_partitions: int = 0 + skipped_artifacts: int = 0 + unreadable: List[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# names and suffixes +# --------------------------------------------------------------------------- + + +def suffix_of(name: str) -> str: + return split_suffix(name)[1] + + +def stem_of(name: str) -> str: + return split_suffix(name)[0] + + +def is_artifact_name(name: str) -> bool: + if name == config_module.DOTFILE_NAME: + return True + return any(fnmatch.fnmatch(name, pattern) for pattern in ARTIFACT_GLOBS) + + +# --------------------------------------------------------------------------- +# Finder tags +# --------------------------------------------------------------------------- + + +def read_tags(path: Path) -> List[str]: + """Finder tags without their ``\\n`` suffix; ``[]`` off macOS.""" + + if sys.platform != "darwin": + return [] + try: + result = subprocess.run( + ["xattr", "-p", "-x", TAG_XATTR, str(path)], + text=True, + capture_output=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return [] + if result.returncode != 0: + return [] + hex_text = "".join(result.stdout.split()) + if not hex_text: + return [] + try: + tags = plistlib.loads(bytes.fromhex(hex_text)) + except Exception: # noqa: BLE001 - a corrupt xattr is not worth a crash + return [] + if not isinstance(tags, list): + return [] + cleaned = [] + for tag in tags: + text = str(tag).split("\n", 1)[0].strip() + if text: + cleaned.append(text) + return cleaned + + +def user_tags(tags: Sequence[str], transient: Sequence[str]) -> List[str]: + """Drop Carl File Organizer's own temporary tag in every language spelling.""" + + blocked = {str(item).casefold() for item in transient} + return [tag for tag in tags if tag.casefold() not in blocked] + + +# --------------------------------------------------------------------------- +# bounded directory statistics +# --------------------------------------------------------------------------- + + +def dir_stats_bounded( + path: Path, + max_entries: int = DEFAULT_MAX_ENTRIES, + time_budget_s: float = DEFAULT_TIME_BUDGET_S, +) -> DirStats: + """Count files and bytes under ``path`` with a hard entry and time cap.""" + + started = time.monotonic() + files = 0 + total = 0 + seen = 0 + truncated = False + stack: List[str] = [str(path)] + + while stack: + if seen >= max_entries or (time.monotonic() - started) > time_budget_s: + truncated = True + break + current = stack.pop() + try: + iterator = os.scandir(current) + except OSError: + truncated = True + continue + with iterator: + while True: + try: + item = next(iterator) + except StopIteration: + break + except OSError: + truncated = True + break + seen += 1 + if seen >= max_entries or (time.monotonic() - started) > time_budget_s: + truncated = True + break + try: + if item.is_symlink(): + continue + if item.is_dir(follow_symlinks=False): + stack.append(item.path) + continue + stat = item.stat(follow_symlinks=False) + except OSError: + truncated = True + continue + files += 1 + total += stat.st_size + return DirStats(files=files, bytes=total, truncated=truncated) + + +# --------------------------------------------------------------------------- +# the scan itself +# --------------------------------------------------------------------------- + + +def _age_hours(modified_ns: int, now: datetime) -> int: + seconds = now.timestamp() - (modified_ns / 1_000_000_000) + if seconds <= 0: + return 0 + return int(seconds // 3600) + + +def make_entry( + path: Path, + *, + now: datetime, + dir_sizes: bool = True, + stat_result: Any = None, + dir_limits: Optional[dict] = None, + read_finder_tags: bool = True, +) -> Entry: + """Build one :class:`Entry`; ``path`` must already be known to be safe.""" + + stat = stat_result if stat_result is not None else os.stat(str(path), follow_symlinks=False) + kind = "dir" if os.path.isdir(str(path)) else "file" + limits = dir_limits or {} + stats = None + if kind == "dir" and dir_sizes: + stats = dir_stats_bounded( + path, + int(limits.get("max_entries", DEFAULT_MAX_ENTRIES)), + float(limits.get("time_budget_s", DEFAULT_TIME_BUDGET_S)), + ) + size = stats.bytes if stats is not None else (0 if kind == "dir" else stat.st_size) + modified_ns = stat.st_mtime_ns + return Entry( + path=path, + name=path.name, + kind=kind, + size_bytes=size, + modified_ns=modified_ns, + age_hours=_age_hours(modified_ns, now), + suffix=suffix_of(path.name) if kind == "file" else "", + tags=read_tags(path) if read_finder_tags else [], + dir_stats=stats, + # The identity always uses the raw stat size, never the walked size: + # directory totals are capped and would change with --no-dir-sizes. + subject_id=_subject_id(path, kind, stat.st_size, modified_ns), + ) + + +def scan_top_level( + root: Path, + config: Any, + *, + now: datetime, + dir_sizes: bool = True, + read_finder_tags: bool = True, +) -> Tuple[List[Entry], ScanNotes]: + """One level deep, sorted by name; partitions and hidden items are skipped.""" + + base = Path(root) + notes = ScanNotes() + partitions = {str(name) for name in config_module.partition_dir_names(config)} + limits = { + "max_entries": DEFAULT_MAX_ENTRIES, + "time_budget_s": DEFAULT_TIME_BUDGET_S, + } + + entries: List[Entry] = [] + try: + children = sorted(base.iterdir(), key=lambda item: item.name.casefold()) + except OSError as error: + raise ValueError("cannot read {0}: {1}".format(base, error)) from error + + for path in children: + name = path.name + if name.startswith("."): + notes.skipped_hidden += 1 + continue + if path.is_symlink(): + notes.skipped_symlinks += 1 + continue + if name in partitions: + notes.skipped_partitions += 1 + continue + if is_artifact_name(name): + notes.skipped_artifacts += 1 + continue + try: + stat = os.stat(str(path), follow_symlinks=False) + except OSError: + notes.unreadable.append(name) + continue + if not (path.is_dir() or path.is_file()): + notes.unreadable.append(name) + continue + entries.append( + make_entry( + path, + now=now, + dir_sizes=dir_sizes, + stat_result=stat, + dir_limits=limits, + read_finder_tags=read_finder_tags, + ) + ) + return entries, notes diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py new file mode 100644 index 0000000..b8845f0 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py @@ -0,0 +1,424 @@ +"""``review --serve``: the review page on 127.0.0.1 with one-click apply. + +Security model (see docs/review-page.md): + +* binds ``127.0.0.1`` only, port 0 by default, one random token per run; +* the token is printed to stderr once and handed to the browser in the URL; + the page wipes it from the address bar and sends it as ``X-GN-Token``; +* every request checks the token with ``hmac.compare_digest``, the ``Host`` + header, and for POST the same-origin ``Origin`` header, JSON content type + and a 1 MiB body cap; +* the server validates nothing about the plan itself. ``/api/apply`` copies + the plan, fills ``approved_action_ids`` / ``overrides`` and calls the very + same ``apply_approved_plan`` the CLI uses, so every safety check lives in + one place; +* single-threaded ``HTTPServer`` plus a lock: executions never overlap. +""" + +from __future__ import annotations + +import copy +import hmac +import json +import secrets +import sys +import threading +import webbrowser +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from urllib.parse import parse_qs, urlsplit + +from . import paths +from .report import render_report + +MAX_BODY_BYTES = 1024 * 1024 +DRAIN_LIMIT_BYTES = 8 * MAX_BODY_BYTES +TOKEN_HEADER = "X-GN-Token" +EXECUTED_STATUSES = ("moved", "trashed", "deleted") +JSON_TYPE = "application/json" + +ApplyFn = Callable[..., Any] + + +class ServerError(RuntimeError): + """Raised when the review server cannot be prepared.""" + + +def _default_apply_fn() -> ApplyFn: + try: + from .executor import apply_approved_plan + except ImportError as error: # pragma: no cover - depends on the executor work package + raise ServerError( + "executor.apply_approved_plan is unavailable ({0}); review --serve needs it".format(error) + ) from error + return apply_approved_plan + + +def _load_plan(plan_path: Union[str, Path]) -> Dict[str, Any]: + path = Path(plan_path) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except OSError as error: + raise ServerError("cannot read plan: {0}".format(error)) from error + except ValueError as error: + raise ServerError("plan is not valid JSON: {0}".format(error)) from error + if not isinstance(data, dict): + raise ServerError("plan must be a JSON object") + return data + + +class ReviewState: + """Everything the handler needs, shared through ``server.gn_state``.""" + + def __init__( + self, + plan: Dict[str, Any], + *, + token: str, + allow_permanent_delete: bool, + apply_fn: Optional[ApplyFn], + ) -> None: + self.plan = plan + self.token = token + self.allow_permanent_delete = allow_permanent_delete + self._apply_fn = apply_fn + self.lock = threading.Lock() + self.executed_ids: List[str] = [] + self.port = 0 + caps = dict(plan.get("capabilities") or {}) + caps["permanent_delete_enabled"] = bool(allow_permanent_delete) + self.plan["capabilities"] = caps + + @property + def apply_fn(self) -> ApplyFn: + if self._apply_fn is None: + self._apply_fn = _default_apply_fn() + return self._apply_fn + + def public_plan(self) -> Dict[str, Any]: + """What leaves this process: the plan with every absolute path removed. + + Only the page and ``/api/plan`` get this. ``self.plan`` keeps its + absolute paths and is what ``/api/apply`` hands to the executor, so + stripping here costs the apply path nothing. + """ + + data = paths.strip_absolute(self.plan) + data["executed_ids"] = list(self.executed_ids) + return data + + def kinds_for(self, action_ids: List[str]) -> Dict[str, str]: + index = {str(a.get("id")): str(a.get("kind", "")) for a in self.plan.get("actions") or [] if isinstance(a, dict)} + return {i: index.get(i, "") for i in action_ids} + + +def _normalize_results(report: Any) -> List[Dict[str, Any]]: + """``ApplyReport.results`` or a bare list of dicts -> list of dicts.""" + + results = getattr(report, "results", report) + if results is None: + return [] + out = [] + for item in results: + if isinstance(item, dict): + out.append(item) + else: + out.append( + { + "action_id": getattr(item, "action_id", None), + "status": getattr(item, "status", None), + "detail": getattr(item, "detail", None), + } + ) + return out + + +class ReviewHandler(BaseHTTPRequestHandler): + server_version = "CarlFileOrganizerReview/0.2" + sys_version = "" + protocol_version = "HTTP/1.0" + + # -- plumbing ----------------------------------------------------------- + + @property + def state(self) -> ReviewState: + return self.server.gn_state # type: ignore[attr-defined] + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib signature + # never echo the query string (it carries the token on first load) + path = urlsplit(self.path).path + sys.stderr.write( + "{0} - {1} {2} {3}\n".format(self.address_string(), self.command, path, args[1] if len(args) > 1 else "") + ) + + def _send_json(self, status: int, payload: Any) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", JSON_TYPE + "; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + self.wfile.write(body) + + def _send_html(self, html: str) -> None: + body = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.send_header("Referrer-Policy", "no-referrer") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header( + "Content-Security-Policy", + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; " + "connect-src 'self'; img-src data:; base-uri 'none'; form-action 'none'", + ) + self.end_headers() + self.wfile.write(body) + + def _origin(self) -> str: + return "http://127.0.0.1:{0}".format(self.state.port) + + def _host_ok(self) -> bool: + return self.headers.get("Host", "") == "127.0.0.1:{0}".format(self.state.port) + + def _token_ok(self, presented: Optional[str]) -> bool: + return hmac.compare_digest(str(presented or ""), self.state.token) + + def _guard_common(self) -> bool: + if not self._host_ok(): + self._send_json(400, {"error": "bad Host header"}) + return False + return True + + # -- GET ---------------------------------------------------------------- + + def do_GET(self) -> None: # noqa: N802 - stdlib naming + if not self._guard_common(): + return + parts = urlsplit(self.path) + if parts.path == "/": + token = (parse_qs(parts.query).get("t") or [""])[0] + if not self._token_ok(token): + self._send_json(401, {"error": "missing or wrong token"}) + return + with self.state.lock: + html = render_report(self.state.public_plan(), mode="serve", token=self.state.token) + self._send_html(html) + return + if not self._token_ok(self.headers.get(TOKEN_HEADER)): + self._send_json(401, {"error": "missing or wrong token"}) + return + if parts.path == "/api/plan": + with self.state.lock: + self._send_json(200, self.state.public_plan()) + return + self._send_json(404, {"error": "not found"}) + + # -- POST --------------------------------------------------------------- + + def _read_json_body(self) -> Optional[Dict[str, Any]]: + origin = self.headers.get("Origin", "") + if origin != self._origin(): + self._send_json(403, {"error": "Origin must be {0}".format(self._origin())}) + return None + if not self._token_ok(self.headers.get(TOKEN_HEADER)): + self._send_json(401, {"error": "missing or wrong token"}) + return None + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + if content_type != JSON_TYPE: + self._send_json(415, {"error": "Content-Type must be application/json"}) + return None + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + self._send_json(400, {"error": "bad Content-Length"}) + return None + if length < 0 or length > MAX_BODY_BYTES: + self._drain(length) + self._send_json(413, {"error": "body larger than 1 MiB"}) + return None + raw = self.rfile.read(length) if length else b"" + try: + data = json.loads(raw.decode("utf-8")) if raw else {} + except ValueError: + self._send_json(400, {"error": "body is not valid JSON"}) + return None + if not isinstance(data, dict): + self._send_json(400, {"error": "body must be a JSON object"}) + return None + return data + + def _drain(self, length: int) -> None: + """Read and discard an oversized body (bounded) so the client sees the 413, not a reset.""" + + remaining = min(max(length, 0), DRAIN_LIMIT_BYTES) + while remaining > 0: + chunk = self.rfile.read(min(65536, remaining)) + if not chunk: + break + remaining -= len(chunk) + + def do_POST(self) -> None: # noqa: N802 - stdlib naming + if not self._guard_common(): + return + path = urlsplit(self.path).path + if path not in ("/api/apply", "/api/shutdown"): + # keep the same checks for unknown paths so probing reveals nothing + if self._read_json_body() is not None: + self._send_json(404, {"error": "not found"}) + return + body = self._read_json_body() + if body is None: + return + if path == "/api/shutdown": + self._send_json(200, {"ok": True, "shutting_down": True}) + threading.Thread(target=self.server.shutdown, daemon=True).start() + return + self._apply(body) + + def _apply(self, body: Dict[str, Any]) -> None: + action_ids = body.get("action_ids") + overrides = body.get("overrides", []) + dry_run = bool(body.get("dry_run", False)) + if not isinstance(action_ids, list) or not all(isinstance(i, str) for i in action_ids): + self._send_json(400, {"error": "action_ids must be a list of strings"}) + return + if not action_ids: + self._send_json(400, {"error": "action_ids is empty"}) + return + if not isinstance(overrides, list) or not all(isinstance(o, dict) for o in overrides): + self._send_json(400, {"error": "overrides must be a list of objects"}) + return + state = self.state + with state.lock: + repeated = sorted(set(action_ids) & set(state.executed_ids)) + if repeated: + self._send_json(409, {"error": "already executed", "action_ids": repeated}) + return + kinds = state.kinds_for(action_ids) + deletes = sorted(i for i, kind in kinds.items() if kind == "delete") + if deletes and not state.allow_permanent_delete: + self._send_json( + 403, + { + "error": "permanent delete is off; restart with --allow-permanent-delete", + "action_ids": deletes, + }, + ) + return + plan = copy.deepcopy(state.plan) + plan["approved_action_ids"] = list(action_ids) + plan["overrides"] = list(overrides) + plan["approved_at"] = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + plan["approved_by"] = "serve" + try: + report = state.apply_fn( + plan, dry_run=dry_run, allow_permanent_delete=state.allow_permanent_delete + ) + except Exception as error: # noqa: BLE001 - executor refusals become one JSON line + self._send_json(422, {"error": "{0}: {1}".format(type(error).__name__, error)}) + return + results = _normalize_results(report) + newly_done = [] + if not dry_run: + for item in results: + if item.get("status") in EXECUTED_STATUSES and item.get("action_id"): + aid = str(item["action_id"]) + if aid not in state.executed_ids: + state.executed_ids.append(aid) + newly_done.append(aid) + payload: Dict[str, Any] = { + "ok": True, + "dry_run": dry_run, + "results": results, + "executed_ids": list(state.executed_ids), + "newly_executed": newly_done, + } + for key in ("run_id", "manifest", "audit"): + value = getattr(report, key, None) + if value is not None: + payload[key] = str(value) + self._send_json(200, payload) + + +# -------------------------------------------------------------------------- +# public API +# -------------------------------------------------------------------------- + + +def build_server( + plan: Union[Dict[str, Any], str, Path], + *, + host: str = "127.0.0.1", + port: int = 0, + allow_permanent_delete: bool = False, + apply_fn: Optional[ApplyFn] = None, + token: Optional[str] = None, +) -> Tuple[HTTPServer, str]: + """Prepare (but do not run) the review server. Returns ``(server, token)``.""" + + data = _load_plan(plan) if not isinstance(plan, dict) else copy.deepcopy(plan) + token = token or secrets.token_urlsafe(32) + state = ReviewState( + data, token=token, allow_permanent_delete=allow_permanent_delete, apply_fn=apply_fn + ) + server = HTTPServer((host, port), ReviewHandler) + state.port = server.server_address[1] + server.gn_state = state # type: ignore[attr-defined] + return server, token + + +def serve_review( + plan_path: Union[str, Path], + *, + host: str = "127.0.0.1", + port: int = 0, + allow_permanent_delete: bool = False, + open_browser: bool = True, + apply_fn: Optional[ApplyFn] = None, +) -> int: + """Run the review server until Ctrl-C or ``POST /api/shutdown``.""" + + server, token = build_server( + plan_path, + host=host, + port=port, + allow_permanent_delete=allow_permanent_delete, + apply_fn=apply_fn, + ) + if apply_fn is None: + # fail early with a clear line instead of a 422 on the first click + server.gn_state.apply_fn # type: ignore[attr-defined] # noqa: B018 + bound_port = server.server_address[1] + url = "http://127.0.0.1:{0}/?t={1}".format(bound_port, token) + sys.stderr.write("review page (token inside, only on this line): {0}\n".format(url)) + if allow_permanent_delete: + sys.stderr.write("permanent delete is ENABLED for this session\n") + sys.stderr.write("press Ctrl-C to stop\n") + if open_browser: + try: + webbrowser.open(url) + except Exception: # noqa: BLE001 - a browser failure is not fatal + pass + try: + server.serve_forever(poll_interval=0.5) + except KeyboardInterrupt: + return 130 + finally: + server.server_close() + return 0 + + +def run(args: Any) -> int: + """CLI entry: ``review --serve [--port N] [--no-open] [--allow-permanent-delete]``.""" + + return serve_review( + args.plan, + port=int(getattr(args, "port", 0) or 0), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + open_browser=bool(getattr(args, "open_browser", True)), + ) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py new file mode 100644 index 0000000..e44a161 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py @@ -0,0 +1,241 @@ +"""Two answers: when this folder was last tidied, and whether it is messy again. + +The first answer comes only from the manifests in the managed directory. A +folder's modification time changes every time anything is downloaded into it, +so it says nothing about whether anyone ever organized it; the manifests are +the only honest record. + +The second answer comes from a fresh read-only plan. When the planner is not +available the status still prints, and simply says the recheck could not run. +""" + +from __future__ import annotations + +import sys +import unicodedata +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from . import i18n +from . import manifest as manifest_module + +TEXT = { + "never": {"zh": "还没整理过", "en": "not tidied yet"}, + "last_line": { + "zh": "{when}，动过 {count} 项（清单 {name}）", + "en": "{when}, {count} item(s) (manifest {name})", + }, + "clean": {"zh": "顶层是干净的", "en": "the top level is clear"}, + "dirty": { + "zh": "散件 {loose} · 静置中 {aging} · 待判断 {pending} · 敏感待转移 {sensitive}", + "en": "loose {loose} · settling {aging} · undecided {pending} · sensitive {sensitive}", + }, + "no_recheck": { + "zh": "无法复查（{error}），下面的数字这次没有算出来", + "en": "recheck unavailable ({error}); the counts below were not computed", + }, + "planner_missing": { + "zh": "扫描模块还没接上", + "en": "the planner is not wired up yet", + }, + "root_line": {"zh": "目录", "en": "Folder"}, +} + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class StatusReport: + root: Path + lang: str = "en" + last_run: Optional[manifest_module.RunInfo] = None + loose: int = 0 + aging: int = 0 + pending: int = 0 + sensitive_pending: int = 0 + recheck_actions: Optional[int] = None + recheck_error: Optional[str] = None + managed: Optional[Path] = None + notes: List[str] = field(default_factory=list) + + +#: Rules that mean the item is waiting for a person to decide where it goes. +PENDING_RULES = ("unknown-ext", "dir:pending", "dir:derivative") + + +def _plan_counts(plan: Dict[str, Any], report: StatusReport) -> None: + actions = plan.get("actions") or [] + loose = set() + aging = set() + pending = set() + sensitive = set() + approvable = 0 + + for action in actions: + if not isinstance(action, dict): + continue + subject = str(action.get("subject_id") or action.get("id") or "") + kind = str(action.get("kind") or "") + tier = str(action.get("tier") or "") + rule = str(action.get("rule") or "") + if action.get("approvable", kind != "hold"): + approvable += 1 + if kind == "hold": + if tier == "aging" or rule == "hold:aging": + aging.add(subject) + continue + loose.add(subject) + if rule in PENDING_RULES or str(action.get("destination_key") or "").startswith( + "inbox.pending" + ): + pending.add(subject) + if tier == "sensitive": + sensitive.add(subject) + + report.loose = len(loose) + report.aging = len(aging) + report.pending = len(pending) + report.sensitive_pending = len(sensitive) + report.recheck_actions = approvable + + +def _default_plan_fn(root: Path, config: Any, now: Optional[datetime]) -> Dict[str, Any]: + """Call ``planner.build_plan`` with whichever keywords it actually accepts.""" + + import inspect + + from . import planner as planner_module + + build = getattr(planner_module, "build_plan", None) + if build is None: + raise RuntimeError(text("planner_missing", "en")) + + options: Dict[str, Any] = {} + try: + parameters = inspect.signature(build).parameters + except (TypeError, ValueError): + parameters = {} + if "config" in parameters: + options["config"] = config + if "now" in parameters and now is not None: + options["now"] = now + if "dir_sizes" in parameters: + options["dir_sizes"] = False + if "hash_duplicates" in parameters: + options["hash_duplicates"] = False + return build(root, **options) + + +def compute_status( + root: Path, + *, + config: Any, + now: Optional[datetime] = None, + plan_fn: Optional[Callable[[], Dict[str, Any]]] = None, +) -> StatusReport: + """Last run from the manifests, current mess from a fresh read-only plan.""" + + from . import config as config_module + + folder = Path(root) + report = StatusReport(root=folder, lang=getattr(config, "lang", "en")) + managed = config_module.managed_dir(config) + report.managed = managed + report.last_run = manifest_module.latest_run(managed) + + try: + plan = plan_fn() if plan_fn is not None else _default_plan_fn(folder, config, now) + except Exception as error: # noqa: BLE001 - status must print even when scanning fails + report.recheck_error = str(error) or error.__class__.__name__ + return report + + if not isinstance(plan, dict): + report.recheck_error = "the planner returned no plan" + return report + + _plan_counts(plan, report) + return report + + +def _display_width(text_value: str) -> int: + """Column width of a label, counting wide CJK characters as two cells.""" + + return sum(2 if unicodedata.east_asian_width(char) in ("W", "F") else 1 for char in text_value) + + +def _pad(text_value: str, width: int) -> str: + return text_value + " " * max(0, width - _display_width(text_value)) + + +def format_status(report: StatusReport, lang: Optional[str] = None) -> str: + """Two labelled lines: when it was last done, and how it looks now.""" + + speech = lang or report.lang or "en" + label_root = text("root_line", speech) + label_last = i18n.t("status_last_run", speech) + label_now = i18n.t("status_dirty_now", speech) + width = max(_display_width(item) for item in (label_root, label_last, label_now)) + + if report.last_run is None: + last = text("never", speech) + else: + stamp = report.last_run.moved_at or "" + if report.last_run.timestamp is not None and not stamp: + stamp = report.last_run.timestamp.strftime("%Y-%m-%d %H:%M:%S") + last = text( + "last_line", + speech, + when=stamp, + count=report.last_run.done, + name=report.last_run.path.name, + ) + + if report.recheck_error is not None: + now_line = text("no_recheck", speech, error=report.recheck_error) + elif not (report.loose or report.aging or report.pending or report.sensitive_pending): + now_line = text("clean", speech) + else: + now_line = text( + "dirty", + speech, + loose=report.loose, + aging=report.aging, + pending=report.pending, + sensitive=report.sensitive_pending, + ) + + lines = [ + "{0} {1}".format(_pad(label_root, width), report.root), + "{0} {1}".format(_pad(label_last, width), last), + "{0} {1}".format(_pad(label_now, width), now_line), + ] + return "\n".join(lines) + + +def run(args: Any) -> int: + """``carl-file-organizer status [dir] [--lang zh|en]``.""" + + config = getattr(args, "config", None) + if config is None: + from . import config as config_module + from . import paths + + root = paths.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + config = config_module.resolve_config(root, lang=getattr(args, "lang", None)) + + report = compute_status(config.root, config=config) + print(format_status(report, config.lang)) + if report.recheck_error: + print(report.recheck_error, file=sys.stderr) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py new file mode 100644 index 0000000..d28ea23 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py @@ -0,0 +1,302 @@ +"""macOS Finder tags: read, write, add, remove, and clear the temporary one. + +The temporary tag is how a person sees what one run touched: open Finder, click +the tag, and there is the whole batch. Tagging is a courtesy, never a +precondition, so every failure here warns on stderr and returns False instead +of stopping a batch that already moved files. + +Everywhere except macOS these functions are honest no-ops. + +This module also owns the ``clear-tags`` subcommand. +""" + +from __future__ import annotations + +import os +import plistlib +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence + +TAG_XATTR = "com.apple.metadata:_kMDItemUserTags" + +#: Finder stores a colour as a second line inside the tag string. +COLOR_SEPARATOR = "\n" + +TAG_LIST_GLOB = "*-move-tags-*.txt" + +MESSAGES = { + "cleared": {"zh": "已清除标签 {tag}：{count} 项", "en": "Cleared tag {tag} from {count} item(s)"}, + "dry_run": { + "zh": "预演：会清除标签 {tag}，共 {count} 项，没有真的改动", + "en": "Dry run: would clear tag {tag} from {count} item(s); nothing changed", + }, + "unsupported": { + "zh": "这台机器不支持 Finder 标签，没有可清除的东西", + "en": "Finder tags are not available on this machine; nothing to clear", + }, + "no_lists": { + "zh": "管理目录里没有标签清单（{glob}），可以用 --from 指定一个", + "en": "No tag list ({glob}) in the managed folder; pass --from to name one", + }, + "missing": {"zh": "找不到：{path}", "en": "missing: {path}"}, + "summary": { + "zh": "检查 {checked} 项，清除 {cleared} 项，跳过 {skipped} 项，失败 {failed} 项", + "en": "checked {checked}, cleared {cleared}, skipped {skipped}, failed {failed}", + }, +} + + +@dataclass +class ClearReport: + tag: str + dry_run: bool + checked: int = 0 + cleared: int = 0 + skipped: int = 0 + missing: int = 0 + failed: int = 0 + lists: List[Path] = field(default_factory=list) + details: List[Dict[str, Any]] = field(default_factory=list) + + +def message(key: str, lang: str = "en", **kw: Any) -> str: + entry = MESSAGES.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def _warn(text: str) -> None: + print("tag_warning: {0}".format(text), file=sys.stderr) + + +def supported() -> bool: + """True only on macOS with the ``xattr`` tool available.""" + + return sys.platform == "darwin" and bool(shutil.which("xattr")) + + +# -------------------------------------------------------------------------- +# raw tag access +# -------------------------------------------------------------------------- + + +def read_tags(path: Path) -> List[str]: + """Every Finder tag on ``path``, colour suffix included, or an empty list.""" + + if not supported(): + return [] + try: + result = subprocess.run( + ["xattr", "-p", "-x", TAG_XATTR, str(path)], + text=True, + capture_output=True, + ) + except (OSError, subprocess.SubprocessError) as error: + _warn("{0}\t{1}".format(path, error)) + return [] + if result.returncode != 0: + return [] + hex_text = "".join(result.stdout.split()) + if not hex_text: + return [] + try: + tags = plistlib.loads(bytes.fromhex(hex_text)) + except Exception: # noqa: BLE001 - a malformed xattr is not worth a traceback + return [] + if not isinstance(tags, list): + return [] + return [str(tag) for tag in tags] + + +def write_tags(path: Path, tags: Sequence[str]) -> bool: + """Replace the tag list on ``path``; warn and return False on any failure.""" + + if not supported(): + return False + try: + payload = plistlib.dumps([str(tag) for tag in tags], fmt=plistlib.FMT_BINARY).hex() + subprocess.run( + ["xattr", "-w", "-x", TAG_XATTR, payload, str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.SubprocessError) as error: + _warn("{0}\t{1}".format(path, error)) + return False + return True + + +def add_tag(path: Path, tag: str) -> bool: + """Add one tag, keeping the others. False means nothing was written.""" + + if not supported(): + return False + current = read_tags(path) + if any(_tag_name(item) == tag for item in current): + return False + return write_tags(path, list(current) + [tag]) + + +def remove_tag(path: Path, tag: str) -> bool: + """Drop one tag, keeping the others. False means nothing was written.""" + + if not supported(): + return False + current = read_tags(path) + remaining = [item for item in current if _tag_name(item) != tag] + if len(remaining) == len(current): + return False + return write_tags(path, remaining) + + +def _tag_name(tag: str) -> str: + """``"Red\\n6"`` -> ``"Red"``: strip the colour line Finder appends.""" + + return str(tag).split(COLOR_SEPARATOR, 1)[0] + + +def user_tags(source: Any, transient: Iterable[str] = ()) -> List[str]: + """Tags a person put there: colours stripped, our temporary tag removed. + + ``source`` is either a path to read or an already read list of tags. + """ + + if isinstance(source, (str, os.PathLike)): + raw: Iterable[str] = read_tags(Path(source)) + else: + raw = source or [] + drop = {str(item) for item in transient} + names: List[str] = [] + for tag in raw: + name = _tag_name(tag) + if not name or name in drop or name in names: + continue + names.append(name) + return names + + +# -------------------------------------------------------------------------- +# clear-tags +# -------------------------------------------------------------------------- + + +def clear_tags(paths: Iterable[Path], tag: str, *, dry_run: bool = False) -> ClearReport: + """Remove ``tag`` from every path that still has it.""" + + report = ClearReport(tag=tag, dry_run=dry_run) + for item in paths: + target = Path(item) + report.checked += 1 + if not os.path.lexists(str(target)): + report.missing += 1 + report.details.append({"path": str(target), "status": "missing"}) + continue + current = read_tags(target) + if not any(_tag_name(entry) == tag for entry in current): + report.skipped += 1 + report.details.append({"path": str(target), "status": "untagged"}) + continue + if dry_run: + report.cleared += 1 + report.details.append({"path": str(target), "status": "would-clear"}) + continue + if remove_tag(target, tag): + report.cleared += 1 + report.details.append({"path": str(target), "status": "cleared"}) + else: + report.failed += 1 + report.details.append({"path": str(target), "status": "failed"}) + return report + + +def find_tag_lists(managed: Path) -> List[Path]: + """Every ``*-move-tags-*.txt`` written by past runs, oldest first.""" + + folder = Path(managed) + if not folder.is_dir(): + return [] + return sorted(folder.glob(TAG_LIST_GLOB)) + + +def paths_from_lists(lists: Sequence[Path], *, home: Optional[Path] = None) -> List[Path]: + """Expand the ``$HOME/...`` lines of the given tag lists, in order, deduped.""" + + from . import manifest as manifest_module + from . import paths as paths_module + + seen = set() + found: List[Path] = [] + for source in lists: + for line in manifest_module.read_tag_list(Path(source)): + expanded = paths_module.expand_portable(line, home) + key = str(expanded) + if key in seen: + continue + seen.add(key) + found.append(expanded) + return found + + +def run(args: Any) -> int: + """``carl-file-organizer clear-tags [dir] [--from list]... [--all-lists] [--tag NAME]``.""" + + from . import config as config_module + from . import paths as paths_module + + root = paths_module.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + cfg = config_module.resolve_config(root, lang=getattr(args, "lang", None)) + lang = cfg.lang + managed = config_module.managed_dir(cfg) + tag = getattr(args, "tag", None) or config_module.transient_tag(cfg) + + chosen: List[Path] = [] + explicit = list(getattr(args, "from_lists", None) or []) + if explicit: + chosen.extend(Path(item) for item in explicit) + if getattr(args, "all_lists", False) or not explicit: + for candidate in find_tag_lists(managed): + if candidate not in chosen: + chosen.append(candidate) + + missing_lists = [item for item in chosen if not Path(item).is_file()] + for item in missing_lists: + print(message("missing", lang, path=item), file=sys.stderr) + chosen = [item for item in chosen if Path(item).is_file()] + + if not chosen: + print(message("no_lists", lang, glob=TAG_LIST_GLOB), file=sys.stderr) + return 2 + + if not supported(): + print(message("unsupported", lang)) + return 0 + + targets = paths_from_lists(chosen) + report = clear_tags(targets, tag, dry_run=bool(getattr(args, "dry_run", False))) + report.lists = chosen + + for entry in report.details: + print("{0}\t{1}".format(entry["status"], entry["path"])) + key = "dry_run" if report.dry_run else "cleared" + print(message(key, lang, tag=tag, count=report.cleared)) + print( + message( + "summary", + lang, + checked=report.checked, + cleared=report.cleared, + skipped=report.skipped + report.missing, + failed=report.failed, + ) + ) + return 2 if report.failed else 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py new file mode 100644 index 0000000..1f44901 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py @@ -0,0 +1,270 @@ +"""Move things to the system trash, or refuse; never fall back to deleting. + +macOS goes through Finder, Linux through ``gio``, Windows through the shell's +own recycle bin call. When none of them is available the backend is ``none`` +and every trash request raises :class:`TrashError`, which is the whole point: a +failed trash must never quietly turn into a permanent delete. + +The three backends are not symmetric. Finder and gio are ordinary commands, so +they run through :func:`subprocess.run` and take the ``runner`` seam. Windows +has no command that reliably reaches the recycle bin, so it calls +``SHFileOperationW`` through ctypes and takes the ``win_runner`` seam instead. +Both seams exist for the same reason: this module has to be testable on a +machine that is not the one being tested for. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable, List, Optional + +FINDER = "finder" +GIO = "gio" +WINDOWS = "win32" +NONE = "none" + +DEFAULT_TIMEOUT_S = 30.0 + +#: AppleScript error raised when the terminal is not allowed to drive Finder. +AUTOMATION_DENIED = "-1743" + +MESSAGES = { + "no_backend": { + "zh": "这台机器没有可用的废纸篓后端，拒绝处置：{path}", + "en": "No trash backend on this machine; refusing to dispose of {path}", + }, + "still_there": { + "zh": "废纸篓命令跑完了，但 {path} 还在原地，当作失败处理", + "en": "The trash command finished but {path} is still there; treating it as a failure", + }, + "automation_denied": { + "zh": ( + "Finder 拒绝了自动化请求（-1743）。去系统设置 > 隐私与安全性 > 自动化，" + "把终端对 Finder 的开关打开，再跑一次。" + ), + "en": ( + "Finder refused the automation request (-1743). Open System Settings > " + "Privacy & Security > Automation and allow your terminal to control Finder, " + "then run this again." + ), + }, + "failed": { + "zh": "废纸篓命令失败（退出码 {code}）：{detail}", + "en": "The trash command failed (exit code {code}): {detail}", + }, + "windows_unavailable": { + "zh": "这台 Windows 上调不到回收站接口（shell32），拒绝处置：{path}", + "en": "The recycle bin interface (shell32) is unreachable on this Windows; refusing to dispose of {path}", + }, + "windows_failed": { + "zh": "回收站调用失败（SHFileOperationW 返回 {code}）", + "en": "The recycle bin call failed (SHFileOperationW returned {code})", + }, + "windows_aborted": { + "zh": "回收站调用被中止，条目留在原地", + "en": "The recycle bin call was aborted; the item stayed where it was", + }, + "timeout": { + "zh": "废纸篓命令超时（{seconds} 秒），可能有确认框在等你点", + "en": "The trash command timed out after {seconds}s; a confirmation dialog may be waiting", + }, +} + + +class TrashError(RuntimeError): + """The item did not reach the trash. Nothing else is attempted.""" + + +def message(key: str, lang: str = "en", **kw: Any) -> str: + entry = MESSAGES.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def detect_backend() -> str: + """``finder`` on macOS, ``win32`` on Windows, ``gio`` elsewhere, else ``none``.""" + + if sys.platform == "darwin" and shutil.which("osascript"): + return FINDER + if sys.platform == "win32": + return WINDOWS if windows_available() else NONE + if shutil.which("gio"): + return GIO + return NONE + + +def applescript_quote(text: str) -> str: + """Escape a path for an AppleScript string literal (backslash first).""" + + return str(text).replace("\\", "\\\\").replace('"', '\\"') + + +def finder_command(path: Path) -> List[str]: + """The exact argv used for the Finder backend; never handed to a shell.""" + + absolute = os.path.abspath(str(path)) + script = 'tell application "Finder" to delete POSIX file "{0}"'.format( + applescript_quote(absolute) + ) + return ["osascript", "-e", script] + + +def gio_command(path: Path) -> List[str]: + return ["gio", "trash", "--", os.path.abspath(str(path))] + + +# --------------------------------------------------------------------------- +# Windows: the recycle bin through SHFileOperationW +# --------------------------------------------------------------------------- + +#: ``FO_DELETE``: the operation itself. +FO_DELETE = 0x0003 + +#: Send it to the recycle bin rather than unlinking it. Without this one flag +#: the very same call is a permanent delete, which is the outcome this whole +#: module exists to prevent, so it is never optional. +FOF_ALLOWUNDO = 0x0040 + +#: Do not put a yes/no dialog in front of a script that nobody is watching. +FOF_NOCONFIRMATION = 0x0010 + +#: No progress window. +FOF_SILENT = 0x0004 + +#: No modal error box either. A dialog nobody can click would hang the process +#: forever, and the return code already tells us everything the box would say. +FOF_NOERRORUI = 0x0400 + +#: The exact flag word this tool uses. One constant, so a reviewer can read the +#: policy in one line instead of reassembling it from call sites. +WINDOWS_FLAGS = FOF_ALLOWUNDO | FOF_NOCONFIRMATION | FOF_SILENT | FOF_NOERRORUI + + +def windows_path_argument(path: Path) -> str: + """The ``pFrom`` string: one absolute path, terminated by two NULs. + + ``SHFileOperationW`` reads a list of paths separated by NUL and ended by an + empty entry, so a single path still needs the second NUL. Building it here + rather than at the call site means the shape is testable on any platform. + """ + + return os.path.abspath(str(path)) + "\0\0" + + +def windows_available() -> bool: + """True when ``shell32`` can actually be reached from this interpreter.""" + + if sys.platform != "win32": + return False + try: + import ctypes + + return ctypes.windll.shell32 is not None # type: ignore[attr-defined] + except (ImportError, AttributeError, OSError): + return False + + +def windows_send_to_recycle_bin(path: Path) -> Any: + """Call ``SHFileOperationW`` for one path; return ``(code, aborted)``. + + Nothing here is imported until it is called, so the module still imports + cleanly on macOS and Linux, where ``ctypes.wintypes`` does not exist. + """ + + import ctypes + from ctypes import wintypes + + class SHFILEOPSTRUCTW(ctypes.Structure): + _fields_ = [ + ("hwnd", wintypes.HWND), + ("wFunc", wintypes.UINT), + ("pFrom", wintypes.LPCWSTR), + ("pTo", wintypes.LPCWSTR), + ("fFlags", ctypes.c_uint16), + ("fAnyOperationsAborted", wintypes.BOOL), + ("hNameMappings", ctypes.c_void_p), + ("lpszProgressTitle", wintypes.LPCWSTR), + ] + + operation = SHFILEOPSTRUCTW() + operation.hwnd = None + operation.wFunc = FO_DELETE + operation.pFrom = windows_path_argument(path) + operation.pTo = None + operation.fFlags = WINDOWS_FLAGS + operation.fAnyOperationsAborted = False + operation.hNameMappings = None + operation.lpszProgressTitle = None + + shell32 = ctypes.windll.shell32 # type: ignore[attr-defined] + shell32.SHFileOperationW.argtypes = [ctypes.POINTER(SHFILEOPSTRUCTW)] + shell32.SHFileOperationW.restype = ctypes.c_int + code = int(shell32.SHFileOperationW(ctypes.byref(operation))) + return code, bool(operation.fAnyOperationsAborted) + + +def _windows_trash( + target: Path, + lang: str, + win_runner: Optional[Callable[[Path], Any]], +) -> None: + call = win_runner if win_runner is not None else windows_send_to_recycle_bin + if win_runner is None and not windows_available(): + raise TrashError(message("windows_unavailable", lang, path=target)) + try: + code, aborted = call(target) + except OSError as error: + raise TrashError(str(error)) from error + if int(code) != 0: + raise TrashError(message("windows_failed", lang, code=int(code))) + if aborted: + raise TrashError(message("windows_aborted", lang)) + + +def move_to_trash( + path: Path, + *, + backend: str, + timeout_s: float = DEFAULT_TIMEOUT_S, + runner: Optional[Callable[..., Any]] = None, + win_runner: Optional[Callable[[Path], Any]] = None, + lang: str = "en", +) -> None: + """Send one item to the trash or raise; there is no third outcome.""" + + target = Path(path) + if backend not in (FINDER, GIO, WINDOWS): + raise TrashError(message("no_backend", lang, path=target)) + + if backend == WINDOWS: + _windows_trash(target, lang, win_runner) + if os.path.lexists(str(target)): + raise TrashError(message("still_there", lang, path=target)) + return + + argv = finder_command(target) if backend == FINDER else gio_command(target) + call = runner if runner is not None else subprocess.run + + try: + result = call(argv, capture_output=True, text=True, timeout=timeout_s) + except subprocess.TimeoutExpired as error: + raise TrashError(message("timeout", lang, seconds=timeout_s)) from error + except OSError as error: + raise TrashError(str(error)) from error + + code = int(getattr(result, "returncode", 0) or 0) + stderr = (getattr(result, "stderr", "") or "").strip() + if code != 0: + if AUTOMATION_DENIED in stderr: + raise TrashError(message("automation_denied", lang)) + raise TrashError(message("failed", lang, code=code, detail=stderr or "no output")) + + if os.path.lexists(str(target)): + raise TrashError(message("still_there", lang, path=target)) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py new file mode 100644 index 0000000..b19aa61 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py @@ -0,0 +1,444 @@ +"""Put things back where they came from, in reverse order. + +Undo reads either the audit log or a TSV manifest, walks it backwards, and +moves each item back to the path it started from. It is deliberately timid: +if the original path is occupied again, or the item is no longer where the +record says it is, that row is skipped and reported rather than forced. + +Trash and permanent delete cannot be undone from here. Trashed items are put +back from the Trash by the person; permanently deleted ones are gone, and +saying so plainly is more useful than pretending. +""" + +from __future__ import annotations + +import os +import shutil +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import manifest as manifest_module +from . import paths +from . import tags as tags_module + +#: Record statuses that describe a completed move, in both spellings. +UNDOABLE_STATUSES = ("moved", "MOVED", "restored", "RESTORED") + +TEXT = { + "trashed": { + "zh": "这一项进了废纸篓，在废纸篓里选放回原处就行，这里不动它", + "en": "This one went to the trash; put it back from there, not from here", + }, + "deleted": { + "zh": "这一项已永久删除，没法还原", + "en": "This one was permanently deleted and cannot be restored", + }, + "not_moved": { + "zh": "这条记录不是一次成功的移动（{status}），跳过", + "en": "This record is not a completed move ({status}); skipped", + }, + "no_paths": { + "zh": "这条记录缺少原路径或当前路径，跳过", + "en": "This record has no original or current path; skipped", + }, + "occupied": { + "zh": "原来的位置已经有同名的东西了，不覆盖，跳过", + "en": "Something is at the original path again; not overwriting, skipped", + }, + "missing": { + "zh": "现在的位置上找不到它，可能已经被人挪走了，跳过", + "en": "It is not where the record says it is; skipped", + }, + "outside": { + "zh": "路径不在整理目录里，拒绝移动", + "en": "That path is outside the folder; refused", + }, + "symlink": {"zh": "路径里有软链，拒绝移动", "en": "A symlink sits in this path; refused"}, + "restored": {"zh": "已放回原处", "en": "put back"}, + "dry_run": {"zh": "预演：会放回原处", "en": "dry run: would put it back"}, + "error": {"zh": "放回失败：{error}", "en": "putting it back failed: {error}"}, + "log": { + "zh": "撤销了 {count} 项移动，已放回原处；清单：`{manifest}`。", + "en": "Undid {count} move(s) and put them back; manifest: `{manifest}`.", + }, + "summary": { + "zh": "共 {total} 条：放回 {restored}，跳过 {skipped}，失败 {failed}", + "en": "{total} record(s): restored {restored}, skipped {skipped}, failed {failed}", + }, + "no_records": { + "zh": "这个文件里没有可撤销的记录：{path}", + "en": "No undoable record in {path}", + }, + "unknown_root": { + "zh": "看不出这些记录属于哪个整理目录：{path}", + "en": "Cannot tell which folder these records belong to: {path}", + }, +} + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class UndoStep: + record: Dict[str, Any] + kind: str + status: str + ok: bool + detail: str = "" + action_id: str = "" + original: Optional[Path] = None + current: Optional[Path] = None + size_bytes: int = 0 + + +@dataclass +class UndoReport: + run_id: str + dry_run: bool + results: List[Dict[str, Any]] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + manifest: Optional[Path] = None + audit: Optional[Path] = None + review_log: Optional[Path] = None + + +# -------------------------------------------------------------------------- +# reading records +# -------------------------------------------------------------------------- + + +def load_records(path: Path) -> List[Dict[str, Any]]: + """Read an ``audit.jsonl`` or a manifest ``.tsv`` into one common shape.""" + + source = Path(path).expanduser() + if not source.is_file(): + raise FileNotFoundError(str(source)) + suffix = source.suffix.casefold() + if suffix in (".jsonl", ".json", ".ndjson"): + return manifest_module.read_audit(source) + if suffix == ".tsv": + return [_from_manifest_row(row) for row in manifest_module.read_manifest(source)] + raise ValueError( + "undo reads an audit .jsonl or a manifest .tsv, not {0}".format(source.name) + ) + + +def _from_manifest_row(row: Dict[str, str]) -> Dict[str, Any]: + """A manifest row wearing the audit record's field names.""" + + target = row.get("target_path") or "" + try: + size = int(row.get("size_bytes") or 0) + except ValueError: + size = 0 + return { + "timestamp": row.get("moved_at", ""), + "action_id": row.get("action_id", ""), + "kind": row.get("kind") or ("move" if target else ""), + "source": row.get("original_path", ""), + "destination": target, + "status": row.get("status", ""), + "size_bytes": size, + "detail": row.get("reason", ""), + "origin": "manifest", + } + + +# -------------------------------------------------------------------------- +# planning the undo +# -------------------------------------------------------------------------- + + +def plan_undo( + records: Sequence[Dict[str, Any]], + *, + root: Path, + home: Optional[Path] = None, + lang: str = "en", +) -> List[UndoStep]: + """Reverse order, and one step per record so nothing disappears silently.""" + + steps: List[UndoStep] = [] + for record in reversed(list(records)): + kind = str(record.get("kind") or "") + status = str(record.get("status") or "") + action_id = str(record.get("action_id") or "") + + if kind == "trash" or status in ("trashed", "TRASHED"): + steps.append( + UndoStep(record, kind or "trash", "skipped", False, text("trashed", lang), action_id) + ) + continue + if kind == "delete" or status in ("deleted", "DELETED"): + steps.append( + UndoStep(record, kind or "delete", "skipped", False, text("deleted", lang), action_id) + ) + continue + if kind and kind != "move": + steps.append( + UndoStep(record, kind, "skipped", False, text("not_moved", lang, status=status), action_id) + ) + continue + if status not in UNDOABLE_STATUSES: + steps.append( + UndoStep(record, "move", "skipped", False, text("not_moved", lang, status=status), action_id) + ) + continue + + original_text = str(record.get("source") or "") + current_text = str(record.get("destination") or "") + if not original_text or not current_text: + steps.append( + UndoStep(record, "move", "skipped", False, text("no_paths", lang), action_id) + ) + continue + + original = paths.expand_portable(original_text, home) + current = paths.expand_portable(current_text, home) + try: + size = int(record.get("size_bytes") or 0) + except (TypeError, ValueError): + size = 0 + steps.append( + UndoStep( + record, + "move", + "ready", + True, + "", + action_id, + original=original, + current=current, + size_bytes=size, + ) + ) + return steps + + +# -------------------------------------------------------------------------- +# running it +# -------------------------------------------------------------------------- + + +def run_undo( + steps: Sequence[UndoStep], + *, + dry_run: bool = False, + now: Optional[datetime] = None, + root: Optional[Path] = None, + home: Optional[Path] = None, + config: Any = None, + lang: Optional[str] = None, + managed: Optional[Path] = None, + transient_tag: Optional[str] = None, +) -> UndoReport: + """Move each ready step back, then write its own manifest, audit and log.""" + + from . import config as config_module + + moment = now or datetime.now().astimezone() + run_id = manifest_module.timestamp_text(moment) + moved_at = moment.isoformat(timespec="seconds") + speech = lang or (getattr(config, "lang", None) or "en") + home_dir = Path(home) if home is not None else Path.home() + + if managed is None and config is not None: + managed = config_module.managed_dir(config) + if transient_tag is None and config is not None: + transient_tag = config_module.transient_tag(config) + + report = UndoReport(run_id=run_id, dry_run=dry_run) + rows: List[Dict[str, Any]] = [] + + for step in steps: + status, detail = _restore_one(step, root=root, dry_run=dry_run, lang=speech) + if status == "restored" and not dry_run and transient_tag and step.original: + try: + tags_module.remove_tag(step.original, transient_tag) + except Exception as error: # noqa: BLE001 - tagging never stops an undo + print("tag_warning: {0}\t{1}".format(step.original, error), file=sys.stderr) + + record = { + "timestamp": moved_at, + "run_id": run_id, + "action_id": step.action_id, + "subject_id": step.record.get("subject_id"), + "kind": "move", + "source": paths.portable(step.current, home_dir) if step.current else None, + "destination": paths.portable(step.original, home_dir) if step.original else None, + "status": "moved" if status == "restored" else status, + "detail": detail, + "size_bytes": step.size_bytes, + "manifest": "", + "tagged": False, + "trash_backend": "", + "undo": True, + } + report.results.append(record) + + if step.original and step.current: + rows.append( + { + "moved_at": moved_at, + "status": "RESTORED" if status == "restored" else status.upper(), + "original_path": paths.portable(step.current, home_dir), + "target_path": paths.portable(step.original, home_dir), + "size_bytes": step.size_bytes, + "reason": detail, + "restore_method": 'mv "{0}" "{1}"'.format( + paths.portable(step.original, home_dir), + paths.portable(step.current, home_dir), + ), + "kind": "move", + "action_id": step.action_id, + } + ) + + counts: Dict[str, int] = {"total": len(report.results)} + for record in report.results: + key = str(record.get("status")) + counts[key] = counts.get(key, 0) + 1 + report.counts = counts + + restored = counts.get("moved", 0) + if not dry_run and managed is not None and rows: + managed = Path(managed) + managed.mkdir(parents=True, exist_ok=True) + root_name = Path(root).name if root is not None else managed.parent.name + manifest_file = manifest_module.manifest_path(managed, root_name, "undo", moment) + manifest_module.write_manifest(manifest_file, rows) + report.manifest = manifest_file + manifest_text = paths.portable(manifest_file, home_dir) + for record in report.results: + record["manifest"] = manifest_text + report.audit = manifest_module.append_audit( + report.results, manifest_module.audit_path(managed) + ) + if restored: + report.review_log = manifest_module.append_review_log( + managed, + speech, + moment.date(), + [text("log", speech, count=restored, manifest=manifest_text)], + name=config_module.review_log_name(config) if config is not None else None, + ) + + return report + + +def _restore_one( + step: UndoStep, *, root: Optional[Path], dry_run: bool, lang: str +) -> Tuple[str, str]: + if not step.ok: + return step.status, step.detail + original = step.original + current = step.current + if original is None or current is None: + return "skipped", text("no_paths", lang) + + if os.path.lexists(str(original)): + return "skipped", text("occupied", lang) + if not os.path.lexists(str(current)): + return "skipped", text("missing", lang) + + if root is not None: + base = Path(root) + if not paths.inside_root(current, base) or not paths.inside_root(original.parent, base): + return "refused", text("outside", lang) + if paths.has_symlink_component(current, base) or paths.has_symlink_component( + original.parent, base + ): + return "refused", text("symlink", lang) + + if dry_run: + return "dry-run", text("dry_run", lang) + + try: + original.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(current), str(original)) + except (OSError, shutil.Error) as error: + return "failed", text("error", lang, error=error) + return "restored", text("restored", lang) + + +# -------------------------------------------------------------------------- +# undo subcommand +# -------------------------------------------------------------------------- + + +def infer_root(record_path: Path, steps: Sequence[UndoStep]) -> Optional[Path]: + """The folder these records belong to: the managed directory's parent.""" + + candidate = paths.realpath(Path(record_path)).parent.parent + for step in steps: + if step.original is not None and paths.inside_root(step.original.parent, candidate): + return candidate + originals = [str(step.original.parent) for step in steps if step.original is not None] + if not originals: + return candidate if candidate.is_dir() else None + try: + common = Path(os.path.commonpath(originals)) + except ValueError: + return None + return common if common.is_dir() else None + + +def run(args: Any) -> int: + """``carl-file-organizer undo [--dry-run]``.""" + + from . import config as config_module + + record_path = Path(args.record).expanduser() + records = load_records(record_path) + if not records: + print(text("no_records", "en", path=record_path), file=sys.stderr) + return 2 + + steps = plan_undo(records, root=Path("/"), home=None) + root = infer_root(record_path, steps) + if root is None: + print(text("unknown_root", "en", path=record_path), file=sys.stderr) + return 2 + + config = config_module.resolve_config(root) + lang = config.lang + steps = plan_undo(records, root=root, home=None, lang=lang) + + report = run_undo( + steps, + dry_run=bool(args.dry_run), + root=root, + config=config, + lang=lang, + managed=config_module.managed_dir(config), + ) + + for record in report.results: + print( + "{0}\t{1}\t{2}".format( + record.get("status"), record.get("source") or "", record.get("detail") + ) + ) + print( + text( + "summary", + lang, + total=report.counts.get("total", 0), + restored=report.counts.get("moved", 0) + report.counts.get("dry-run", 0), + skipped=report.counts.get("skipped", 0) + report.counts.get("refused", 0), + failed=report.counts.get("failed", 0), + ) + ) + if report.manifest: + print("manifest: {0}".format(report.manifest)) + return 2 if report.counts.get("failed", 0) else 0 diff --git a/skills/ops/carl-file-organizer/scripts/demo_fixture.py b/skills/ops/carl-file-organizer/scripts/demo_fixture.py new file mode 100644 index 0000000..0744353 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/demo_fixture.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Build a fake, throwaway "Downloads"-like directory for CI smoke tests, +manual command walkthroughs, and documentation screenshots. + +Zero third-party dependencies. Every path and filename here is invented; +nothing about it should ever resemble a real user's machine or files. + +Usage (paths are relative to this skill folder, and the target is yours to pick): + python3 scripts/demo_fixture.py [--now ISO] [--partitions zh|en] + python3 scripts/organize.py plan --lang zh + +Nothing outside is written, and is created if missing. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# --------------------------------------------------------------------------- +# time helpers +# --------------------------------------------------------------------------- + + +def _parse_now(value: str | None) -> datetime: + if value is None: + return datetime.now(timezone.utc) + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +_UTIME_SUPPORTS_SYMLINKS = os.utime in os.supports_follow_symlinks + + +def set_mtime(path: Path, when: datetime, *, is_symlink: bool = False) -> None: + ts = when.timestamp() + if is_symlink: + if _UTIME_SUPPORTS_SYMLINKS: + os.utime(path, (ts, ts), follow_symlinks=False) + # else: leave the symlink's own mtime alone (platform can't set it). + return + os.utime(path, (ts, ts)) + + +# --------------------------------------------------------------------------- +# small content writers +# --------------------------------------------------------------------------- + + +def write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def write_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def write_repeated_bytes(path: Path, size: int, pattern: bytes = b"GN-demo-") -> None: + """Write `size` real (non-sparse) bytes, cheaply, without pulling in any + third-party dependency.""" + path.parent.mkdir(parents=True, exist_ok=True) + chunk = (pattern * (65536 // len(pattern) + 1))[:65536] + with path.open("wb") as fh: + written = 0 + while written < size: + take = min(len(chunk), size - written) + fh.write(chunk[:take]) + written += take + + +def write_sparse_file(path: Path, size: int) -> None: + """Create a file that *reports* `size` bytes but only consumes disk + space for the one block actually written (a sparse file).""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("wb") as fh: + if size > 0: + fh.seek(size - 1) + fh.write(b"\0") + else: + fh.truncate(0) + + +def make_symlink(link_path: Path, target_name: str) -> None: + link_path.parent.mkdir(parents=True, exist_ok=True) + if link_path.exists() or link_path.is_symlink(): + link_path.unlink() + link_path.symlink_to(target_name) + + +# --------------------------------------------------------------------------- +# fixture assembly +# --------------------------------------------------------------------------- + +PARTITION_NAMES = { + "zh": { + "inbox": "00_收件箱", + "library_pdf": "20_知识库/文档资料/PDF", + }, + "en": { + "inbox": "00_Inbox", + "library_pdf": "20_Library/Documents/PDF", + }, +} + + +def build(target: Path, now: datetime, partitions: str) -> None: + target.mkdir(parents=True, exist_ok=True) + ago = lambda **kw: now - timedelta(**kw) # noqa: E731 + + # -- settled documents (past the document aging window) ----------------- + write_text(target / "quarterly-report.pdf", "%PDF-1.4\n% demo placeholder\n") + set_mtime(target / "quarterly-report.pdf", ago(days=5)) + + docs = { + "notes.md": "# Demo notes\n\nJust placeholder text.\n", + "slides.pptx": "PK\x03\x04 demo placeholder pptx bytes\n", + "data.csv": "col_a,col_b\n1,2\n3,4\n", + "readme.txt": "This is a placeholder readme for the demo fixture.\n", + } + for name, content in docs.items(): + write_text(target / name, content) + set_mtime(target / name, ago(days=3)) + + # -- fresh, still inside the aging window -------------------------------- + write_text(target / "fresh-draft.md", "# Draft\n\nStill warm.\n") + set_mtime(target / "fresh-draft.md", ago(hours=1)) + + write_bytes(target / "just-downloaded.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "just-downloaded.zip", ago(hours=2)) + + # -- image / video / web page (settled) ---------------------------------- + media = { + "photo.png": b"\x89PNG\r\n\x1a\n" + b"demo-placeholder-bytes", + "screenshot.jpg": b"\xff\xd8\xff\xe0" + b"demo-placeholder-bytes", + "clip.mp4": b"\x00\x00\x00\x18ftypmp42" + b"demo-placeholder-bytes", + "page.html": b"demo

    placeholder

    ", + } + for name, data in media.items(): + write_bytes(target / name, data) + set_mtime(target / name, ago(days=3)) + + # -- cold archives / installers (past the 7-day archive window) --------- + write_bytes(target / "old-tool.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "old-tool.zip", ago(days=10)) + + write_repeated_bytes(target / "installer.dmg", 2 * 1024 * 1024) + set_mtime(target / "installer.dmg", ago(days=12)) + + # -- paired archive + extracted directory -------------------------------- + write_bytes(target / "Archive.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "Archive.zip", ago(days=3)) + + write_text(target / "Archive" / "readme-inside.txt", "Extracted placeholder file one.\n") + write_text(target / "Archive" / "data-inside.txt", "Extracted placeholder file two.\n") + for p in (target / "Archive" / "readme-inside.txt", target / "Archive" / "data-inside.txt"): + set_mtime(p, ago(days=3)) + set_mtime(target / "Archive", ago(days=3)) + + # -- duplicates ----------------------------------------------------------- + same_content = "Invoice placeholder, both copies are byte-identical.\n" + write_text(target / "invoice.pdf", same_content) + write_text(target / "invoice (1).pdf", same_content) + set_mtime(target / "invoice.pdf", ago(days=3)) + set_mtime(target / "invoice (1).pdf", ago(days=3)) + + write_bytes(target / "export.png", b"\x89PNG\r\n\x1a\n" + b"export-A-placeholder") + write_bytes(target / "export_1.png", b"\x89PNG\r\n\x1a\n" + b"export-B-different-bytes") + set_mtime(target / "export.png", ago(days=3)) + set_mtime(target / "export_1.png", ago(days=3)) + + # -- sensitive naming (content is placeholder, never a real credential) -- + write_text(target / "api-credential.txt", "PLACEHOLDER_NOT_A_REAL_CREDENTIAL\n") + write_text( + target / "server.pem", + "-----BEGIN PLACEHOLDER-----\nthis is not a real certificate\n-----END PLACEHOLDER-----\n", + ) + write_text(target / "prod.env", "EXAMPLE_KEY=placeholder-not-real\n") + for name in ("api-credential.txt", "server.pem", "prod.env"): + set_mtime(target / name, ago(days=2)) + + # -- a project-shaped directory ------------------------------------------- + proj = target / "my-project" + write_text(proj / "package.json", '{"name": "demo-project", "version": "0.0.0"}\n') + write_text(proj / "src" / "index.js", "console.log('demo');\n") + write_text(proj / "node_modules" / "left-pad" / "index.js", "module.exports = function leftPad() {};\n") + write_text(proj / ".git" / "HEAD", "ref: refs/heads/main\n") + for p in ( + proj / "package.json", + proj / "src" / "index.js", + proj / "node_modules" / "left-pad" / "index.js", + proj / ".git" / "HEAD", + proj / "src", + proj / "node_modules" / "left-pad", + proj / "node_modules", + proj / ".git", + proj, + ): + set_mtime(p, ago(days=2)) + + # -- a worktree-shaped directory (.git is a *file*, not a directory) ----- + worktree = target / "my-project-worktree-fix" + write_text(worktree / ".git", "gitdir: /tmp/example/.git/worktrees/fix\n") + set_mtime(worktree / ".git", ago(days=2)) + set_mtime(worktree, ago(days=2)) + + # -- one big (sparse) file ------------------------------------------------- + write_sparse_file(target / "big-video.mov", 600 * 1024 * 1024) + set_mtime(target / "big-video.mov", ago(days=10)) + + # -- unknown extension / no extension --------------------------------------- + write_bytes(target / "mystery.blob", b"\x00\x01\x02demo-placeholder-bytes") + set_mtime(target / "mystery.blob", ago(days=5)) + + write_text(target / "README", "Placeholder file with no extension.\n") + set_mtime(target / "README", ago(days=5)) + + # -- hidden file + symlink --------------------------------------------------- + write_bytes(target / ".DS_Store", b"\x00\x00\x00\x01Bud1demo-placeholder") + set_mtime(target / ".DS_Store", ago(days=1)) + + make_symlink(target / "link-to-notes.md", "notes.md") + set_mtime(target / "link-to-notes.md", ago(days=3), is_symlink=True) + + # -- already-existing partition directories (established structure) ------- + names = PARTITION_NAMES[partitions] + inbox = target / names["inbox"] + inbox.mkdir(parents=True, exist_ok=True) + set_mtime(inbox, ago(days=30)) + + pdf_dir = target / names["library_pdf"] + write_text(pdf_dir / "quarterly-report.pdf", "%PDF-1.4\n% already-filed placeholder copy\n") + set_mtime(pdf_dir / "quarterly-report.pdf", ago(days=20)) + # walk the partition chain and set an established mtime on each level + parts = names["library_pdf"].split("/") + cur = target + for part in parts: + cur = cur / part + set_mtime(cur, ago(days=30)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("target_dir", type=Path, help="directory to fill with fixture files") + parser.add_argument("--now", type=str, default=None, help="ISO 8601 timestamp to treat as 'now'") + parser.add_argument( + "--partitions", + choices=("zh", "en"), + default="zh", + help="language for the pre-existing partition directory names (default: zh)", + ) + args = parser.parse_args(argv) + + now = _parse_now(args.now) + build(args.target_dir.resolve(), now, args.partitions) + print(f"Demo fixture written to: {args.target_dir.resolve()}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/hygiene.sh b/skills/ops/carl-file-organizer/scripts/hygiene.sh new file mode 100755 index 0000000..fb718ec --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/hygiene.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Repository hygiene: nothing that ships should carry one person's machine, one +# person's private project names, or borrowed vocabulary. +# +# The patterns live in this one file, and this file excludes itself from every +# search. Spelling them inline in the CI workflow made the workflow match +# itself, so the check failed on a clean tree. +# +# Two scopes, because the four rules are not the same kind of rule: +# +# repo the whole collection. A private path or an unattributed source is a +# publishing problem wherever it sits, so every skill is searched. +# skill this skill's own folder. Traffic-light wording and the product names +# this tool grew out of are house style here; another skill in the +# collection has its own history and its own vocabulary. +# +# tests/ is left out of the private-path rule on purpose. Nothing under tests/ +# is installed or published, and a test that proves a path gets redacted has to +# spell out the path being redacted. This rule protects what people receive. +# +# Usage: bash skills/ops/carl-file-organizer/scripts/hygiene.sh +# (exit 0 clean, exit 1 with the offending lines; run it from anywhere) + +set -uo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +skill_dir="$(cd "$here/.." && pwd)" +repo_root="$(cd "$skill_dir/../../.." && pwd)" +cd "$repo_root" + +skill_rel="${skill_dir#"$repo_root"/}" + +COMMON=(-r -n -E + --exclude-dir=.git + --exclude-dir=fixtures + --exclude-dir=__pycache__ + --exclude-dir=node_modules + --exclude-dir=.venv + --exclude-dir=dist + --exclude-dir=build + --exclude-dir=.gstack + --exclude=hygiene.sh +) + +failed=0 + +report() { + echo "::error::$1" + failed=1 +} + +# 1. No absolute home directories and no one machine's account name, anywhere +# that ships. See the note above about tests/. +# +# The path pattern wants /Users/ followed by something that looks like a real +# account, so code and prose that handle the prefix itself -- a redaction +# regex, a startswith check, a sentence listing the shapes -- read as what +# they are. Two rules make that work: an account name starts with a letter +# or digit, and it is longer than one character. Which makes /Users/x the +# house placeholder: use it when a document has to show the shape. +if grep "${COMMON[@]}" --exclude-dir=tests '/Users/[A-Za-z0-9_-]{2,}|carl2077|\$CODEX_HOME' .; then + report "private path pattern found; write \$HOME/... instead, or /Users/x when a document has to show the shape" +fi + +# 2. The acknowledgements table in the READMEs is the only place a source is +# named, and that holds for every skill in the collection. +matches=$(grep "${COMMON[@]}" -l '卡兹克|Khazix|storage-analyzer' . 2>/dev/null || true) +if [ -n "$matches" ]; then + outside=$(printf '%s\n' "$matches" | grep -vE '(^|/)README(\.en)?\.md$' || true) + if [ -n "$outside" ]; then + report "source named outside the README acknowledgements: $outside" + fi +fi + +# 3. Borrowed traffic-light vocabulary, in this skill's own writing. +if grep "${COMMON[@]}" '红灯|黄灯|绿灯' "$skill_rel"; then + report "traffic-light terminology found" +fi + +# 4. Private product and service names from the notes this tool grew out of, +# in this skill's own writing. +if grep "${COMMON[@]}" -i 'tikhub|deepseek|goodcase|订阅链接' "$skill_rel"; then + report "private product name found; use an invented name instead" +fi + +if [ "$failed" -eq 0 ]; then + echo "hygiene: clean" +fi +exit "$failed" diff --git a/skills/ops/carl-file-organizer/scripts/organize.py b/skills/ops/carl-file-organizer/scripts/organize.py new file mode 100755 index 0000000..e2df107 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/organize.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Entry point for the file organizer: tidy one folder, with approval in between. + +The engine is the ``carl_file_organizer`` package sitting next to this file. +Nothing is installed, nothing is on PYTHONPATH, so this script puts its own +directory at the front of ``sys.path`` and hands over to the package CLI. That +is the whole job; every flag and every subcommand lives in +``carl_file_organizer/cli.py``. + +Usage (run it with any Python 3.9 or newer; there are no dependencies): + + python3 organize.py plan ~/Downloads --lang zh + python3 organize.py build ~/Downloads/00_.../plan.json --notes notes.json + python3 organize.py review --serve + python3 organize.py apply + python3 organize.py undo + python3 organize.py status ~/Downloads + python3 organize.py clear-tags ~/Downloads + python3 organize.py --version + +``plan`` only reads. ``build`` only rewrites the plan's prose. ``apply`` is the +one that moves anything, and it moves only the ids you approved. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +if HERE not in sys.path: + sys.path.insert(0, HERE) + +from carl_file_organizer.cli import main # noqa: E402 - the path shim has to run first + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/storage_scan.py b/skills/ops/carl-file-organizer/scripts/storage_scan.py new file mode 100644 index 0000000..61691dd --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/storage_scan.py @@ -0,0 +1,1090 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""整机只读盘点扫描器（macOS + Windows）。 + +它只做两件事：给这台机器拍一张体检照，再把最占地方的那些目录按组量出来， +写成一份 storage-scan.json 交给 Agent 去读。它不移动、不删除、不改权限， +连一个文件的内容都不打开，只看 stat 里的大小和时间。 + +用法： + + python3 storage_scan.py [--out storage-scan.json] [--budget-seconds 60] + [--top-files 30] [--home PATH] + +尺寸口径：目录体积等于其下所有文件 st_size 之和，也就是逻辑大小。跟 du 的 +块大小口径会有出入，稀疏文件和 APFS 压缩过的文件会偏大。要的是量级判断， +不是审计账。 + +预算口径：--budget-seconds 是整轮硬上限，按组、按条目层层分摊。到点就停， +量到一半的条目带 partial=True，顶层 budget_hit=True。宁可标不准，不报假数。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time + +SCHEMA = "storage-scan/1" + +DEFAULT_BUDGET_SECONDS = 60.0 +DEFAULT_TOP_FILES = 30 +DEFAULT_MIN_FILE_MB = 200 # 进 top_files 的门槛 +DEFAULT_MIN_SIZE_MB = 50 # 进 groups 的门槛，比它小的条目不进输出 +SUBPROCESS_TIMEOUT = 5.0 # 每个体检子进程的硬超时 +MAX_ENTRIES_PER_ITEM = 400000 # 单个条目最多遍历的目录项数 +MEMO_MAX_DEPTH = 5 # 只缓存浅层目录体积，避免吃内存 +TIME_CHECK_EVERY = 128 # 每遍历多少项看一次表 + +# 有些组天生就没那么大，用全局门槛会把它们全滤没。安装包尤其明显。 +GROUP_MIN_SIZE_OVERRIDE = { + "downloads_installers": 10 * 1024 * 1024, + "trash": 1024 * 1024, + "recycle_bin": 1024 * 1024, +} + +GB = 1024 ** 3 + +# ---------------------------------------------------------------- 分类词表 + +REGENERABLE_DIR_NAMES = ( + "node_modules", ".next", "dist", "build", "target", "coverage", + "__pycache__", ".turbo", ".nuxt", ".svelte-kit", ".parcel-cache", +) + +INSTALLER_EXTS = ( + ".dmg", ".pkg", ".zip", ".iso", ".exe", ".msi", ".msix", ".appimage", + ".deb", ".rpm", ".xip", +) + +MEDIA_EXTS = ( + ".mp4", ".mov", ".m4v", ".mkv", ".avi", ".webm", ".mp3", ".wav", ".aiff", + ".flac", ".png", ".jpg", ".jpeg", ".heic", ".gif", ".tiff", ".psd", ".ai", + ".sketch", ".fig", ".key", ".pptx", +) + +# 命中即 dev_cache。写成路径段序列，小写、正斜杠。 +DEV_CACHE_PATHS = ( + ".npm", ".pnpm-store", ".yarn", ".cargo", ".gradle", ".m2", ".nuget", + ".cache", ".bun", ".deno", "go/pkg", "go-build", + "library/caches/pip", "library/caches/uv", "library/caches/ms-playwright", + "library/caches/homebrew", "library/caches/com.apple.dt.xcode", + "library/developer/xcode/deriveddata", + "library/developer/xcode/archives", + "library/developer/xcode/ios devicesupport", + "library/developer/coresimulator", + "appdata/local/pip/cache", "appdata/local/npm-cache", + "appdata/local/yarn", "appdata/local/nuget", "appdata/local/uv", + "appdata/local/ms-playwright", "appdata/local/temp", + "appdata/local/go-build", "appdata/local/microsoft/vscode", +) + +# 禁刀区。删了要么丢账号、要么丢聊天记录、要么丢一整个库。 +RED_PATHS = ( + "vm_bundles", "virtual machines", "parallels", "hyper-v", + "docker-desktop-data", "wsl", "user data", "profiles", + "library/safari", + "library/application support/google/chrome", + "library/application support/chromium", + "library/application support/firefox", + "library/application support/bravesoftware", + "library/application support/microsoft edge", + "library/keychains", "library/messages", "library/mail", + "library/mobile documents", "library/calendars", "library/contacts", + "library/containers/com.apple.mail", + "library/containers/com.apple.messages", + "library/group containers/group.com.apple.notes", + "com.apple.photos", ".git", ".obsidian", "onedrive", "dropbox", + "icloud drive", "appdata/roaming/microsoft/crypto", + "appdata/local/microsoft/edge/user data", + "appdata/local/google/chrome/user data", + "appdata/roaming/mozilla/firefox/profiles", +) + +# 目录名以这些结尾的一律 red。 +RED_SUFFIXES = ( + ".photoslibrary", ".utm", ".pvm", ".vmwarevm", ".sparsebundle", + ".musiclibrary", ".tvlibrary", ".imovielibrary", ".fcpbundle", +) + +# 遍历时撞见这些目录名，就说明条目里面埋着禁刀区，整体不能当删除单元。 +# 名字必须足够特异：messages、mail、profiles、.git 这种词随便哪个代码库里 +# 都有，拿来下沉会把 Downloads、Documents、projects 一起染红，三色就废了。 +RED_DIR_NAMES = frozenset(( + "vm_bundles", "user data", "virtual machines", "hyper-v", + "docker-desktop-data", +)) + +# 这些只作提示，不改颜色：值得 Agent 知道，但不足以判整个条目死刑。 +NOTABLE_DIR_NAMES = frozenset(( + ".git", ".obsidian", "keychains", "messages", "mail", "profiles", +)) + +CACHE_SEGMENTS = ("caches", "cache", "temp", "tmp", ".trash", "$recycle.bin") + +APP_DATA_SEGMENTS = ("application support", "containers", "group containers", + "appdata") + +MEDIA_SEGMENTS = ("movies", "music", "pictures") + +WORKSPACE_CANDIDATES = ( + "projects", "agent-workbench", "workspace", "work", "worktrees", + "dev", "src", "code", "repos", "Developer", +) + +# ---------------------------------------------------------------- 小工具 + + +def human(num_bytes): + """字节数转成人能念出来的字符串。""" + if num_bytes is None: + return "?" + value = float(num_bytes) + for unit in ("B", "KB", "MB", "GB", "TB", "PB"): + if abs(value) < 1024.0 or unit == "PB": + if unit == "B": + return "%d B" % int(value) + return "%.1f %s" % (value, unit) + value /= 1024.0 + return "%.1f PB" % value + + +def iso_time(timestamp): + try: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + except Exception: + return None + + +def run_cmd(args, timeout=SUBPROCESS_TIMEOUT): + """跑一个只读命令。超时、没这个命令、报错，都当没采到。""" + try: + proc = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=timeout, + ) + except Exception: + return "" + try: + return proc.stdout.decode("utf-8", "replace") + except Exception: + return "" + + +def path_segments(path): + """拆成小写路径段，正反斜杠都认。""" + return [seg for seg in path.replace("\\", "/").lower().split("/") if seg] + + +def haystack(path): + """给段序列匹配用的干草堆，两头都带斜杠。""" + return "/" + "/".join(path_segments(path)) + "/" + + +def matches_any(path, needles): + """路径里是否出现过这些段序列之一。按段匹配，.git 不会命中 .gitignore。""" + hay = haystack(path) + for needle in needles: + if ("/" + needle.strip("/") + "/") in hay: + return True + return False + + +def _windows_drive_letters(): + """列出 Windows 上所有盘符。测试里会把这个函数整体换掉。""" + try: + import ctypes + + bitmask = ctypes.windll.kernel32.GetLogicalDrives() # type: ignore[attr-defined] + except Exception: + return [] + return [chr(ord("A") + i) for i in range(26) if bitmask >> i & 1] + + +# ---------------------------------------------------------------- 扫描器 + + +class StorageScanner(object): + """只读扫描器，一个实例负责一轮扫描。""" + + def __init__( + self, + home=None, + platform_name=None, + budget_seconds=DEFAULT_BUDGET_SECONDS, + top_files=DEFAULT_TOP_FILES, + min_file_bytes=DEFAULT_MIN_FILE_MB * 1024 * 1024, + min_size_bytes=DEFAULT_MIN_SIZE_MB * 1024 * 1024, + with_system=True, + ): + self.platform = platform_name or sys.platform + self.is_windows = self.platform.startswith("win") + default_home = os.environ.get( + "USERPROFILE" if self.is_windows else "HOME" + ) or os.path.expanduser("~") + self.home = os.path.abspath(home or default_home) + self.budget_seconds = float(budget_seconds) + self.top_files_limit = int(top_files) + self.min_file_bytes = int(min_file_bytes) + self.min_size_bytes = int(min_size_bytes) + self.with_system = with_system + + self.started_monotonic = time.monotonic() + self.deadline = self.started_monotonic + self.budget_seconds + self.budget_hit = False + + self._memo = {} + self._memo_red = {} + self._denied = {} + self._big_files = {} + self._entries_this_item = 0 + self._red_inside = [] + self._notable_inside = [] + self._tick = 0 + self._abort = False + self.dropped_small = 0 + + # -------------------------------------------------- 路径 + + @property + def home_token(self): + return "%USERPROFILE%" if self.is_windows else "$HOME" + + @property + def sep(self): + return "\\" if self.is_windows else "/" + + def join(self, *parts): + return os.path.join(*parts) + + def portable(self, path): + """绝对路径写成 $HOME/... 或 %USERPROFILE%\\...，不落用户名。""" + try: + rel = os.path.relpath(path, self.home) + except ValueError: + return path + if rel == os.curdir: + return self.home_token + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return path + return self.home_token + self.sep + rel.replace(os.sep, self.sep) + + # -------------------------------------------------- 预算 + + def time_left(self): + return self.deadline - time.monotonic() + + def out_of_time(self): + if self.time_left() <= 0: + self.budget_hit = True + return True + return False + + def slice_deadline(self, share_of_remaining): + """给一段工作切一块时间，最少 0.3 秒。""" + remaining = max(self.time_left(), 0.0) + return time.monotonic() + max(remaining * share_of_remaining, 0.3) + + # -------------------------------------------------- 遍历 + + def note_denied(self, path, reason, counted_bytes=0): + if path in self._denied: + return + self._denied[path] = { + "path": path, + "path_portable": self.portable(path), + "reason": reason, + "counted_bytes": counted_bytes, + } + + def note_big_file(self, path, size, mtime): + if size < self.min_file_bytes or path in self._big_files: + return + self._big_files[path] = { + "path": path, + "path_portable": self.portable(path), + "name": os.path.basename(path), + "size_bytes": size, + "size_human": human(size), + "mtime": mtime, + "kind": self.guess_kind(path, is_dir=False), + } + + @staticmethod + def dir_marker(name): + """这个目录名是禁刀区标志物（red），还是只值得一提（note）。""" + low = name.lower() + if low in RED_DIR_NAMES: + return ("red", low) + for suffix in RED_SUFFIXES: + if low.endswith(suffix): + return ("red", low) + if low in NOTABLE_DIR_NAMES: + return ("note", low) + return None + + def walk_size(self, path, deadline, depth=0): + """递归量一个目录，返回 (bytes, partial, marks)。只读，不跟 symlink。 + + marks 是一串 (severity, name)，记下这棵树里埋着什么值得注意的东西。 + """ + memoized = self._memo.get(path) + if memoized is not None: + return memoized, False, self._memo_red.get(path, frozenset()) + + total = 0 + partial = False + marks = set() + try: + iterator = os.scandir(path) + except PermissionError: + self.note_denied(path, "permission_denied") + return 0, True, marks + except OSError as exc: + self.note_denied(path, "os_error:" + exc.__class__.__name__) + return 0, True, marks + + try: + with iterator as entries: + for entry in entries: + # 超时是全栈的：一层喊停，所有祖先层立刻跟着停， + # 否则每一层都要再攒够一轮才轮到看表，越深漂得越远。 + if self._abort: + partial = True + break + self._entries_this_item += 1 + self._tick += 1 + if self._tick % TIME_CHECK_EVERY == 0 and time.monotonic() > deadline: + self._abort = True + partial = True + break + if self._entries_this_item > MAX_ENTRIES_PER_ITEM: + self._abort = True + partial = True + break + try: + if entry.is_symlink(): + continue + if entry.is_dir(follow_symlinks=False): + marker = self.dir_marker(entry.name) + if marker and len(marks) < 12: + marks.add(marker) + sub, sub_partial, sub_marks = self.walk_size( + entry.path, deadline, depth + 1 + ) + total += sub + partial = partial or sub_partial + if sub_marks and len(marks) < 12: + marks.update(sub_marks) + else: + stat = entry.stat(follow_symlinks=False) + total += stat.st_size + if stat.st_size >= self.min_file_bytes: + self.note_big_file( + entry.path, stat.st_size, iso_time(stat.st_mtime) + ) + except PermissionError: + self.note_denied(entry.path, "permission_denied") + partial = True + except OSError: + continue + except PermissionError: + self.note_denied(path, "permission_denied", total) + partial = True + except OSError: + partial = True + + if not partial and depth <= MEMO_MAX_DEPTH: + self._memo[path] = total + self._memo_red[path] = frozenset(marks) + return total, partial, marks + + def measure(self, path, deadline): + """量一个路径：目录递归，文件 stat。返回 (bytes, partial, mtime)。""" + self._entries_this_item = 0 + self._red_inside = [] + self._notable_inside = [] + self._abort = False + try: + stat = os.stat(path) + except PermissionError: + self.note_denied(path, "permission_denied") + return None, True, None + except OSError: + return None, False, None + mtime = iso_time(stat.st_mtime) + if os.path.isdir(path): + size, partial, marks = self.walk_size(path, deadline) + self._red_inside = sorted(n for sev, n in marks if sev == "red") + self._notable_inside = sorted(n for sev, n in marks if sev == "note") + return size, partial, mtime + self.note_big_file(path, stat.st_size, mtime) + return stat.st_size, False, mtime + + # -------------------------------------------------- 分类 + + def guess_kind(self, path, is_dir=True): + segs = path_segments(path) + name = segs[-1] if segs else "" + + if is_dir and name in REGENERABLE_DIR_NAMES: + return "build_artifact" + if matches_any(path, DEV_CACHE_PATHS): + return "dev_cache" + if not is_dir: + for ext in INSTALLER_EXTS: + if name.endswith(ext): + return "installer" + for ext in MEDIA_EXTS: + if name.endswith(ext): + return "media" + for seg in CACHE_SEGMENTS: + if seg in segs: + return "cache" + for seg in APP_DATA_SEGMENTS: + if seg in segs: + return "app_data" + for seg in MEDIA_SEGMENTS: + if seg in segs: + return "media" + for workspace in WORKSPACE_CANDIDATES: + if workspace.lower() in segs: + return "project" + if is_dir: + try: + if os.path.isdir(os.path.join(path, ".git")): + return "project" + except OSError: + pass + return "unknown" + + def is_red(self, path): + segs = path_segments(path) + for seg in segs: + for suffix in RED_SUFFIXES: + if seg.endswith(suffix): + return True + if matches_any(path, RED_PATHS): + return True + # 目录里有 .obsidian 就是知识库主库，别碰。 + try: + if os.path.isdir(os.path.join(path, ".obsidian")): + return True + except OSError: + pass + return False + + def suggest_color(self, path, kind, group, red_inside=()): + """三色只是建议，最终解释权在 Agent 手里，但绿的门槛必须硬。 + + 判定顺序：自己在禁刀区就是 red；纯缓存和可再生产物是 green，哪怕里面 + 混着一个叫 profiles 的目录也照样可再生；剩下的，里面埋着禁刀区的算 + red，其余一律 yellow，交给人看一眼。 + """ + if self.is_red(path): + return "red" + if group in ("trash", "recycle_bin"): + return "green" + if kind in ("cache", "dev_cache", "build_artifact"): + return "green" + if red_inside: + return "red" + return "yellow" + + def restore_hint(self, path, kind, group): + name = (path_segments(path) or [""])[-1] + if name == "node_modules": + return "在项目根目录重跑 npm install / pnpm install / bun install" + if name in (".next", "dist", "build", "target", ".turbo", "coverage"): + return "重跑一次构建或 dev server 就会重新生成" + if name == "__pycache__": + return "下次导入模块时自动重建" + if group in ("trash", "recycle_bin") or name in (".trash", "$recycle.bin"): + return "清空之后不可恢复，先确认里面没有要留的东西" + if kind == "dev_cache": + return "下次装依赖或构建时会重新下载，第一次会慢一点" + if kind == "cache": + return "应用自己会重建，可能第一次启动稍慢" + if kind == "installer": + return "需要时去官网重新下载，老版本安装包一般不必留" + if kind == "app_data": + return "应用的账号、配置或本地数据，删了要重新登录或重新配置" + if kind == "project": + return "有远端仓库的可以重新 clone，没有的就是唯一副本" + if kind == "media": + return "素材本体，删了只能靠备份找回" + return "没有自动恢复途径，动之前先确认别处有没有副本" + + def make_item(self, path, group, size, partial, mtime, is_dir=True): + kind = self.guess_kind(path, is_dir=is_dir) + red_inside = list(self._red_inside) if is_dir else [] + item = { + "name": (path_segments(path) or [path])[-1], + "path": path, + "path_portable": self.portable(path), + "size_bytes": size, + "size_human": human(size), + "kind": kind, + "suggested_color": self.suggest_color(path, kind, group, red_inside), + "restore_hint": self.restore_hint(path, kind, group), + "mtime": mtime, + "group": group, + "partial": bool(partial), + } + if item["suggested_color"] == "red": + if self.is_red(path): + item["red_reason"] = "路径本身属于禁刀区" + elif red_inside: + item["red_reason"] = "里面埋着 " + "、".join(red_inside[:4]) + notable = list(self._notable_inside) if is_dir else [] + if notable: + item["contains_notable"] = notable[:6] + return item + + # -------------------------------------------------- 组 + + def children_of(self, path): + """一层子项，目录和文件都算，symlink 跳过。""" + out = [] + try: + with os.scandir(path) as entries: + for entry in entries: + try: + if entry.is_symlink(): + continue + except OSError: + continue + out.append(entry.path) + except PermissionError: + self.note_denied(path, "permission_denied") + except OSError: + pass + return sorted(out) + + def installers_in(self, path): + """Downloads 顶层的安装包，只看扩展名不进子目录。""" + out = [] + try: + with os.scandir(path) as entries: + for entry in entries: + try: + if entry.is_symlink() or entry.is_dir(follow_symlinks=False): + continue + except OSError: + continue + low = entry.name.lower() + if any(low.endswith(ext) for ext in INSTALLER_EXTS): + out.append(entry.path) + except PermissionError: + self.note_denied(path, "permission_denied") + except OSError: + pass + return sorted(out) + + def regenerable_under(self, roots, max_depth=3): + """工作区下找可再生目录，命中即当叶子，不再往里走。""" + found = [] + for root in roots: + if not os.path.isdir(root): + continue + stack = [(root, 0)] + while stack: + if self.out_of_time(): + return sorted(found) + current, depth = stack.pop() + try: + with os.scandir(current) as entries: + for entry in entries: + try: + if entry.is_symlink(): + continue + if not entry.is_dir(follow_symlinks=False): + continue + except OSError: + continue + if entry.name in REGENERABLE_DIR_NAMES: + found.append(entry.path) + continue + if entry.name.startswith(".") or depth + 1 >= max_depth: + continue + stack.append((entry.path, depth + 1)) + except PermissionError: + self.note_denied(current, "permission_denied") + except OSError: + continue + return sorted(found) + + def group_specs(self): + if self.is_windows: + return self.windows_group_specs(self.home) + return self.macos_group_specs(self.home) + + def macos_group_specs(self, home): + """顺序是有讲究的：先量深层，再量浅层，浅层就能吃到 memo。""" + library = self.join(home, "Library") + return [ + ("dev_caches", [ + self.join(home, ".npm"), + self.join(home, ".pnpm-store"), + self.join(home, ".cache"), + self.join(home, ".cargo"), + self.join(home, ".gradle"), + self.join(home, ".m2"), + self.join(home, ".nuget"), + self.join(home, ".bun"), + self.join(home, "go", "pkg"), + self.join(library, "Caches", "pip"), + self.join(library, "Caches", "uv"), + self.join(library, "Caches", "ms-playwright"), + self.join(library, "Caches", "Homebrew"), + self.join(library, "Caches", "com.apple.dt.Xcode"), + ]), + ("downloads_installers", ("installers", self.join(home, "Downloads"))), + ("trash", [self.join(home, ".Trash")]), + ("developer", [ + self.join(library, "Developer", "Xcode", "DerivedData"), + self.join(library, "Developer", "Xcode", "Archives"), + self.join(library, "Developer", "Xcode", "iOS DeviceSupport"), + self.join(library, "Developer", "CoreSimulator"), + ]), + ("caches", ("children", self.join(library, "Caches"))), + ("app_support", ("children", self.join(library, "Application Support"))), + ("containers", ("children", self.join(library, "Containers"))), + ("group_containers", ("children", self.join(library, "Group Containers"))), + ("logs", ("children", self.join(library, "Logs"))), + ("projects_regenerable", ("regenerable", [ + self.join(home, name) for name in WORKSPACE_CANDIDATES + ])), + ("library", ("children", library)), + ("home_top", ("children", home)), + ] + + def windows_group_specs(self, home): + local = self.join(home, "AppData", "Local") + roaming = self.join(home, "AppData", "Roaming") + system_drive = os.environ.get("SystemDrive", "C:") + return [ + ("dev_caches", [ + self.join(home, ".cache"), + self.join(home, ".npm"), + self.join(home, ".gradle"), + self.join(home, ".m2"), + self.join(home, ".nuget"), + self.join(home, ".cargo"), + ]), + ("downloads_installers", ("installers", self.join(home, "Downloads"))), + ("recycle_bin", [system_drive + "\\$Recycle.Bin"]), + ("appdata_local", ("children_plus", (local, [ + self.join(local, "Temp"), + self.join(local, "pip", "Cache"), + self.join(local, "npm-cache"), + self.join(local, "Yarn"), + self.join(local, "NuGet"), + self.join(local, "uv"), + self.join(local, "ms-playwright"), + self.join(local, "Google", "Chrome", "User Data"), + self.join(local, "Microsoft", "Edge", "User Data"), + self.join(local, "Mozilla", "Firefox", "Profiles"), + ]))), + ("appdata_roaming", ("children", roaming)), + ("projects_regenerable", ("regenerable", [ + self.join(home, name) for name in WORKSPACE_CANDIDATES + ])), + ("user_top", ("children", home)), + ] + + def resolve_spec(self, spec): + """把组的声明解析成一串待测路径。""" + if isinstance(spec, list): + return [path for path in spec if os.path.exists(path)] + mode, payload = spec + if mode == "children": + return self.children_of(payload) + if mode == "children_plus": + base, extras = payload + paths = self.children_of(base) + known = set(paths) + for extra in extras: + if extra not in known and os.path.exists(extra): + paths.append(extra) + return sorted(paths) + if mode == "installers": + return self.installers_in(payload) + if mode == "regenerable": + return self.regenerable_under(payload) + return [] + + def scan_one_group(self, name, spec, group_deadline): + paths = self.resolve_spec(spec) + floor = self.min_size_bytes + override = GROUP_MIN_SIZE_OVERRIDE.get(name) + if override is not None: + floor = min(floor, override) + items = [] + for index, path in enumerate(paths): + if os.path.islink(path): + continue + remaining_items = len(paths) - index + now = time.monotonic() + if now > group_deadline or self.out_of_time(): + self.budget_hit = True + break + share = max((group_deadline - now) / remaining_items, 0.3) + item_deadline = min(now + share, self.deadline) + is_dir = os.path.isdir(path) + size, partial, mtime = self.measure(path, item_deadline) + if size is None: + continue + if size < floor and not partial: + self.dropped_small += 1 + continue + items.append(self.make_item(path, name, size, partial, mtime, is_dir)) + items.sort(key=lambda entry: entry["size_bytes"], reverse=True) + return items + + def scan_groups(self): + specs = self.group_specs() + groups = {} + skipped = [] + for index, (name, spec) in enumerate(specs): + if self.out_of_time(): + groups[name] = [] + skipped.append(name) + continue + remaining_groups = len(specs) - index + group_deadline = self.slice_deadline(1.0 / remaining_groups) + groups[name] = self.scan_one_group(name, spec, group_deadline) + groups["system_notes"] = self.system_notes() + return groups, skipped + + def system_notes(self): + """只写不扫。这些地方要空间得走系统自己的路子，不能直接删。""" + if self.is_windows: + notes = [ + ("WinSxS 组件存储", "C:\\Windows\\WinSxS", + "系统组件的历史版本，看着大，里面大量是硬链接，直接删会让更新和修复失败。", + "管理员执行 DISM /Online /Cleanup-Image /StartComponentCleanup"), + ("Windows 更新缓存", "C:\\Windows\\SoftwareDistribution\\Download", + "已下载的更新包，装完就没用了，但更新服务在跑的时候删会卡住。", + "用磁盘清理里的 Windows 更新清理，或先停 wuauserv 再清"), + ("休眠文件 hiberfil.sys", "C:\\hiberfil.sys", + "内存镜像，通常跟物理内存一个量级，删掉等于关掉休眠和快速启动。", + "管理员执行 powercfg /hibernate off"), + ("虚拟内存 pagefile.sys", "C:\\pagefile.sys", + "系统页面文件，直接删要么蓝屏要么立刻被重建。", + "在系统属性的虚拟内存里改大小或换盘"), + ("系统还原点", "C:\\System Volume Information", + "还原点和卷影副本，占的是保留配额，资源管理器里看不到。", + "系统保护里调低磁盘使用比例，或删掉旧还原点"), + ] + else: + notes = [ + ("可清除空间 Purgeable", "/System/Volumes/Data", + "系统标记为可清除的部分，df 里算在已用，磁盘紧张时系统会自己回收。", + "真的缺空间时自动释放，也可以在存储设置里查看"), + ("时间机器本地快照", "/Volumes/com.apple.TimeMachine.localsnapshots", + "本地快照挂在系统卷上，用户目录里看不见，却实实在在占地方。", + "tmutil listlocalsnapshots / 由系统按空间压力自动回收"), + ("系统级缓存", "/Library/Caches", + "全机共用的缓存，属于系统卷，普通用户改不动。", + "需要管理员权限，一般不建议手动清"), + ] + out = [] + for title, path, what, how in notes: + out.append({ + "name": title, + "path": path, + "path_portable": path, + "size_bytes": None, + "size_human": "未测量", + "kind": "system", + "suggested_color": "red", + "restore_hint": how, + "mtime": None, + "group": "system_notes", + "partial": False, + "scanned": False, + "note": what, + }) + return out + + # -------------------------------------------------- 体检 + + def collect_disks(self): + if self.is_windows: + return self.collect_disks_windows() + return self.collect_disks_macos() + + def collect_disks_macos(self): + disks = [] + output = run_cmd(["df", "-k"]) + for line in output.splitlines()[1:]: + parts = line.split(None, 8) + if len(parts) < 9: + continue + device, blocks, used, avail, mount = ( + parts[0], parts[1], parts[2], parts[3], parts[8] + ) + if not device.startswith("/dev/"): + continue + try: + total_b = int(blocks) * 1024 + used_b = int(used) * 1024 + free_b = int(avail) * 1024 + except ValueError: + continue + disks.append(self.make_disk(mount, device, total_b, used_b, free_b, "df")) + if not disks: + fallback = self.fallback_disk("/") + if fallback: + disks.append(fallback) + return disks + + def collect_disks_windows(self): + disks = [] + for letter in _windows_drive_letters(): + root = letter + ":\\" + try: + total, used, free = shutil.disk_usage(root) + except Exception: + continue + disks.append( + self.make_disk(root, root, total, used, free, "shutil.disk_usage") + ) + if not disks: + fallback = self.fallback_disk(os.environ.get("SystemDrive", "C:") + "\\") + if fallback: + disks.append(fallback) + return disks + + def fallback_disk(self, root): + try: + total, used, free = shutil.disk_usage(root) + except Exception: + return None + return self.make_disk(root, root, total, used, free, "shutil.disk_usage") + + def make_disk(self, mount, device, total, used, free, source): + capacity = 0.0 + if used + free > 0: + capacity = round(100.0 * used / float(used + free), 1) + flags = [] + if capacity >= 85.0: + flags.append("high_usage") + if free < 30 * GB: + flags.append("low_free") + return { + "mount": mount, + "device": device, + "total_bytes": total, + "used_bytes": used, + "free_bytes": free, + "total_human": human(total), + "used_human": human(used), + "free_human": human(free), + "capacity_percent": capacity, + "flags": flags, + "source": source, + } + + def collect_system_info(self): + info = { + "platform": self.platform, + "home_token": self.home_token, + "python": "%d.%d.%d" % sys.version_info[:3], + "cpu_count": os.cpu_count(), + } + if self.is_windows: + import platform as platform_mod + + info["os"] = platform_mod.system() + " " + platform_mod.release() + info["os_build"] = platform_mod.version() + info["arch"] = os.environ.get( + "PROCESSOR_ARCHITECTURE", platform_mod.machine() + ) + return info + + product = run_cmd(["sw_vers", "-productVersion"]).strip() + if product: + info["os"] = "macOS " + product + build = run_cmd(["sw_vers", "-buildVersion"]).strip() + if build: + info["os_build"] = build + arch = run_cmd(["uname", "-m"]).strip() + if arch: + info["arch"] = arch + brand = run_cmd(["sysctl", "-n", "machdep.cpu.brand_string"]).strip() + if brand: + info["cpu"] = brand + memsize = run_cmd(["sysctl", "-n", "hw.memsize"]).strip() + if memsize.isdigit(): + info["memory_bytes"] = int(memsize) + info["memory_human"] = human(int(memsize)) + pressure = run_cmd(["memory_pressure"]) + if pressure: + info["memory_pressure_head"] = "\n".join(pressure.splitlines()[:8]) + vm_stat = run_cmd(["vm_stat"]) + if vm_stat: + info["vm_stat_head"] = "\n".join(vm_stat.splitlines()[:6]) + uptime = run_cmd(["uptime"]).strip() + if uptime: + info["uptime"] = uptime + return info + + # -------------------------------------------------- 汇总 + + def top_files(self): + files = sorted( + self._big_files.values(), + key=lambda entry: entry["size_bytes"], + reverse=True, + ) + return files[: self.top_files_limit] + + def run(self): + if self.with_system: + system = self.collect_system_info() + disks = self.collect_disks() + else: + system = {"platform": self.platform, "home_token": self.home_token} + disks = [] + + groups, skipped = self.scan_groups() + top = self.top_files() + denied = sorted(self._denied.values(), key=lambda entry: entry["path"]) + + measured = 0 + item_count = 0 + partial_count = 0 + for items in groups.values(): + for item in items: + if item.get("size_bytes"): + measured += item["size_bytes"] + item_count += 1 + if item.get("partial"): + partial_count += 1 + + denied_note = "" + if denied: + denied_note = ( + "有 %d 个目录读不到，体积没有计入，实际占用比这份数字更大。" + % len(denied) + ) + + return { + "schema": SCHEMA, + "generated_at": time.strftime("%Y-%m-%d %H:%M:%S %z"), + "platform": self.platform, + "home_token": self.home_token, + "system": system, + "disks": disks, + "groups": groups, + "top_files": top, + "denied": denied, + "elapsed_seconds": round(time.monotonic() - self.started_monotonic, 2), + "budget_seconds": self.budget_seconds, + "budget_hit": self.budget_hit, + "skipped_groups": skipped, + "summary": { + "group_count": len(groups), + "item_count": item_count, + "partial_item_count": partial_count, + "measured_bytes": measured, + "measured_human": human(measured), + "sizes_are_lower_bounds": partial_count > 0, + "partial_note": ( + "有 %d 个条目没量完就到点了，它们的体积是下限，实际更大。" + % partial_count + ) if partial_count else "", + "dropped_below_floor": self.dropped_small, + "min_size_bytes": self.min_size_bytes, + "min_file_bytes": self.min_file_bytes, + "denied_count": len(denied), + "denied_note": denied_note, + }, + } + + +# ---------------------------------------------------------------- 入口 + + +def run_scan(**kwargs): + """跑一轮扫描，返回 storage-scan.json 的内容（dict）。""" + return StorageScanner(**kwargs).run() + + +def format_summary(data): + lines = ["扫描用时 %.2fs%s" % ( + data["elapsed_seconds"], "，撞到预算上限" if data["budget_hit"] else "" + )] + for disk in data.get("disks", []): + lines.append("盘 %s：%s 已用 / %s 可用 / %s%%" % ( + disk["mount"], disk["used_human"], disk["free_human"], + disk["capacity_percent"], + )) + for name in sorted(data["groups"]): + items = data["groups"][name] + if items: + lines.append(" %-22s %d 项" % (name, len(items))) + top = data.get("top_files", [])[:5] + if top: + lines.append("最大的几个文件：") + for entry in top: + lines.append(" %10s %s" % (entry["size_human"], entry["path_portable"])) + for key in ("partial_note", "denied_note"): + if data["summary"].get(key): + lines.append(data["summary"][key]) + return "\n".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="整机只读盘点扫描（macOS + Windows），只量大小，不动文件。" + ) + parser.add_argument("--out", default="storage-scan.json", + help="输出文件路径，写 - 就打到标准输出") + parser.add_argument("--budget-seconds", type=float, + default=DEFAULT_BUDGET_SECONDS, help="整轮时间上限") + parser.add_argument("--top-files", type=int, default=DEFAULT_TOP_FILES, + help="大文件榜取前几名") + parser.add_argument("--min-file-mb", type=float, default=DEFAULT_MIN_FILE_MB, + help="进大文件榜的门槛，单位 MB") + parser.add_argument("--min-size-mb", type=float, default=DEFAULT_MIN_SIZE_MB, + help="进分组清单的门槛，单位 MB") + parser.add_argument("--home", default=None, help="指定要扫的家目录，默认当前用户") + parser.add_argument("--quiet", action="store_true", help="不打人类可读摘要") + args = parser.parse_args(argv) + + data = run_scan( + home=args.home, + budget_seconds=args.budget_seconds, + top_files=args.top_files, + min_file_bytes=int(args.min_file_mb * 1024 * 1024), + min_size_bytes=int(args.min_size_mb * 1024 * 1024), + ) + + payload = json.dumps(data, ensure_ascii=False, indent=2) + if args.out == "-": + sys.stdout.write(payload + "\n") + else: + out_path = os.path.abspath(args.out) + parent = os.path.dirname(out_path) + if parent and not os.path.isdir(parent): + os.makedirs(parent, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as handle: + handle.write(payload + "\n") + if not args.quiet: + sys.stdout.write("已写入 %s\n" % out_path) + if not args.quiet: + sys.stdout.write(format_summary(data) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/cfo_helpers.py b/tests/cfo_helpers.py new file mode 100644 index 0000000..9fe04e4 --- /dev/null +++ b/tests/cfo_helpers.py @@ -0,0 +1,85 @@ +"""Shared scaffolding for the engine tests. + +Two rules keep these tests honest: time is always injected, and the safety net +never touches the real machine. ``GUARD_OFFLINE`` gives every planner test an +empty reference list, an empty symlink scan root and no lsof at all, so a test +run cannot depend on what happens to be open on the developer's laptop. +""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +TZ = timezone(timedelta(hours=9)) +NOW = datetime(2026, 9, 2, 12, 0, 0, tzinfo=TZ) + +#: The safety net, wired to nothing. +GUARD_OFFLINE: Dict[str, Any] = { + "reference_files": [], + "scan_roots": [], + "lsof_enabled": False, +} + + +def guard_with(**overrides: Any) -> Dict[str, Any]: + options = dict(GUARD_OFFLINE) + options.update(overrides) + return options + + +def write(path: Path, content: str = "placeholder\n") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def age(path: Path, hours: float, now: datetime = NOW) -> Path: + """Backdate ``path`` so it reads as ``hours`` old relative to ``now``.""" + + when = (now - timedelta(hours=hours)).timestamp() + os.utime(str(path), (when, when)) + return path + + +def fake_lsof(mapping: Optional[Dict[str, str]] = None): + """An lsof stand-in: path substring -> ``command,pid``.""" + + table = mapping or {} + + class _Result(object): + def __init__(self, stdout: str, returncode: int = 0) -> None: + self.stdout = stdout + self.returncode = returncode + + def runner(argv, timeout_s): # noqa: ANN001 - mirrors subprocess.run's shape + lines = [] + for argument in argv: + for needle, who in table.items(): + if needle in str(argument): + command, pid = who.split(",") + lines.append("p{0}".format(pid)) + lines.append("c{0}".format(command)) + lines.append("n{0}".format(argument)) + return _Result("\n".join(lines) + ("\n" if lines else ""), 0 if lines else 1) + + return runner + + +def action_for(plan: Dict[str, Any], filename: str, kind: str = "move") -> Dict[str, Any]: + for action in plan["actions"]: + if action["filename"] == filename and action["kind"] == kind: + return action + raise AssertionError( + "no {0} action for {1}; saw {2}".format( + kind, + filename, + sorted({(a["filename"], a["kind"]) for a in plan["actions"]}), + ) + ) + + +def actions_for(plan: Dict[str, Any], filename: str): + return [action for action in plan["actions"] if action["filename"] == filename] diff --git a/tests/cfo_path.py b/tests/cfo_path.py new file mode 100644 index 0000000..2d83c40 --- /dev/null +++ b/tests/cfo_path.py @@ -0,0 +1,24 @@ +"""Put the file organizer's scripts directory on ``sys.path``. + +The engine lives inside a skill folder (``skills/ops/carl-file-organizer/scripts``) +rather than in an installed package, so every test that says +``import carl_file_organizer`` imports this module first. Importing the whole +package once is much cheaper than loading twenty modules by file path, and it +keeps the intra-package imports (``from . import guard``) working. + +Importing this module twice is free; the insert is guarded. +""" + +from __future__ import annotations + +import os +import sys + +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(TESTS_DIR) +SCRIPTS_DIR = os.path.join(REPO_ROOT, "skills", "ops", "carl-file-organizer", "scripts") +FIXTURES_DIR = os.path.join(TESTS_DIR, "fixtures", "carl_file_organizer") + +for _entry in (SCRIPTS_DIR, TESTS_DIR): + if _entry not in sys.path: + sys.path.insert(0, _entry) diff --git a/tests/fixtures/carl_file_organizer/plan-v2-sample.json b/tests/fixtures/carl_file_organizer/plan-v2-sample.json new file mode 100644 index 0000000..5c523ca --- /dev/null +++ b/tests/fixtures/carl_file_organizer/plan-v2-sample.json @@ -0,0 +1,1230 @@ +{ + "schema_version": 2, + "generator": { + "name": "carl-file-organizer", + "version": "0.3.0" + }, + "created_at": "2026-09-02T10:15:00+09:00", + "now": "2026-09-02T10:15:00+09:00", + "lang": "zh", + "lang_source": "existing-dirs", + "profile": { + "name": "tiered", + "source": "builtin+dotfile", + "dotfile": "$HOME/Downloads/.carl-file-organizer.json", + "hash": "3f9a1c0e2b77" + }, + "source_root": "/Users/example/Downloads", + "source_root_portable": "$HOME/Downloads", + "managed_dir": "/Users/example/Downloads/00_下载目录管理", + "managed_dir_portable": "$HOME/Downloads/00_下载目录管理", + "names": { + "inbox": "00_收件箱", + "inbox.pending": "00_收件箱/待判断", + "inbox.by_tag": "00_收件箱/按标签", + "inbox.dup": "00_收件箱/重复待确认", + "work": "10_工作区", + "work.tools": "10_工作区/工具与导出", + "work.tools.scripts": "10_工作区/工具与导出/脚本", + "library": "20_知识库", + "library.docs": "20_知识库/文档资料", + "library.docs.pdf": "20_知识库/文档资料/PDF", + "library.docs.txt": "20_知识库/文档资料/TXT", + "sensitive": "60_敏感信息", + "sensitive.pending": "60_敏感信息/待转移", + "sensitive.dup": "60_敏感信息/重复待确认", + "archive": "90_归档", + "archive.zip": "90_归档/压缩包", + "archive.zip.zip": "90_归档/压缩包/ZIP", + "archive.installers": "90_归档/安装包", + "archive.installers.dmg": "90_归档/安装包/DMG" + }, + "capabilities": { + "trash_backend": "finder", + "finder_tags": true, + "permanent_delete_offered": true + }, + "scan_policy": { + "depth": 1, + "hidden_files": "ignored", + "symlinks": "ignored", + "managed_partitions": "skipped", + "regenerable_scan_depth": 3, + "dir_size_limit": { + "max_entries": 50000, + "time_budget_s": 1.5 + }, + "hash_duplicates": true, + "hash_max_bytes": 2147483648, + "allow_referenced": false + }, + "disk": { + "total_bytes": 994662584320, + "used_bytes": 815394881536, + "free_bytes": 179267702784, + "used_percent": 81.98, + "target_percent": 90.0, + "bytes_to_remove_for_target": 0, + "note": "Moves on this volume reclaim 0 bytes." + }, + "mess": { + "score": 52, + "color": "yellow", + "color_source": "rule", + "counts": { + "loose": 12, + "aging": 1, + "pending": 2, + "groups": 3, + "top_dirs": 3 + }, + "reason": { + "zh": "散件 12 件，其中 2 件要你看一眼，还没到失控的地步", + "en": "12 loose items, 2 of them want a look from you; not out of hand yet" + } + }, + "summary": { + "entries": 12, + "files": 9, + "dirs": 3, + "bytes": 49115218011, + "by_tier": { + "aging": 1, + "duplicate": 9, + "forbidden": 1, + "in_use": 1, + "referenced": 1, + "regenerable": 2, + "routine": 3, + "sensitive": 1 + }, + "by_kind": { + "delete": 2, + "hold": 4, + "move": 8, + "trash": 5 + }, + "by_color": { + "green": { + "actions": 8, + "bytes": 419153355 + }, + "yellow": { + "actions": 7, + "bytes": 277336072 + }, + "red": { + "actions": 4, + "bytes": 48844385348 + } + }, + "by_destination": { + "00_收件箱/待判断": 2, + "00_收件箱/待判断/Archive": 2, + "20_知识库/文档资料/PDF": 3, + "60_敏感信息/待转移": 1 + }, + "groups": { + "pair": 1, + "duplicate": 1, + "regenerable": 1 + }, + "potential_reclaimable_bytes": 218429399, + "proposed_reclaimed_bytes": 0 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "actions": [ + { + "id": "9c1f0a7b2d3e4f55", + "subject_id": "1a2b3c4d5e6f7a8b", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/report.pdf", + "source_portable": "$HOME/Downloads/report.pdf", + "filename": "report.pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/report.pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 2411008, + "modified_ns": 1756700000000000000, + "age_hours": 71, + "rule": "ext:.pdf", + "reason": { + "zh": "文档按类型归档到 20_知识库/文档资料/PDF", + "en": "Document filed by type into 20_知识库/文档资料/PDF" + }, + "detail": { + "zh": "静置 71 小时，已满 48 小时", + "en": "aged 71h, past the 48h threshold" + }, + "confidence": 0.98, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "0b7e4411aa93c2d1", + "subject_id": "2b3c4d5e6f7a8b9c", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/api-key-backup.txt", + "source_portable": "$HOME/Downloads/api-key-backup.txt", + "filename": "api-key-backup.txt", + "destination": "/Users/example/Downloads/60_敏感信息/待转移/api-key-backup.txt", + "destination_portable": "$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt", + "destination_key": "sensitive.pending", + "size_bytes": 1204, + "modified_ns": 1756500000000000000, + "age_hours": 126, + "rule": "sensitive:marker", + "reason": { + "zh": "名字看着像凭证，先单独隔离，未读取内容", + "en": "The name looks like a credential, so it is isolated; contents were never read" + }, + "detail": { + "zh": "命中敏感词 api-key，不看静置期", + "en": "matched marker api-key; the settling period is skipped" + }, + "confidence": 0.9, + "tier": "sensitive", + "group_id": null, + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "a1b2c30d4e5f6071", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": "/Users/example/Downloads/00_收件箱/待判断/Archive/Archive.zip", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip", + "destination_key": "inbox.pending", + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "b2c3d40e5f607182", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 8915621, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "c3d4e50f60718293", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "delete", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [ + "allow-permanent-delete" + ], + "reclaims_bytes": 8915621, + "restore_method": { + "zh": "删掉就没了，只对可再生的构建产物用这一项", + "en": "Permanent; only use this for regenerable build output" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "d4e5f60718293a4b", + "subject_id": "4d5e6f7a8b9c0d1e", + "kind": "move", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Archive", + "source_portable": "$HOME/Downloads/Archive", + "filename": "Archive", + "destination": "/Users/example/Downloads/00_收件箱/待判断/Archive/Archive", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/Archive/Archive", + "destination_key": "inbox.pending", + "size_bytes": 9104882, + "modified_ns": 1756110000000000000, + "age_hours": 237, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 42, + "bytes": 9104882, + "truncated": false + }, + "color": "green" + }, + { + "id": "e5f60718293a4b5c", + "subject_id": "4d5e6f7a8b9c0d1e", + "kind": "trash", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Archive", + "source_portable": "$HOME/Downloads/Archive", + "filename": "Archive", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 9104882, + "modified_ns": 1756110000000000000, + "age_hours": 237, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 9104882, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 42, + "bytes": 9104882, + "truncated": false + }, + "color": "yellow" + }, + { + "id": "f60718293a4b5c6d", + "subject_id": "5e6f7a8b9c0d1e2f", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide.pdf", + "source_portable": "$HOME/Downloads/guide.pdf", + "filename": "guide.pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/guide.pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 1048576, + "modified_ns": 1756300000000000000, + "age_hours": 160, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "0718293a4b5c6d7e", + "subject_id": "5e6f7a8b9c0d1e2f", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide.pdf", + "source_portable": "$HOME/Downloads/guide.pdf", + "filename": "guide.pdf", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 1048576, + "modified_ns": 1756300000000000000, + "age_hours": 160, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 1048576, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "18293a4b5c6d7e8f", + "subject_id": "6f7a8b9c0d1e2f30", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide (1).pdf", + "source_portable": "$HOME/Downloads/guide (1).pdf", + "filename": "guide (1).pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/guide (1).pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 1048576, + "modified_ns": 1756310000000000000, + "age_hours": 157, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "293a4b5c6d7e8f90", + "subject_id": "6f7a8b9c0d1e2f30", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide (1).pdf", + "source_portable": "$HOME/Downloads/guide (1).pdf", + "filename": "guide (1).pdf", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 1048576, + "modified_ns": 1756310000000000000, + "age_hours": 157, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 1048576, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "3a4b5c6d7e8f9001", + "subject_id": "7a8b9c0d1e2f3041", + "kind": "move", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype", + "source_portable": "$HOME/Downloads/web-prototype", + "filename": "web-prototype", + "destination": "/Users/example/Downloads/00_收件箱/待判断/web-prototype", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/web-prototype", + "destination_key": "inbox.pending", + "size_bytes": 214748364, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "dir:pending", + "reason": { + "zh": "整个目录保留原名，先进待判断", + "en": "Whole folder keeps its name and waits in Pending" + }, + "detail": null, + "confidence": 0.6, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": true, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/web-prototype\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/web-prototype\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [ + "git-repo" + ], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [ + "git-dir", + "node-project" + ] + }, + "dir_stats": { + "files": 18422, + "bytes": 214748364, + "truncated": false + }, + "color": "yellow" + }, + { + "id": "4b5c6d7e8f900112", + "subject_id": "8b9c0d1e2f304152", + "kind": "trash", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype/node_modules", + "source_portable": "$HOME/Downloads/web-prototype/node_modules", + "filename": "node_modules", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 198311744, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "regenerable:node_modules", + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "detail": null, + "confidence": 0.99, + "tier": "regenerable", + "group_id": "regen-5c1d77ab", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 198311744, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 18120, + "bytes": 198311744, + "truncated": false + }, + "color": "green" + }, + { + "id": "5c6d7e8f90011223", + "subject_id": "8b9c0d1e2f304152", + "kind": "delete", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype/node_modules", + "source_portable": "$HOME/Downloads/web-prototype/node_modules", + "filename": "node_modules", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 198311744, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "regenerable:node_modules", + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "detail": null, + "confidence": 0.99, + "tier": "regenerable", + "group_id": "regen-5c1d77ab", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [ + "allow-permanent-delete" + ], + "reclaims_bytes": 198311744, + "restore_method": { + "zh": "删掉就没了，只对可再生的构建产物用这一项", + "en": "Permanent; only use this for regenerable build output" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 18120, + "bytes": 198311744, + "truncated": false + }, + "color": "green" + }, + { + "id": "6d7e8f9001122334", + "subject_id": "9c0d1e2f30415263", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/installer-2026.dmg", + "source_portable": "$HOME/Downloads/installer-2026.dmg", + "filename": "installer-2026.dmg", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 612368384, + "modified_ns": 1756750000000000000, + "age_hours": 57, + "rule": "hold:aging", + "reason": { + "zh": "还在静置期，已放 57 小时，满 168 小时再动", + "en": "Still settling: 57h of the required 168h" + }, + "detail": null, + "confidence": 1.0, + "tier": "aging", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "还在静置期，已放 57 小时，满 168 小时再动", + "en": "Still settling: 57h of the required 168h", + "ready_at": "2026-09-07T19:15:00+09:00" + }, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "7e8f900112233445", + "subject_id": "0d1e2f3041526374", + "kind": "hold", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Photos Library.photoslibrary", + "source_portable": "$HOME/Downloads/Photos Library.photoslibrary", + "filename": "Photos Library.photoslibrary", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 48231923712, + "modified_ns": 1756600000000000000, + "age_hours": 98, + "rule": "hold:forbidden", + "reason": { + "zh": "禁刀区，只展示不处理（forbidden:suffix:.photoslibrary）", + "en": "No-go zone, shown but never touched (forbidden:suffix:.photoslibrary)" + }, + "detail": null, + "confidence": 1.0, + "tier": "forbidden", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "系统媒体库，任何情况下都不动", + "en": "A system media library; never touched", + "label": "forbidden:suffix:.photoslibrary" + }, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 0, + "bytes": 48231923712, + "truncated": true + }, + "color": "red" + }, + { + "id": "8f90011223344556", + "subject_id": "1e2f304152637485", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/session.log", + "source_portable": "$HOME/Downloads/session.log", + "filename": "session.log", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 91204, + "modified_ns": 1756790000000000000, + "age_hours": 6, + "rule": "hold:in_use", + "reason": { + "zh": "现在有进程开着它：tail(48213)，先不动", + "en": "A process still has it open: tail(48213)" + }, + "detail": null, + "confidence": 1.0, + "tier": "in_use", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "进程 tail 正在读它，等它退出再整理", + "en": "tail is reading it; tidy up after it exits" + }, + "guard": { + "open_by": [ + { + "command": "tail", + "pid": 48213 + } + ], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "900112233445566a", + "subject_id": "2f30415263748596", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/backup-tool.sh", + "source_portable": "$HOME/Downloads/backup-tool.sh", + "filename": "backup-tool.sh", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 2048, + "modified_ns": 1756200000000000000, + "age_hours": 210, + "rule": "hold:referenced", + "reason": { + "zh": "有配置或脚本引用了这个路径：$HOME/Library/LaunchAgents/com.example.backup.plist:14，动它之前先改引用", + "en": "A config or script references this path: $HOME/Library/LaunchAgents/com.example.backup.plist:14; fix the reference first" + }, + "detail": null, + "confidence": 1.0, + "tier": "referenced", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "一个 launchd 任务写死了这个路径", + "en": "A launchd job hard-codes this path" + }, + "guard": { + "open_by": [], + "referenced_in": [ + { + "file": "$HOME/Library/LaunchAgents/com.example.backup.plist", + "line": 14, + "form": "$HOME/Downloads/backup-tool.sh" + }, + { + "file": "$HOME/.zshrc", + "line": 82, + "form": "~/Downloads/backup-tool.sh" + } + ], + "incoming_symlinks": [ + "$HOME/.local/bin/backup-tool" + ], + "shape": [ + "executable", + "shebang" + ] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "0112233445566a7b", + "subject_id": "30415263748596a7", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/dataset.parquet", + "source_portable": "$HOME/Downloads/dataset.parquet", + "filename": "dataset.parquet", + "destination": "/Users/example/Downloads/00_收件箱/待判断/dataset.parquet", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/dataset.parquet", + "destination_key": "inbox.pending", + "size_bytes": 33554432, + "modified_ns": 1756450000000000000, + "age_hours": 118, + "rule": "unknown-ext", + "reason": { + "zh": "扩展名不在规则表里，先进待判断，可以在页面上改去向", + "en": "Extension is not in the rule table; it waits in Pending and can be rerouted" + }, + "detail": null, + "confidence": 0.55, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": true, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/dataset.parquet\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/dataset.parquet\" \"$HOME/Downloads/\"" + }, + "tags": [ + "数据" + ], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + } + ], + "groups": [ + { + "group_id": "pair-7d2c1e9f", + "kind": "pair", + "tier": "duplicate", + "stem": "archive", + "evidence": "name", + "sha256": null, + "members": [ + "4d5e6f7a8b9c0d1e", + "3c4d5e6f7a8b9c0d" + ], + "member_labels": { + "4d5e6f7a8b9c0d1e": "Archive/", + "3c4d5e6f7a8b9c0d": "Archive.zip" + }, + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "options": [ + { + "key": "keep_all", + "label": { + "zh": "都保留，一起搬进待判断", + "en": "Keep both, move them together" + }, + "action_ids": [ + "d4e5f60718293a4b", + "a1b2c30d4e5f6071" + ], + "requires": [] + }, + { + "key": "keep:4d5e6f7a8b9c0d1e", + "label": { + "zh": "留目录，压缩包进废纸篓", + "en": "Keep the folder, trash the archive" + }, + "action_ids": [ + "d4e5f60718293a4b", + "b2c3d40e5f607182" + ], + "requires": [] + }, + { + "key": "keep:3c4d5e6f7a8b9c0d", + "label": { + "zh": "留压缩包，目录进废纸篓", + "en": "Keep the archive, trash the folder" + }, + "action_ids": [ + "a1b2c30d4e5f6071", + "e5f60718293a4b5c" + ], + "requires": [] + }, + { + "key": "keep:4d5e6f7a8b9c0d1e:delete", + "label": { + "zh": "留目录，压缩包永久删除", + "en": "Keep the folder, permanently delete the archive" + }, + "action_ids": [ + "d4e5f60718293a4b", + "c3d4e50f60718293" + ], + "requires": [ + "allow-permanent-delete" + ] + } + ], + "default_choice": "keep:4d5e6f7a8b9c0d1e", + "potential_bytes": 8915621, + "color": "yellow" + }, + { + "group_id": "dup-3ba91c04", + "kind": "duplicate", + "tier": "duplicate", + "stem": "guide.pdf", + "evidence": "sha256", + "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + "members": [ + "5e6f7a8b9c0d1e2f", + "6f7a8b9c0d1e2f30" + ], + "member_labels": { + "5e6f7a8b9c0d1e2f": "guide.pdf", + "6f7a8b9c0d1e2f30": "guide (1).pdf" + }, + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "options": [ + { + "key": "keep_all", + "label": { + "zh": "都保留", + "en": "Keep both" + }, + "action_ids": [ + "f60718293a4b5c6d", + "18293a4b5c6d7e8f" + ], + "requires": [] + }, + { + "key": "keep:5e6f7a8b9c0d1e2f", + "label": { + "zh": "留 guide.pdf，另一份进废纸篓", + "en": "Keep guide.pdf, trash the other" + }, + "action_ids": [ + "f60718293a4b5c6d", + "293a4b5c6d7e8f90" + ], + "requires": [] + }, + { + "key": "keep:6f7a8b9c0d1e2f30", + "label": { + "zh": "留 guide (1).pdf，另一份进废纸篓", + "en": "Keep guide (1).pdf, trash the other" + }, + "action_ids": [ + "18293a4b5c6d7e8f", + "0718293a4b5c6d7e" + ], + "requires": [] + } + ], + "default_choice": "keep:5e6f7a8b9c0d1e2f", + "potential_bytes": 1048576, + "color": "yellow" + }, + { + "group_id": "regen-5c1d77ab", + "kind": "regenerable", + "tier": "regenerable", + "stem": "node_modules", + "evidence": "basename", + "sha256": null, + "members": [ + "8b9c0d1e2f304152" + ], + "member_labels": { + "8b9c0d1e2f304152": "web-prototype/node_modules" + }, + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "options": [ + { + "key": "trash_all", + "label": { + "zh": "全部进废纸篓", + "en": "Trash all of them" + }, + "action_ids": [ + "4b5c6d7e8f900112" + ], + "requires": [] + }, + { + "key": "delete_all", + "label": { + "zh": "全部永久删除", + "en": "Permanently delete all of them" + }, + "action_ids": [ + "5c6d7e8f90011223" + ], + "requires": [ + "allow-permanent-delete" + ] + } + ], + "default_choice": "trash_all", + "potential_bytes": 198311744, + "color": "yellow" + } + ], + "notes": null, + "approved_action_ids": [], + "overrides": [], + "approved_at": null, + "approved_by": null +} diff --git a/tests/fixtures/carl_file_organizer/storage-analysis-sample.json b/tests/fixtures/carl_file_organizer/storage-analysis-sample.json new file mode 100644 index 0000000..45d9025 --- /dev/null +++ b/tests/fixtures/carl_file_organizer/storage-analysis-sample.json @@ -0,0 +1,245 @@ +{ + "schema": "carl-file-organizer/storage-analysis", + "schema_version": 1, + "generator": {"name": "carl-file-organizer", "version": "0.3.0"}, + "created_at": "2026-09-02T14:05:00+09:00", + "lang": "zh", + "scan": {"source": "storage-scan.json", "elapsed_s": 41.6, "denied": ["$HOME/Library/Mail", "$HOME/Library/Messages"], "top_n": 5}, + "machine": { + "hostname": "carl-macbook", + "os": "macOS 15.6", + "arch": "arm64", + "cpu": "Apple M2 Pro", + "memory_bytes": 34359738368, + "uptime_days": 12, + "home_portable": "$HOME" + }, + "disks": [ + {"mount": "/", "name": "Macintosh HD", "filesystem": "apfs", "total_bytes": 994662584320, "used_bytes": 815394881536, "free_bytes": 179267702784, "used_percent": 81.98, "primary": true}, + {"mount": "/Volumes/T7", "name": "T7", "filesystem": "exfat", "total_bytes": 2000398934016, "used_bytes": 612345678901, "free_bytes": 1388053255115, "used_percent": 30.61, "primary": false} + ], + "summary": { + "scanned_bytes": 401234567890, + "reclaimable_bytes": 61239487232, + "by_color": { + "green": {"count": 4, "bytes": 38654705664}, + "yellow": {"count": 3, "bytes": 22584781568}, + "red": {"count": 3, "bytes": 96636764160} + } + }, + "overview": { + "headline": "系统盘用了 82%，其中 36 GB 是能直接清的开发缓存，21 GB 要你看一眼，剩下的别碰。", + "priority": [ + {"text": "先清 Xcode DerivedData 和 pip、Homebrew 缓存，三项加起来 31 GB，删了下次构建自动重建。", "item_ids": ["st-derived-data", "st-pip-cache", "st-brew-cache"]}, + {"text": "Downloads 里 6 个安装包已经装过了，看一眼没用的丢废纸篓。", "item_ids": ["st-dl-installers"]}, + {"text": "Chrome 用户数据 9 GB，里面有登录态，别清，改在浏览器里清缓存。", "item_ids": ["st-chrome-profile"]} + ], + "long_term": [ + "把 ~/projects 里超过半年没动的仓库整个搬到 T7，node_modules 不用跟着走。", + "Time Machine 本地快照占着 40 GB，接上备份盘跑一次备份会自动瘦下去。", + "Photos 图库 45 GB 建议开 iCloud 优化存储，本地只留缩略图。" + ] + }, + "top5": [ + {"id": "st-photos", "name": "Photos Library.photoslibrary", "path_portable": "$HOME/Pictures/Photos Library.photoslibrary", "size_bytes": 48318382080, "kind": "media", "color": "red", "note": "系统媒体库，只能在照片 app 里管理"}, + {"id": "st-tm-snapshots", "name": "Time Machine 本地快照", "path_portable": "/System/Volumes/Data/.MobileBackups", "size_bytes": 42949672960, "kind": "system", "color": "red", "note": "系统自己管，接上备份盘就会释放"}, + {"id": "st-derived-data", "name": "DerivedData", "path_portable": "$HOME/Library/Developer/Xcode/DerivedData", "size_bytes": 23622320128, "kind": "build", "color": "green", "note": "Xcode 编译中间产物"}, + {"id": "st-chrome-profile", "name": "Chrome", "path_portable": "$HOME/Library/Application Support/Google/Chrome", "size_bytes": 9663676416, "kind": "app-data", "color": "yellow", "note": "浏览器配置和登录态"}, + {"id": "st-old-node-modules", "name": "node_modules（旧项目）", "path_portable": "$HOME/projects/legacy-dashboard/node_modules", "size_bytes": 8589934592, "kind": "build", "color": "green", "note": "半年没动的项目依赖"} + ], + "items": [ + { + "id": "st-derived-data", + "name": "DerivedData", + "path_portable": "$HOME/Library/Developer/Xcode/DerivedData", + "size_bytes": 23622320128, + "kind": "build", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "Xcode 每次编译留下的中间产物和索引。", + "why": "只要项目源码还在，这些东西下次打开工程会自动重建。", + "if_removed": "第一次编译会慢几分钟，之后一切照旧。", + "action": "trash", + "trash_paths": ["$HOME/Library/Developer/Xcode/DerivedData"], + "restore": "不需要恢复，重新编译即可。", + "commands": ["rm -rf \"$HOME/Library/Developer/Xcode/DerivedData\""], + "kill_processes": ["Xcode"], + "open_by": [] + }, + { + "id": "st-pip-cache", + "name": "pip 缓存", + "path_portable": "$HOME/Library/Caches/pip", + "size_bytes": 4294967296, + "kind": "cache", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "pip 下载过的 wheel 包缓存。", + "why": "纯缓存，装包时命中会快一点，没有也能从 PyPI 重新拉。", + "if_removed": "下次 pip install 多花一点下载时间。", + "action": "trash", + "trash_paths": ["$HOME/Library/Caches/pip"], + "restore": "不需要恢复。", + "commands": ["pip cache purge"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-brew-cache", + "name": "Homebrew 缓存", + "path_portable": "$HOME/Library/Caches/Homebrew", + "size_bytes": 3221225472, + "kind": "cache", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "brew 下载过的安装包。", + "why": "已经装好的软件不依赖它，brew 自己也建议定期清。", + "if_removed": "重装同一个包时要重新下载。", + "action": "trash", + "trash_paths": ["$HOME/Library/Caches/Homebrew"], + "restore": "不需要恢复。", + "commands": ["brew cleanup --prune=all"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-old-node-modules", + "name": "node_modules（旧项目）", + "path_portable": "$HOME/projects/legacy-dashboard/node_modules", + "size_bytes": 8589934592, + "kind": "build", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "legacy-dashboard 项目的依赖目录，项目本身 7 个月没改过。", + "why": "package-lock.json 还在，随时能 npm install 回来。", + "if_removed": "再打开这个项目前要先装一遍依赖。", + "action": "trash", + "trash_paths": ["$HOME/projects/legacy-dashboard/node_modules"], + "restore": "cd 到项目目录跑 npm install。", + "commands": ["cd \"$HOME/projects/legacy-dashboard\" && npm install"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-chrome-profile", + "name": "Chrome 用户数据", + "path_portable": "$HOME/Library/Application Support/Google/Chrome", + "size_bytes": 9663676416, + "kind": "app-data", + "tier": "user-data", + "color": "yellow", + "disk": "/", + "what": "Chrome 的配置、扩展、历史、登录态和缓存都在这里。", + "why": "里面有你所有网站的登录状态，整目录删掉等于重装浏览器。", + "if_removed": "所有网站要重新登录，扩展要重装，书签如果没同步就没了。", + "action": "manual", + "trash_paths": [], + "restore": "只能靠 Google 账号同步找回一部分。", + "commands": [], + "kill_processes": [], + "open_by": [], + "note": "想腾空间就在 Chrome 设置里清缓存，别动这个目录。" + }, + { + "id": "st-dl-installers", + "name": "Downloads 里的安装包", + "path_portable": "$HOME/Downloads", + "size_bytes": 6442450944, + "kind": "installer", + "tier": "cold", + "color": "yellow", + "disk": "/", + "what": "6 个 dmg 和 pkg，对应的软件都已经在应用程序里了。", + "why": "装过就没用了，但也许你想留着某个特定版本。", + "if_removed": "以后要重装得重新下载。", + "action": "trash", + "trash_paths": ["$HOME/Downloads/Docker.dmg", "$HOME/Downloads/Cursor-0.52.dmg", "$HOME/Downloads/Obsidian-1.8.pkg"], + "restore": "废纸篓里放回原处。", + "commands": [], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-ios-backups", + "name": "iPhone 备份", + "path_portable": "$HOME/Library/Application Support/MobileSync/Backup", + "size_bytes": 6479380480, + "kind": "app-data", + "tier": "user-data", + "color": "yellow", + "disk": "/", + "what": "Finder 给 iPhone 做的本地整机备份，最近一次是 3 个月前。", + "why": "如果你已经用 iCloud 备份，这份就是多余的；如果没有，它是唯一的备份。", + "if_removed": "手机出问题时没有本地备份可以恢复。", + "action": "manual", + "trash_paths": [], + "restore": "无法恢复，只能重新备份。", + "commands": [], + "kill_processes": [], + "open_by": [], + "note": "在 Finder 里点手机，进管理备份，确认过再删。" + }, + { + "id": "st-photos", + "name": "Photos Library.photoslibrary", + "path_portable": "$HOME/Pictures/Photos Library.photoslibrary", + "size_bytes": 48318382080, + "kind": "media", + "tier": "forbidden", + "color": "red", + "disk": "/", + "what": "照片 app 的整个图库，原图、缩略图和数据库都在里面。", + "why": "禁刀区。在 Finder 里动它任何一部分都会把图库弄坏。", + "if_removed": "照片全丢，iCloud 没开优化存储的话没得找回。", + "action": "keep", + "trash_paths": [], + "restore": "无法恢复。", + "commands": [], + "kill_processes": [], + "open_by": [{"command": "photolibraryd", "pid": 512}] + }, + { + "id": "st-tm-snapshots", + "name": "Time Machine 本地快照", + "path_portable": "/System/Volumes/Data/.MobileBackups", + "size_bytes": 42949672960, + "kind": "system", + "tier": "forbidden", + "color": "red", + "disk": "/", + "what": "备份盘不在身边时 macOS 自己留的本地快照。", + "why": "系统管理的空间，磁盘紧张时会自动腾出来，手动删没意义。", + "if_removed": "不该手动删。真要释放，接上备份盘跑一次备份。", + "action": "keep", + "trash_paths": [], + "restore": "不适用。", + "commands": ["tmutil listlocalsnapshots /"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-mail-containers", + "name": "邮件 app 数据", + "path_portable": "$HOME/Library/Containers/com.apple.mail", + "size_bytes": 5368709120, + "kind": "app-data", + "tier": "in_use", + "color": "red", + "disk": "/", + "what": "邮件 app 下载到本地的邮件和附件。", + "why": "现在有进程开着它，而且里面是你的邮件，删了要重新从服务器拉。", + "if_removed": "邮件 app 要重新同步所有账号，POP 账号的邮件直接没了。", + "action": "keep", + "trash_paths": [], + "restore": "只能从邮件服务器重新同步。", + "commands": [], + "kill_processes": [], + "open_by": [{"command": "Mail", "pid": 2210}] + } + ], + "executed_ids": [] +} diff --git a/tests/test_cfo_carl_file_organizer.py b/tests/test_cfo_carl_file_organizer.py new file mode 100644 index 0000000..1c22e6b --- /dev/null +++ b/tests/test_cfo_carl_file_organizer.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import tempfile +import unittest +from datetime import datetime, timedelta +from pathlib import Path + +from carl_file_organizer.classifier import classify +from carl_file_organizer.config import resolve_config +from carl_file_organizer.executor import PlanError, apply_approved_plan +from carl_file_organizer.planner import build_plan +from carl_file_organizer.report import render_report + +# The safety net has nowhere to look in these tests: no launch agents, no +# symlink scan roots, no lsof. Behaviour must not depend on the machine. +GUARD_OFFLINE = {"reference_files": [], "scan_roots": [], "lsof_enabled": False} + + +def simple_config(root: Path): + return resolve_config(root, profile="simple", lang="en") + + +def plan_for(root: Path, **kwargs): + # A day of imaginary distance, so freshly written fixture files are past the + # ten-minute "just touched it" guard. + later = datetime.now().astimezone() + timedelta(days=1) + return build_plan( + root, + config=kwargs.pop("config", None) or simple_config(root), + home=kwargs.pop("home", root), + now=kwargs.pop("now", later), + guard_options=kwargs.pop("guard_options", GUARD_OFFLINE), + **kwargs + ) + + +def action_for(plan, filename, kind="move"): + return next( + action + for action in plan["actions"] + if action["filename"] == filename and action["kind"] == kind + ) + + +class CarlFileOrganizerTests(unittest.TestCase): + def test_classifier_is_small_and_predictable(self) -> None: + self.assertEqual(classify(Path("paper.PDF"))[0], "Documents") + self.assertEqual(classify(Path("photo.heic"))[0], "Images") + self.assertEqual(classify(Path("unknown.blob"))[0], "Other") + + def test_plan_is_shallow_and_ignores_boundaries(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "notes.txt").write_text("hello", encoding="utf-8") + (root / ".secret.txt").write_text("hidden", encoding="utf-8") + nested = root / "Already Organized" + nested.mkdir() + (nested / "inside.pdf").write_text("stay", encoding="utf-8") + + plan = plan_for(root) + + self.assertEqual(plan["summary"]["files"], 1) + self.assertEqual( + action_for(plan, "notes.txt")["destination"], + str(root / "Documents" / "notes.txt"), + ) + self.assertEqual(plan["summary"]["proposed_reclaimed_bytes"], 0) + # the folder travels whole, and nothing inside it is filed separately + self.assertEqual(action_for(plan, "Already Organized")["subject_kind"], "dir") + self.assertTrue((nested / "inside.pdf").exists()) + + def test_duplicate_detection_hashes_equal_size_candidates(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "one.txt").write_text("same", encoding="utf-8") + (root / "two.txt").write_text("same", encoding="utf-8") + (root / "other.txt").write_text("diff", encoding="utf-8") + + plan = plan_for(root, hash_duplicates=True) + + groups = [group for group in plan["groups"] if group["kind"] == "duplicate"] + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["evidence"], "sha256") + self.assertEqual(groups[0]["potential_bytes"], 4) + + def test_only_approved_action_moves(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = root / "first.pdf" + second = root / "second.jpg" + first.write_text("document", encoding="utf-8") + second.write_text("image", encoding="utf-8") + config = simple_config(root) + plan = plan_for(root, config=config) + chosen = action_for(plan, "first.pdf") + plan["approved_action_ids"] = [chosen["id"]] + + report = apply_approved_plan(plan, config=config, home=root) + + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue((root / "Documents" / "first.pdf").exists()) + self.assertTrue(second.exists()) + + def test_unknown_approval_id_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "first.pdf").write_text("document", encoding="utf-8") + plan = plan_for(root) + plan["approved_action_ids"] = ["invented"] + with self.assertRaises(PlanError): + apply_approved_plan(plan, home=root) + + def test_tampered_destination_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "first.pdf").write_text("document", encoding="utf-8") + config = simple_config(root) + plan = plan_for(root, config=config) + action = action_for(plan, "first.pdf") + plan["approved_action_ids"] = [action["id"]] + action["destination"] = str(root / "Other" / "renamed.pdf") + action["destination_portable"] = "$HOME/Other/renamed.pdf" + with self.assertRaises(PlanError): + apply_approved_plan(plan, config=config, home=root) + + def test_report_embeds_readable_plan_and_export(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "paper.pdf").write_text("document", encoding="utf-8") + plan = plan_for(root) + html = render_report(plan) + self.assertIn("paper.pdf", html) + self.assertIn(json.dumps(plan["schema_version"]), html) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_classifier.py b/tests/test_cfo_classifier.py new file mode 100644 index 0000000..fe326ff --- /dev/null +++ b/tests/test_cfo_classifier.py @@ -0,0 +1,140 @@ +"""Rule matching, and above all the sensitive naming rules. + +The interesting cases are the near misses: a short word must never win by +substring, because that is exactly how ``r2`` once swallowed ``pixverse-r2.md``. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import unittest +from pathlib import Path + +from carl_file_organizer import classifier +from carl_file_organizer.config import resolve_config +from carl_file_organizer.classifier import ( + classify, + derivative_hint, + is_regenerable_name, + is_sensitive, + match_name_pattern, + match_pinned, + match_rule, +) + + +def tiered(): + return resolve_config(Path.cwd(), profile="tiered", lang="zh") + + +class ClassifierTests(unittest.TestCase): + def setUp(self) -> None: + self.config = tiered() + + # -- the original taxonomy stays untouched --------------------------- + def test_legacy_classify_still_answers(self) -> None: + self.assertEqual(classify(Path("paper.PDF"))[0], "Documents") + self.assertEqual(classify(Path("unknown.blob"))[0], "Other") + + # -- extension rules -------------------------------------------------- + def test_match_rule_reads_the_profile(self) -> None: + rule = match_rule(".pdf", self.config) + self.assertIsNotNone(rule) + self.assertEqual(rule.dest_key, "library.docs.pdf") + self.assertEqual(rule.min_age_hours, 48) + + def test_archive_rules_carry_the_longer_settling_period(self) -> None: + self.assertEqual(match_rule(".zip", self.config).min_age_hours, 168) + self.assertEqual(match_rule(".dmg", self.config).min_age_hours, 168) + + def test_compound_suffix_is_a_rule_of_its_own(self) -> None: + rule = match_rule(".tar.gz", self.config) + self.assertIsNotNone(rule) + self.assertEqual(rule.dest_key, "archive.zip.other") + + def test_unknown_suffix_has_no_rule(self) -> None: + self.assertIsNone(match_rule(".parquet", self.config)) + + # -- sensitive naming -------------------------------------------------- + def test_sensitive_exact_never_matches_a_substring(self) -> None: + self.assertIsNotNone(is_sensitive("key.txt", self.config)) + self.assertIsNone(is_sensitive("monkey.txt", self.config)) + self.assertIsNone(is_sensitive("pixverse-r2.md", self.config)) + self.assertIsNone(is_sensitive("keyboard-shortcuts.md", self.config)) + + def test_sensitive_layers_are_reported(self) -> None: + self.assertEqual(is_sensitive("id_rsa", self.config).where, "exact_names") + self.assertEqual(is_sensitive("key.txt", self.config).where, "exact_stems") + self.assertEqual(is_sensitive("server.pem", self.config).layer, "suffix") + hit = is_sensitive("my-password-list.md", self.config) + self.assertEqual(hit.layer, "marker") + self.assertEqual(hit.needle, "password") + + def test_dot_env_matches_as_a_suffix_and_as_a_whole_name(self) -> None: + self.assertIsNotNone(is_sensitive("prod.env", self.config)) + self.assertIsNotNone(is_sensitive(".env", self.config)) + + def test_chinese_markers_match_as_substrings(self) -> None: + self.assertIsNotNone(is_sensitive("公司营业执照扫描件.pdf", self.config)) + self.assertIsNotNone(is_sensitive("阿里云密钥备份.txt", self.config)) + + def test_a_short_ascii_marker_is_refused_at_load_time(self) -> None: + from carl_file_organizer.config import merge_profile + + with self.assertRaises(ValueError): + merge_profile(self.config.profile, {"sensitive": {"markers": ["r2"]}}) + + # -- derived copies, regenerable names, pinned ------------------------- + def test_derivative_hint(self) -> None: + self.assertEqual(derivative_hint("web-prototype-fix-20260830", self.config), "-fix-") + self.assertEqual(derivative_hint("repo-worktree-topic", self.config), "-worktree-") + self.assertIsNone(derivative_hint("fixture.pdf", self.config)) + + def test_is_regenerable_name(self) -> None: + self.assertTrue(is_regenerable_name("node_modules", self.config)) + self.assertTrue(is_regenerable_name("__pycache__", self.config)) + self.assertFalse(is_regenerable_name("src", self.config)) + + def test_match_pinned_takes_exact_names_and_globs(self) -> None: + self.assertEqual( + match_pinned("carl-file-organizer-approved-1.json", self.config), "carl-file-organizer-*.json" + ) + self.assertIsNone(match_pinned("notes.md", self.config)) + + def test_pinned_from_the_settings_file_is_exact(self) -> None: + config = resolve_config(Path.cwd(), profile="tiered", lang="zh") + config.profile["pinned"] = list(config.profile["pinned"]) + ["fable51-table"] + self.assertEqual(match_pinned("fable51-table", config), "fable51-table") + self.assertIsNone(match_pinned("fable51-table-copy", config)) + + # -- user written name rules ------------------------------------------ + def test_name_patterns_respect_suffix_filters(self) -> None: + config = tiered() + config.profile["name_patterns"] = [ + { + "id": "reports-sql", + "match": "contains", + "needle": "reports", + "suffixes": [".sql"], + "dest": "10_工作区/数据库与查询/SQL", + } + ] + self.assertIsNotNone(match_name_pattern("monthly-reports.sql", config)) + self.assertIsNone(match_name_pattern("monthly-reports.pdf", config)) + + def test_name_patterns_refuse_a_three_letter_substring(self) -> None: + config = tiered() + config.profile["name_patterns"] = [ + {"id": "tiny", "match": "contains", "needle": "abc", "dest": "00_收件箱/待判断"} + ] + self.assertIsNone(match_name_pattern("abcdef.md", config)) + + def test_archive_and_installer_suffix_lookups(self) -> None: + self.assertTrue(classifier.is_archive_suffix(".tar.gz", self.config)) + self.assertTrue(classifier.is_installer_suffix(".dmg", self.config)) + self.assertFalse(classifier.is_installer_suffix(".pdf", self.config)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_colors.py b/tests/test_cfo_colors.py new file mode 100644 index 0000000..d440bf7 --- /dev/null +++ b/tests/test_cfo_colors.py @@ -0,0 +1,303 @@ +"""The three colours: one per action, one per group, one for the folder. + +The colour is what the review page turns into a button, so these tests care +about one property above all others: no colour is ever written by hand. It is +derived from the decision the rules already made, which means a rule change can +break a test here but a wording change never can. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import planner +from carl_file_organizer.config import resolve_config +from carl_file_organizer.planner import ( + COLORS, + GREEN, + MESS_THRESHOLDS, + RED, + YELLOW, + action_color, + build_plan, + color_totals, + mess_color, + mess_counts, + mess_score, +) + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + action_for, + actions_for, + age, + write, +) + + +def plan_for(root: Path, **kwargs): + config = kwargs.pop("config", None) or resolve_config(root, profile="tiered", lang="zh") + return build_plan( + root, + config=config, + now=kwargs.pop("now", NOW), + home=kwargs.pop("home", root), + guard_options=kwargs.pop("guard_options", GUARD_OFFLINE), + trash_backend=kwargs.pop("trash_backend", "finder"), + **kwargs + ) + + +class ActionColorTests(unittest.TestCase): + """The pure function, with no plan around it.""" + + def test_every_hold_is_red_whatever_held_it(self) -> None: + for tier in ("forbidden", "in_use", "referenced", "aging", "pinned"): + self.assertEqual(action_color({"kind": "hold", "tier": tier}), RED, tier) + + def test_a_settled_move_is_green_and_a_pending_one_is_yellow(self) -> None: + settled = {"kind": "move", "tier": "routine", "reroutable": False} + pending = {"kind": "move", "tier": "routine", "reroutable": True} + self.assertEqual(action_color(settled), GREEN) + self.assertEqual(action_color(pending), YELLOW) + + def test_a_sensitive_move_is_green_because_isolating_is_the_safe_move(self) -> None: + action = {"kind": "move", "tier": "sensitive", "reroutable": False} + self.assertEqual(action_color(action), GREEN) + + def test_a_move_inside_a_group_is_green_the_disposal_next_to_it_is_not(self) -> None: + self.assertEqual( + action_color({"kind": "move", "tier": "duplicate", "reroutable": False}), GREEN + ) + for kind in ("trash", "delete"): + self.assertEqual(action_color({"kind": kind, "tier": "duplicate"}), YELLOW, kind) + + def test_only_regenerable_disposal_is_green(self) -> None: + for kind in ("trash", "delete"): + self.assertEqual(action_color({"kind": kind, "tier": "regenerable"}), GREEN, kind) + self.assertEqual(action_color({"kind": kind, "tier": "cold"}), YELLOW, kind) + + +class ColorInAPlanTests(unittest.TestCase): + def test_a_filed_document_is_green_and_an_unknown_extension_is_yellow(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + plan = plan_for(root) + + self.assertEqual(action_for(plan, "notes.md")["color"], GREEN) + self.assertEqual(action_for(plan, "mystery.zzz")["color"], YELLOW) + + def test_a_settling_archive_is_red_while_it_waits(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "bundle.zip", "PK\x03\x04"), hours=0.2) + plan = plan_for(root) + + hold = action_for(plan, "bundle.zip", kind="hold") + self.assertEqual(hold["tier"], "aging") + self.assertEqual(hold["color"], RED) + self.assertFalse(hold["approvable"]) + + def test_no_red_action_is_ever_approvable(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "id_rsa"), hours=100) + age(write(root / "fresh.pdf"), hours=0.1) + plan = plan_for(root) + + reds = [a for a in plan["actions"] if a["color"] == RED] + self.assertTrue(reds) + for action in reds: + self.assertFalse(action["approvable"], action["id"]) + self.assertEqual(action["kind"], "hold") + + def test_a_group_is_always_a_yellow_question(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "release.zip", "PK\x03\x04"), hours=100) + age(write(root / "release" / "readme.txt"), hours=100) + age(root / "release", hours=100) + plan = plan_for(root) + + self.assertTrue(plan["groups"]) + for group in plan["groups"]: + self.assertEqual(group["color"], YELLOW, group["group_id"]) + # the moves inside the group stay green; only the disposals are yellow + moves = [a for a in plan["actions"] if a["kind"] == "move" and a["group_id"]] + self.assertTrue(moves) + for action in moves: + self.assertEqual(action["color"], GREEN, action["id"]) + + def test_a_duplicate_head_keeps_its_colour_after_the_tier_is_rewritten(self) -> None: + """build_plan rewrites the tier of a duplicate's move after the fact.""" + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf", "same body\n"), hours=100) + age(write(root / "report copy.pdf", "same body\n"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + heads = [a for a in plan["actions"] if a["kind"] == "move" and a["group_id"]] + self.assertTrue(heads) + for head in heads: + self.assertEqual(head["tier"], "duplicate") + self.assertFalse(head["reroutable"]) + self.assertEqual(head["color"], GREEN, head["id"]) + for action in actions_for(plan, "report copy.pdf"): + if action["kind"] in ("trash", "delete"): + self.assertEqual(action["color"], YELLOW, action["id"]) + + def test_by_color_adds_up_to_every_action(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "fresh.pdf"), hours=0.1) + plan = plan_for(root) + + by_color = plan["summary"]["by_color"] + self.assertEqual(set(by_color), set(COLORS)) + self.assertEqual( + sum(slot["actions"] for slot in by_color.values()), len(plan["actions"]) + ) + for colour in COLORS: + wearing = [a for a in plan["actions"] if a["color"] == colour] + self.assertEqual(by_color[colour]["actions"], len(wearing), colour) + self.assertEqual( + by_color[colour]["bytes"], + sum(a["size_bytes"] for a in wearing), + colour, + ) + + def test_color_totals_ignores_an_action_with_no_colour(self) -> None: + totals = color_totals([{"color": "chartreuse", "size_bytes": 10}]) + self.assertEqual(sum(slot["actions"] for slot in totals.values()), 0) + + +class MessTests(unittest.TestCase): + def test_thresholds_are_the_ones_written_down(self) -> None: + self.assertEqual(MESS_THRESHOLDS[GREEN], {"loose": 3, "pending": 0}) + self.assertEqual(MESS_THRESHOLDS[YELLOW], {"loose": 15, "pending": 5}) + + def test_a_tidy_folder_is_green(self) -> None: + counts = {"loose": 3, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), GREEN) + + def test_one_thing_waiting_on_you_takes_green_away(self) -> None: + counts = {"loose": 2, "aging": 0, "pending": 1, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), YELLOW) + + def test_a_big_pile_with_few_questions_is_still_only_yellow(self) -> None: + counts = {"loose": 200, "aging": 0, "pending": 4, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), YELLOW) + + def test_many_items_and_many_questions_is_red(self) -> None: + counts = {"loose": 40, "aging": 3, "pending": 12, "groups": 4, "top_dirs": 9} + self.assertEqual(mess_color(counts), RED) + + def test_score_is_bounded_and_grows_with_the_mess(self) -> None: + tidy = {"loose": 1, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + awful = {"loose": 900, "aging": 90, "pending": 400, "groups": 60, "top_dirs": 80} + self.assertLess(mess_score(tidy), mess_score(awful)) + self.assertGreaterEqual(mess_score(tidy), 0) + self.assertLessEqual(mess_score(awful), 100) + + def test_the_bar_never_pegs_so_it_can_always_move(self) -> None: + """A merely untidy folder must not use up the whole bar.""" + + untidy = {"loose": 27, "aging": 2, "pending": 3, "groups": 4, "top_dirs": 3} + self.assertEqual(mess_color(untidy), YELLOW) + self.assertLess(mess_score(untidy), 90) + + landslide = {"loose": 400, "aging": 30, "pending": 90, "groups": 40, "top_dirs": 40} + self.assertEqual(mess_color(landslide), RED) + self.assertGreater(mess_score(landslide), mess_score(untidy)) + self.assertLess(mess_score(landslide), 100) + + def test_clearing_a_pile_visibly_moves_the_bar(self) -> None: + before = {"loose": 200, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + after = {"loose": 170, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertGreater(mess_score(before) - mess_score(after), 0) + + def test_loose_and_pending_are_not_counted_twice(self) -> None: + """They describe the same pile two ways; adding them would double it.""" + + counts = {"loose": 15, "aging": 0, "pending": 5, "groups": 0, "top_dirs": 0} + only_loose = dict(counts, pending=0) + self.assertEqual(mess_score(counts), mess_score(only_loose)) + + def test_score_and_colour_are_computed_separately(self) -> None: + """The bar length is not allowed to become the verdict, or the reverse.""" + + counts = {"loose": 200, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertGreater(mess_score(counts), 80) + self.assertEqual(mess_color(counts), YELLOW) + + def test_counts_come_from_the_scan_not_from_the_action_list(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "folder" / "inner.txt"), hours=100) + age(root / "folder", hours=100) + plan = plan_for(root) + + counts = plan["mess"]["counts"] + self.assertEqual(counts["loose"], plan["summary"]["entries"]) + self.assertEqual(counts["top_dirs"], plan["summary"]["dirs"]) + self.assertEqual(counts["groups"], len(plan["groups"])) + self.assertEqual( + counts["pending"], + len({a["subject_id"] for a in plan["actions"] if a["reroutable"]}), + ) + + def test_a_plan_says_out_loud_who_chose_the_folder_colour(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + + self.assertEqual(plan["mess"]["color_source"], "rule") + self.assertIn(plan["mess"]["color"], COLORS) + self.assertEqual(set(plan["mess"]["reason"]), {"zh", "en"}) + self.assertIn(str(plan["mess"]["counts"]["loose"]), plan["mess"]["reason"]["zh"]) + + def test_an_empty_folder_is_green(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan = plan_for(root) + + self.assertEqual(plan["mess"]["color"], GREEN) + self.assertEqual(plan["mess"]["score"], 0) + self.assertEqual(plan["mess"]["counts"]["loose"], 0) + + def test_mess_counts_reads_entry_objects_and_actions_together(self) -> None: + class FakeEntry(object): + def __init__(self, is_dir: bool) -> None: + self.is_dir = is_dir + + entries = [FakeEntry(False), FakeEntry(True), FakeEntry(True)] + actions = [ + {"subject_id": "a", "kind": "hold", "tier": "aging", "reroutable": False}, + {"subject_id": "b", "kind": "move", "tier": "routine", "reroutable": True}, + {"subject_id": "b", "kind": "trash", "tier": "cold", "reroutable": False}, + ] + counts = mess_counts(entries, actions, [{"group_id": "g"}]) + self.assertEqual( + counts, {"loose": 3, "aging": 1, "pending": 1, "groups": 1, "top_dirs": 2} + ) + + def test_the_module_only_knows_three_colours(self) -> None: + self.assertEqual(planner.COLORS, ("green", "yellow", "red")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_config.py b/tests/test_cfo_config.py new file mode 100644 index 0000000..6ad5e3b --- /dev/null +++ b/tests/test_cfo_config.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import os +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest import mock + +from carl_file_organizer import config, paths + + +def _write_dotfile(root: Path, payload: dict) -> Path: + path = root / config.DOTFILE_NAME + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +class LanguageDetectionTests(unittest.TestCase): + def setUp(self) -> None: + patcher = mock.patch.object(config, "_apple_languages", return_value=None) + patcher.start() + self.addCleanup(patcher.stop) + self.profile = config.load_builtin_profile("tiered") + + def test_lang_precedence(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "00_收件箱").mkdir() + + self.assertEqual( + config.detect_lang( + root, + "en", + env={config.LANG_ENV: "zh"}, + dotfile={"lang": "zh"}, + profile=self.profile, + ), + ("en", "flag"), + ) + self.assertEqual( + config.detect_lang( + root, + None, + env={config.LANG_ENV: "en"}, + dotfile={"lang": "zh"}, + profile=self.profile, + ), + ("zh", "dotfile"), + ) + self.assertEqual( + config.detect_lang( + root, None, env={config.LANG_ENV: "en"}, dotfile={}, profile=self.profile + ), + ("en", "env"), + ) + self.assertEqual( + config.detect_lang(root, None, env={}, dotfile={}, profile=self.profile), + ("zh", "existing-dirs"), + ) + + with tempfile.TemporaryDirectory() as temporary: + bare = Path(temporary) + self.assertEqual( + config.detect_lang( + bare, None, env={"LANG": "zh_CN.UTF-8"}, dotfile={}, profile=self.profile + ), + ("zh", "posix-locale"), + ) + self.assertEqual( + config.detect_lang(bare, None, env={}, dotfile={}, profile=self.profile), + ("en", "default"), + ) + + def test_apple_languages_beats_posix_locale(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + bare = Path(temporary) + with mock.patch.object(config, "_apple_languages", return_value="zh-Hans-CN"): + self.assertEqual( + config.detect_lang( + bare, None, env={"LANG": "en_US.UTF-8"}, dotfile={}, profile=self.profile + ), + ("zh", "apple-languages"), + ) + + def test_existing_dirs_sniff_zh_en(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + zh_root = Path(temporary) / "zh" + en_root = Path(temporary) / "en" + both_root = Path(temporary) / "both" + for folder in (zh_root, en_root, both_root): + folder.mkdir() + (zh_root / "00_收件箱").mkdir() + (en_root / "00_Inbox").mkdir() + (both_root / "00_收件箱").mkdir() + (both_root / "00_Inbox").mkdir() + + self.assertEqual( + config.detect_lang(zh_root, None, env={}, dotfile={}, profile=self.profile), + ("zh", "existing-dirs"), + ) + self.assertEqual( + config.detect_lang(en_root, None, env={}, dotfile={}, profile=self.profile), + ("en", "existing-dirs"), + ) + with self.assertRaises(config.LanguageAmbiguous): + config.detect_lang(both_root, None, env={}, dotfile={}, profile=self.profile) + + # an explicit language always wins over the ambiguous layout + self.assertEqual( + config.detect_lang(both_root, "zh", env={}, dotfile={}, profile=self.profile), + ("zh", "flag"), + ) + + +class ProfileMergeTests(unittest.TestCase): + def test_dotfile_merge_union_and_override(self) -> None: + base = config.load_builtin_profile("tiered") + merged = config.merge_profile( + base, + { + "lang": "zh", + "large_bytes": 1024, + "aging": {"default_hours": 12}, + "names": {"library.docs": {"zh": "文档"}}, + "rules": {".numbers": {"dest": "library.docs.csv"}}, + "pinned": ["notes.md"], + "sensitive": {"exact_names": ["work-key.txt"], "markers": ["subscription"]}, + "forbidden": {"basenames": ["Sync"]}, + "regenerable": {"basenames": ["vendor"]}, + "guard": {"reference_files": ["$HOME/.config/mytool.conf"], "recent_minutes": 30}, + "name_patterns": [ + {"id": "reports", "match": "contains", "needle": "report", "dest": "20_知识库/报告"} + ], + }, + ) + + # union keeps the built-in entries + self.assertIn("carl-file-organizer-*.json", merged["pinned"]) + self.assertIn("notes.md", merged["pinned"]) + self.assertIn(".env", merged["sensitive"]["exact_names"]) + self.assertIn("work-key.txt", merged["sensitive"]["exact_names"]) + self.assertIn("credential", merged["sensitive"]["markers"]) + self.assertIn("subscription", merged["sensitive"]["markers"]) + self.assertIn(".git", merged["forbidden"]["basenames"]) + self.assertIn("Sync", merged["forbidden"]["basenames"]) + self.assertIn("node_modules", merged["regenerable"]["basenames"]) + self.assertIn("vendor", merged["regenerable"]["basenames"]) + self.assertIn("$HOME", merged["guard"]["reference_scan_roots"]) + self.assertIn("$HOME/.config/mytool.conf", merged["guard"]["reference_files"]) + self.assertEqual([p["id"] for p in merged["name_patterns"]], ["reports"]) + + # deep merge on names / rules + self.assertEqual(merged["names"]["library.docs"]["zh"], "文档") + self.assertEqual(merged["names"]["library.docs"]["en"], "Documents") + self.assertEqual(merged["rules"][".numbers"]["dest"], "library.docs.csv") + self.assertEqual(merged["rules"][".pdf"]["dest"], "library.docs.pdf") + + # plain overrides + self.assertEqual(merged["lang"], "zh") + self.assertEqual(merged["large_bytes"], 1024) + self.assertEqual(merged["aging"]["default_hours"], 12) + self.assertEqual(merged["aging"]["archive_hours"], 168) + self.assertEqual(merged["guard"]["recent_minutes"], 30) + self.assertEqual(merged["guard"]["lsof_timeout_s"], 5) + + # the built-in profile is never mutated + self.assertNotIn("notes.md", base["pinned"]) + + def test_short_ascii_marker_is_refused(self) -> None: + base = config.load_builtin_profile("tiered") + with self.assertRaises(ValueError) as caught: + config.merge_profile(base, {"sensitive": {"markers": ["r2"]}}) + self.assertIn("exact_stems", str(caught.exception)) + # a short CJK marker is fine, it cannot collide the way a two-letter word does + merged = config.merge_profile(base, {"sensitive": {"markers": ["证件"]}}) + self.assertIn("证件", merged["sensitive"]["markers"]) + + def test_save_dotfile_only_if_missing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cfg = config.resolve_config(root, lang="zh", profile="tiered") + written = config.save_dotfile(cfg, only_if_missing=True) + self.assertIsNotNone(written) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8")), + {"profile": "tiered", "lang": "zh"}, + ) + + cfg.lang = "en" + self.assertIsNone(config.save_dotfile(cfg, only_if_missing=True)) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8"))["lang"], "zh" + ) + + self.assertIsNotNone(config.save_dotfile(cfg, only_if_missing=False)) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8"))["lang"], "en" + ) + + def test_dotfile_is_read_back_by_resolve_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + _write_dotfile(root, {"profile": "simple", "lang": "en", "pinned": ["keepme.txt"]}) + cfg = config.resolve_config(root) + self.assertEqual(cfg.profile_name, "simple") + self.assertEqual(cfg.lang, "en") + self.assertEqual(cfg.lang_source, "dotfile") + self.assertEqual(cfg.profile_source, "builtin+dotfile") + self.assertIn("keepme.txt", cfg.profile["pinned"]) + self.assertEqual(len(cfg.profile_hash), 12) + + def test_recorded_language_stays_out_of_the_settings_digest(self) -> None: + base = config.load_builtin_profile("tiered") + zh = config.merge_profile(base, {"profile": "tiered", "lang": "zh"}) + en = config.merge_profile(base, {"profile": "tiered", "lang": "en"}) + self.assertEqual(zh["lang"], "zh") + self.assertEqual(en["lang"], "en") + self.assertEqual(config.profile_hash(zh), config.profile_hash(en)) + self.assertEqual(config.profile_hash(zh), config.profile_hash(base)) + + def test_an_edited_rule_changes_the_settings_digest(self) -> None: + base = config.load_builtin_profile("tiered") + edited = config.merge_profile(base, {"pinned": ["something-new"]}) + self.assertNotEqual(config.profile_hash(base), config.profile_hash(edited)) + + def test_broken_dotfile_reports_position(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / config.DOTFILE_NAME).write_text('{"lang": ', encoding="utf-8") + with self.assertRaises(ValueError) as caught: + config.load_dotfile(root) + self.assertIn("line", str(caught.exception)) + + +class NameResolutionTests(unittest.TestCase): + def test_dir_name_three_levels(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + zh = config.resolve_config(root, lang="zh", profile="tiered") + self.assertEqual(config.dir_name(zh, "library"), "20_知识库") + self.assertEqual(config.dir_name(zh, "library.docs"), "20_知识库/文档资料") + self.assertEqual(config.dir_name(zh, "library.docs.pdf"), "20_知识库/文档资料/PDF") + self.assertEqual( + config.dest_dir(zh, "library.docs.pdf"), root / "20_知识库/文档资料/PDF" + ) + self.assertEqual(config.managed_dir(zh), root / "00_下载目录管理") + + en = config.resolve_config(root, lang="en", profile="tiered") + self.assertEqual(config.dir_name(en, "library.docs.pdf"), "20_Library/Documents/PDF") + self.assertEqual(config.managed_dir(en), root / "00_File_Organizer") + + with self.assertRaises(ValueError): + config.dir_name(zh, "library.docs.pdf.extra") + with self.assertRaises(KeyError): + config.dir_name(zh, "library.nope") + + def test_partition_prefixes(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + cfg = config.resolve_config(Path(temporary), lang="zh", profile="tiered") + self.assertEqual( + config.partition_prefixes(cfg), ("00_", "10_", "20_", "60_", "90_") + ) + names = config.partition_dir_names(cfg) + self.assertIn("00_收件箱", names) + self.assertIn("00_Inbox", names) + self.assertIn("00_下载目录管理", names) + + +class PathGuardTests(unittest.TestCase): + def test_refuse_root(self) -> None: + home = Path.home() + with self.assertRaises(ValueError): + paths.refuse_root(Path("/")) + with self.assertRaises(ValueError): + paths.refuse_root(home) + if (home / "Library").is_dir(): + with self.assertRaises(ValueError): + paths.refuse_root(home / "Library") + + with tempfile.TemporaryDirectory() as temporary: + outside = Path(temporary) + with self.assertRaises(ValueError): + paths.refuse_root(outside) + self.assertEqual( + paths.refuse_root(outside, allow_outside_home=True), + paths.realpath(outside), + ) + + linked = outside / "link" + os.symlink(str(outside), str(linked)) + with self.assertRaises(ValueError): + paths.refuse_root(linked, allow_outside_home=True) + + # a folder inside a fake home is accepted + fake_home = outside / "home" + (fake_home / "Downloads").mkdir(parents=True) + self.assertEqual( + paths.refuse_root(fake_home / "Downloads", home=fake_home), + paths.realpath(fake_home / "Downloads"), + ) + with self.assertRaises(ValueError): + paths.refuse_root(fake_home, home=fake_home) + + def test_portable_round_trip_and_safe_names(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = home / "Downloads" / "report.pdf" + self.assertEqual(paths.portable(target, home=home), "$HOME/Downloads/report.pdf") + self.assertEqual( + paths.expand_portable("$HOME/Downloads/report.pdf", home=home), target + ) + self.assertEqual(paths.safe_dir_name("$CODE_HOME"), "CODE_HOME") + self.assertEqual(paths.safe_dir_name('"'), "quote-dir") + self.assertEqual(paths.safe_dir_name(" "), "unnamed") + self.assertEqual(paths.safe_dir_name("a/b"), "a-b") + + def test_conflict_name_keeps_the_suffix_last(self) -> None: + stamp = datetime(2026, 9, 2, 12, 0, 0) + self.assertEqual( + paths.conflict_name("quarterly-report.pdf", stamp), + "quarterly-report-2026-09-02_12-00-00.pdf", + ) + self.assertEqual( + paths.conflict_name("archive.tar.gz", stamp), + "archive-2026-09-02_12-00-00.tar.gz", + ) + self.assertEqual( + paths.conflict_name("Report.PDF", stamp), "Report-2026-09-02_12-00-00.PDF" + ) + self.assertEqual(paths.conflict_name("README", stamp), "README-2026-09-02_12-00-00") + self.assertEqual( + paths.conflict_name("my.project", stamp, is_dir=True), + "my.project-2026-09-02_12-00-00", + ) + # safe_dir_name eats a leading dot before the stamp is added; hidden + # entries never reach the planner, so this only pins the existing rule. + self.assertEqual(paths.conflict_name(".bashrc", stamp), "bashrc-2026-09-02_12-00-00") + + +class PlanPrivacyTests(unittest.TestCase): + """strip_absolute drops the four absolute fields; hydrate_absolute rebuilds them. + + Both homes below are invented rather than real ``//`` + paths, because this repository refuses to carry one machine's absolute paths + at all: the same rule this feature exists to enforce on report.html. + """ + + HOME = "/opt/gn-test/example" + OTHER = "/opt/gn-test/someone-else" + + def _plan(self) -> dict: + home = self.HOME + return { + "schema_version": 2, + "source_root": home + "/Downloads", + "source_root_portable": "$HOME/Downloads", + "managed_dir": home + "/Downloads/00_File_Organizer", + "managed_dir_portable": "$HOME/Downloads/00_File_Organizer", + "actions": [ + { + "id": "a1", + "kind": "move", + "source": home + "/Downloads/report.pdf", + "source_portable": "$HOME/Downloads/report.pdf", + "destination": home + "/Downloads/PDF/report.pdf", + "destination_portable": "$HOME/Downloads/PDF/report.pdf", + "guard": { + "referenced_in": [ + { + "file": home + "/.zshrc", + "line": 82, + "form": home + "/Downloads/report.pdf", + } + ], + "incoming_symlinks": [home + "/.local/bin/report"], + }, + }, + { + "id": "a2", + "kind": "trash", + "source": home + "/Downloads/old.dmg", + "source_portable": "$HOME/Downloads/old.dmg", + "destination": None, + "destination_portable": None, + }, + ], + } + + def test_strip_removes_every_absolute_path_including_the_reference_form(self) -> None: + stripped = paths.strip_absolute(self._plan()) + text = json.dumps(stripped, ensure_ascii=False) + + for field in ("source_root", "managed_dir"): + self.assertNotIn(field, stripped) + for action in stripped["actions"]: + self.assertNotIn("source", action) + self.assertNotIn("destination", action) + self.assertEqual(stripped["source_root_portable"], "$HOME/Downloads") + + guard = stripped["actions"][0]["guard"] + self.assertEqual(guard["referenced_in"][0]["file"], "$HOME/.zshrc") + self.assertEqual(guard["incoming_symlinks"], ["$HOME/.local/bin/report"]) + # form is no longer exempt: it gets the same $HOME rewrite as everything else + self.assertEqual( + guard["referenced_in"][0]["form"], "$HOME/Downloads/report.pdf" + ) + + self.assertNotIn(self.HOME, text) + + def test_strip_leaves_the_original_alone(self) -> None: + plan = self._plan() + paths.strip_absolute(plan) + self.assertEqual(plan["source_root"], self.HOME + "/Downloads") + self.assertEqual(plan["actions"][0]["source"], self.HOME + "/Downloads/report.pdf") + + def test_hydrate_rebuilds_against_the_running_home(self) -> None: + stripped = paths.strip_absolute(self._plan()) + back = paths.hydrate_absolute(stripped, Path(self.OTHER)) + + self.assertEqual(back["source_root"], self.OTHER + "/Downloads") + self.assertEqual(back["managed_dir"], self.OTHER + "/Downloads/00_File_Organizer") + self.assertEqual(back["actions"][0]["source"], self.OTHER + "/Downloads/report.pdf") + self.assertEqual( + back["actions"][0]["destination"], self.OTHER + "/Downloads/PDF/report.pdf" + ) + self.assertIsNone(back["actions"][1]["destination"]) + + def test_hydrate_is_a_no_op_on_a_full_plan(self) -> None: + plan = self._plan() + self.assertEqual(paths.hydrate_absolute(plan, Path(self.HOME)), plan) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_executor.py b/tests/test_cfo_executor.py new file mode 100644 index 0000000..cb4c2bb --- /dev/null +++ b/tests/test_cfo_executor.py @@ -0,0 +1,949 @@ +"""Execution: what it refuses, what it skips, and what it puts back. + +Every plan in here is built from real temporary files with ``ids.subject_id`` +and ``ids.action_id`` computed on the spot, so the tests exercise the same +identity check the executor uses, without depending on the planner. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import contextlib +import io +import json +import os +import shutil +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest import mock + +from carl_file_organizer import config as config_module +from carl_file_organizer import executor, ids, manifest, paths +from carl_file_organizer.executor import ExecutionRefused, PlanError + +HOME = Path.home() +NOW = datetime(2026, 9, 2, 10, 20, 11).astimezone() + +NAMES = { + "inbox": "00_收件箱", + "inbox.pending": "00_收件箱/待判断", + "inbox.dup": "00_收件箱/重复待确认", + "library": "20_知识库", + "library.docs": "20_知识库/文档资料", + "library.docs.pdf": "20_知识库/文档资料/PDF", + "sensitive": "60_敏感信息", + "sensitive.pending": "60_敏感信息/待转移", + "work": "10_工作区", + "work.tools": "10_工作区/工具与导出", + "work.tools.scripts": "10_工作区/工具与导出/脚本", +} + + +def make_action( + path: Path, + kind: str, + *, + destination=None, + destination_key=None, + tier: str = "routine", + rule: str = "ext:.pdf", + approvable: bool = True, + reroutable: bool = False, + group_id=None, + size_bytes=None, + home: Path = HOME, +): + stat = path.stat() + subject_kind = "dir" if path.is_dir() else "file" + subject = ids.subject_id(path, subject_kind, stat.st_size, stat.st_mtime_ns) + destination_portable = ( + paths.portable(Path(destination), home) if destination is not None else None + ) + recorded_size = stat.st_size if size_bytes is None else size_bytes + return { + "id": ids.action_id(subject, kind, destination_portable), + "subject_id": subject, + "kind": kind, + "subject_kind": subject_kind, + "source": str(path), + "source_portable": paths.portable(path, home), + "filename": path.name, + "destination": str(destination) if destination is not None else None, + "destination_portable": destination_portable, + "destination_key": destination_key, + "size_bytes": recorded_size, + "modified_ns": stat.st_mtime_ns, + "age_hours": 100, + "rule": rule, + "reason": {"zh": "测试用的理由", "en": "reason for the test"}, + "detail": None, + "confidence": 0.98, + "tier": tier, + "group_id": group_id, + "approvable": approvable, + "reroutable": reroutable, + "default_selected": False, + "requires": ["allow-permanent-delete"] if kind == "delete" else [], + "reclaims_bytes": recorded_size if kind in ("trash", "delete") else 0, + "restore_method": None, + "tags": [], + "hints": [], + "hold_reason": None, + "guard": {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []}, + "dir_stats": None, + } + + +def make_plan( + root: Path, + actions, + *, + approved=(), + overrides=(), + groups=(), + lang: str = "zh", + trash_backend: str = "finder", + home: Path = HOME, +): + return { + "schema_version": 2, + "generator": {"name": "carl-file-organizer", "version": "0.3.0"}, + "created_at": NOW.isoformat(), + "now": NOW.isoformat(), + "lang": lang, + "lang_source": "flag", + "profile": {"name": "tiered", "source": "builtin", "dotfile": None, "hash": ""}, + "source_root": str(root), + "source_root_portable": paths.portable(root, home), + "managed_dir": str(root / "00_下载目录管理"), + "managed_dir_portable": paths.portable(root / "00_下载目录管理", home), + "names": dict(NAMES), + "capabilities": { + "trash_backend": trash_backend, + "finder_tags": True, + "permanent_delete_offered": True, + }, + "scan_policy": {"depth": 1}, + "disk": {}, + "summary": {}, + "pinned": [], + "actions": list(actions), + "groups": list(groups), + "approved_action_ids": list(approved), + "overrides": list(overrides), + "approved_at": None, + "approved_by": "agent:test", + } + + +class Fixture: + """A temporary folder plus the config the executor would resolve for it.""" + + def __init__(self, stack: unittest.TestCase) -> None: + temporary = tempfile.TemporaryDirectory() + stack.addCleanup(temporary.cleanup) + self.root = paths.realpath(Path(temporary.name)) + self.config = config_module.resolve_config(self.root, lang="zh", profile="tiered") + self.managed = config_module.managed_dir(self.config) + + def file(self, name: str, content: str = "hello") -> Path: + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + def folder(self, name: str, entries=("a.txt", "b.txt")) -> Path: + target = self.root / name + target.mkdir(parents=True, exist_ok=True) + for entry in entries: + (target / entry).write_text(entry, encoding="utf-8") + return target + + def pdf_dest(self, name: str) -> Path: + return self.root / NAMES["library.docs.pdf"] / name + + def apply(self, plan, **kw): + options = {"config": self.config, "now": NOW, "open_handles_fn": lambda path: []} + options.update(kw) + return executor.apply_approved_plan(plan, **options) + + def audit_records(self): + return manifest.read_audit(manifest.audit_path(self.managed)) + + def manifest_file(self): + found = sorted(self.managed.glob("*.tsv")) + return found[0] if found else None + + +class ValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_only_approved_action_moves(self) -> None: + first = self.fixture.file("first.pdf", "document") + second = self.fixture.file("second.pdf", "image") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"), + destination_key="library.docs.pdf"), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf"), + destination_key="library.docs.pdf"), + ] + plan = make_plan(self.fixture.root, actions, approved=[actions[0]["id"]]) + + report = self.fixture.apply(plan) + + self.assertEqual(len(report.results), 1) + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(self.fixture.pdf_dest("first.pdf").exists()) + self.assertFalse(first.exists()) + self.assertTrue(second.exists()) + + def test_schema_version_one_is_refused(self) -> None: + plan = make_plan(self.fixture.root, []) + plan["schema_version"] = 1 + with self.assertRaises(PlanError): + executor.validate_plan(plan, allow_permanent_delete=False) + + def test_unknown_approval_id_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + actions = [make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"))] + plan = make_plan(self.fixture.root, actions, approved=["invented"]) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_tampered_destination_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + action = make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"), + destination_key="library.docs.pdf") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + action["destination"] = str(self.fixture.root / "20_知识库" / "renamed.pdf") + action["destination_portable"] = paths.portable(Path(action["destination"]), HOME) + + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_destination_outside_the_root_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + action = make_action(first, "move", destination=outside / "first.pdf") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_two_actions_on_the_same_subject_are_rejected(self) -> None: + target = self.fixture.file("Archive.zip") + move = make_action(target, "move", destination=self.fixture.root / NAMES["inbox.pending"] / "Archive.zip", + tier="duplicate", rule="pair") + drop = make_action(target, "trash", tier="duplicate", rule="pair") + plan = make_plan(self.fixture.root, [move, drop], approved=[move["id"], drop["id"]]) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("same item", str(caught.exception)) + self.assertTrue(target.exists()) + + def test_hold_id_is_rejected(self) -> None: + target = self.fixture.file("installer.dmg") + hold = make_action(target, "hold", tier="aging", rule="hold:aging", approvable=False) + plan = make_plan(self.fixture.root, [hold], approved=[hold["id"]]) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("not approvable", str(caught.exception)) + self.assertTrue(target.exists()) + + def test_root_that_is_a_symlink_is_refused(self) -> None: + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + link = self.fixture.root / "link-root" + link.symlink_to(outside, target_is_directory=True) + plan = make_plan(link, []) + with self.assertRaises(PlanError): + executor.validate_plan(plan, allow_permanent_delete=False) + + +class DeleteGateTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def _node_modules(self): + project = self.fixture.folder("web-prototype", entries=()) + modules = project / "node_modules" + modules.mkdir() + (modules / "left-pad.js").write_text("module", encoding="utf-8") + return modules + + def test_delete_requires_the_flag_and_refuses_the_whole_batch(self) -> None: + modules = self._node_modules() + other = self.fixture.file("report.pdf") + move = make_action(other, "move", destination=self.fixture.pdf_dest("report.pdf")) + drop = make_action(modules, "delete", tier="regenerable", rule="regenerable:node_modules") + plan = make_plan(self.fixture.root, [move, drop], approved=[move["id"], drop["id"]]) + + with self.assertRaises(ExecutionRefused): + self.fixture.apply(plan) + + self.assertTrue(modules.exists()) + self.assertTrue(other.exists(), "a refused batch must not move anything either") + self.assertFalse(self.fixture.managed.exists()) + + def test_delete_with_the_flag_removes_regenerable_output(self) -> None: + modules = self._node_modules() + drop = make_action(modules, "delete", tier="regenerable", rule="regenerable:node_modules") + plan = make_plan(self.fixture.root, [drop], approved=[drop["id"]]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual(report.results[0]["status"], "deleted") + self.assertFalse(modules.exists()) + + def test_delete_outside_the_regenerable_list_is_refused(self) -> None: + project = self.fixture.folder("web-prototype", entries=()) + source = project / "src" + source.mkdir() + (source / "index.js").write_text("real work", encoding="utf-8") + drop = make_action(source, "delete", tier="regenerable", rule="regenerable:src") + plan = make_plan(self.fixture.root, [drop], approved=[drop["id"]]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(source.exists()) + self.assertIn("src", report.results[0]["detail"]) + + def test_duplicate_delete_needs_a_surviving_copy(self) -> None: + first = self.fixture.file("guide.pdf", "same") + second = self.fixture.file("guide (1).pdf", "same") + keep = make_action(first, "move", destination=self.fixture.pdf_dest("guide.pdf"), + tier="duplicate", rule="dup:sha256", group_id="dup-1") + drop = make_action(second, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + group = { + "group_id": "dup-1", + "kind": "duplicate", + "members": [keep["subject_id"], drop["subject_id"]], + } + + plan = make_plan(self.fixture.root, [keep, drop], approved=[keep["id"], drop["id"]], + groups=[group]) + report = self.fixture.apply(plan, allow_permanent_delete=True) + self.assertEqual(report.status_of(drop["id"]), "deleted") + self.assertFalse(second.exists()) + self.assertTrue(self.fixture.pdf_dest("guide.pdf").exists()) + + def test_deleting_every_member_of_a_group_is_refused(self) -> None: + first = self.fixture.file("guide.pdf", "same") + second = self.fixture.file("guide (1).pdf", "same") + one = make_action(first, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + two = make_action(second, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + group = {"group_id": "dup-1", "kind": "duplicate", + "members": [one["subject_id"], two["subject_id"]]} + plan = make_plan(self.fixture.root, [one, two], approved=[one["id"], two["id"]], + groups=[group]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual({record["status"] for record in report.results}, {"refused"}) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + +class PreflightTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_changed_source_is_skipped(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + stat = target.stat() + os.utime(str(target), ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "skipped") + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.pdf_dest("report.pdf").exists()) + + def test_vanished_source_is_skipped(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + target.unlink() + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "skipped") + + def test_no_overwrite_at_apply_time(self) -> None: + target = self.fixture.file("report.pdf", "mine") + destination = self.fixture.pdf_dest("report.pdf") + action = make_action(target, "move", destination=destination) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("someone else", encoding="utf-8") + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "conflict") + self.assertEqual(target.read_text(encoding="utf-8"), "mine") + self.assertEqual(destination.read_text(encoding="utf-8"), "someone else") + + def test_symlink_escape_is_refused(self) -> None: + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + real = outside / "secret.pdf" + real.write_text("outside the folder", encoding="utf-8") + link = self.fixture.root / "report.pdf" + link.symlink_to(real) + + action = make_action(link, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(real.exists()) + self.assertTrue(link.is_symlink()) + + def test_forbidden_zone_is_refused(self) -> None: + repository = self.fixture.folder(".git", entries=("HEAD",)) + action = make_action(repository, "move", + destination=self.fixture.root / NAMES["inbox.pending"] / ".git") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(repository.exists()) + + def test_an_open_file_is_skipped(self) -> None: + target = self.fixture.file("session.log") + action = make_action(target, "move", + destination=self.fixture.root / NAMES["inbox.pending"] / "session.log") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply( + plan, open_handles_fn=lambda path: [{"pid": "123", "command": "tail", "name": str(path)}] + ) + self.assertEqual(report.results[0]["status"], "skipped") + self.assertTrue(target.exists()) + + def test_spotlight_reading_a_file_does_not_count_as_in_use(self) -> None: + target = self.fixture.file("report.pdf") + + class Result: + returncode = 0 + stdout = "p501\ncmdworker_shared\nn{0}\n".format(target) + stderr = "" + + with mock.patch.object(executor.shutil, "which", lambda name: "/usr/sbin/lsof"), \ + mock.patch.object(executor.subprocess, "run", lambda *a, **kw: Result()): + self.assertEqual(executor.open_handles(target), []) + + class Real: + returncode = 0 + stdout = "p777\nctail\nn{0}\n".format(target) + stderr = "" + + with mock.patch.object(executor.shutil, "which", lambda name: "/usr/sbin/lsof"), \ + mock.patch.object(executor.subprocess, "run", lambda *a, **kw: Real()): + found = executor.open_handles(target) + self.assertEqual(found, [{"pid": "777", "command": "tail", "name": str(target)}]) + + def test_no_lsof_means_no_objection(self) -> None: + target = self.fixture.file("report.pdf") + with mock.patch.object(executor.shutil, "which", lambda name: None): + self.assertEqual(executor.open_handles(target), []) + + def test_a_folder_keeps_its_name_and_moves_whole(self) -> None: + folder = self.fixture.folder("web-prototype") + destination = self.fixture.root / NAMES["inbox.pending"] / "web-prototype" + action = make_action(folder, "move", destination=destination, rule="dir:pending", + reroutable=True, destination_key="inbox.pending") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue((destination / "a.txt").exists()) + + def test_a_folder_whose_recorded_size_was_walked_still_matches(self) -> None: + folder = self.fixture.folder("web-prototype") + destination = self.fixture.root / NAMES["inbox.pending"] / "web-prototype" + action = make_action(folder, "move", destination=destination, rule="dir:pending") + # A planner may record the walked size rather than the inode size. + walked = sum(item.stat().st_size for item in folder.iterdir()) + action["size_bytes"] = walked + action["subject_id"] = ids.subject_id(folder, "dir", walked, folder.stat().st_mtime_ns) + action["id"] = ids.action_id( + action["subject_id"], "move", paths.portable(destination, HOME) + ) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "moved") + + +class OverrideTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def _pending(self): + target = self.fixture.file("dataset.parquet") + action = make_action( + target, + "move", + destination=self.fixture.root / NAMES["inbox.pending"] / "dataset.parquet", + destination_key="inbox.pending", + rule="unknown-ext", + reroutable=True, + ) + return target, action + + def test_override_reroutes_through_the_names_table(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "work.tools.scripts"}], + ) + + report = self.fixture.apply(plan) + + moved_to = self.fixture.root / NAMES["work.tools.scripts"] / "dataset.parquet" + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(moved_to.exists()) + self.assertFalse(target.exists()) + + def test_override_to_an_unknown_key_is_rejected(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "somewhere.else"}], + ) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(target.exists()) + + def test_override_cannot_carry_a_literal_path(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "../../etc"}], + ) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(target.exists()) + + def test_override_on_a_rule_bound_action_is_rejected(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf", rule="ext:.pdf") + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "inbox.pending"}], + ) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("cannot be rerouted", str(caught.exception)) + self.assertTrue(target.exists()) + + +class VerificationTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_verify_move_checks_size_and_both_ends(self) -> None: + source = self.fixture.file("a.txt", "hello") + destination = self.fixture.file("b.txt", "hello!") + + # A source still sitting there means the move never happened. + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 5}) + self.assertFalse(ok) + self.assertIn("source still there", detail) + + source.unlink() + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 5}) + self.assertFalse(ok) + self.assertIn("size mismatch", detail) + + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 6}) + self.assertTrue(ok) + + ok, detail = executor.verify_move(source, self.fixture.root / "nope.txt", + {"kind": "file", "size": 6}) + self.assertFalse(ok) + + def test_verify_move_counts_folder_entries(self) -> None: + folder = self.fixture.folder("stuff", entries=("a.txt", "b.txt")) + moved = self.fixture.root / "moved" + shutil.move(str(folder), str(moved)) + self.assertEqual( + executor.verify_move(folder, moved, {"kind": "dir", "entries": 2})[0], True + ) + self.assertEqual( + executor.verify_move(folder, moved, {"kind": "dir", "entries": 5})[0], False + ) + + def test_a_move_that_does_not_verify_goes_back_and_stops_the_batch(self) -> None: + first = self.fixture.file("first.pdf", "one") + second = self.fixture.file("second.pdf", "two") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf")), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf")), + ] + plan = make_plan(self.fixture.root, actions, + approved=[actions[0]["id"], actions[1]["id"]]) + + def wrong_shape(path): + return {"kind": "file", "size": 999999} + + with mock.patch.object(executor, "_shape", wrong_shape): + report = self.fixture.apply(plan) + + self.assertEqual(report.results[0]["status"], "failed") + self.assertIn("原路移回", report.results[0]["detail"]) + self.assertEqual(report.results[1]["status"], "skipped") + self.assertTrue(report.stopped) + + self.assertEqual(first.read_text(encoding="utf-8"), "one") + self.assertFalse(self.fixture.pdf_dest("first.pdf").exists()) + self.assertTrue(second.exists()) + self.assertFalse(self.fixture.pdf_dest("second.pdf").exists()) + + def test_continue_on_error_keeps_going(self) -> None: + first = self.fixture.file("first.pdf", "one") + second = self.fixture.file("second.pdf", "two") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf")), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf")), + ] + plan = make_plan(self.fixture.root, actions, + approved=[actions[0]["id"], actions[1]["id"]]) + + with mock.patch.object(executor, "_shape", lambda path: {"kind": "file", "size": 999999}): + report = self.fixture.apply(plan, continue_on_error=True) + + self.assertEqual([record["status"] for record in report.results], ["failed", "failed"]) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + def test_manifest_is_written_before_execution(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + seen = {} + + real_move = shutil.move + + def exploding_move(source, destination, *args, **kw): + found = sorted(self.fixture.managed.glob("*.tsv")) + seen["manifest"] = found[0].read_text(encoding="utf-8") if found else "" + raise OSError("disk went away") + + with mock.patch.object(executor.shutil, "move", exploding_move): + report = self.fixture.apply(plan) + + self.assertIn("PLANNED", seen["manifest"]) + self.assertIn("report.pdf", seen["manifest"]) + self.assertEqual(report.results[0]["status"], "failed") + self.assertTrue(target.exists()) + + records = self.fixture.audit_records() + self.assertEqual(records[-1]["status"], "failed") + self.assertIn("disk went away", records[-1]["detail"]) + self.assertIn("FAILED", self.fixture.manifest_file().read_text(encoding="utf-8")) + self.assertIs(shutil.move, real_move) + + +class PaperTrailTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_dry_run_writes_nothing_at_all(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan, dry_run=True) + + self.assertEqual(report.results[0]["status"], "dry-run") + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.managed.exists()) + self.assertIsNone(report.manifest) + self.assertIsNone(report.audit) + + def test_a_full_run_writes_manifest_audit_and_log(self) -> None: + report_pdf = self.fixture.file("report.pdf", "a document") + secret = self.fixture.file("api-key-backup.txt", "never read") + occupied = self.fixture.file("guide.pdf", "mine") + taken = self.fixture.pdf_dest("guide.pdf") + taken.parent.mkdir(parents=True, exist_ok=True) + taken.write_text("already here", encoding="utf-8") + + actions = [ + make_action(report_pdf, "move", destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf"), + make_action(secret, "move", + destination=self.fixture.root / NAMES["sensitive.pending"] / "api-key-backup.txt", + destination_key="sensitive.pending", tier="sensitive", + rule="sensitive:marker"), + make_action(occupied, "move", destination=taken, + destination_key="library.docs.pdf"), + ] + plan = make_plan(self.fixture.root, actions, + approved=[action["id"] for action in actions]) + + result = self.fixture.apply(plan, recheck_fn=lambda: 0) + + self.assertEqual( + [record["status"] for record in result.results], ["moved", "moved", "conflict"] + ) + + rows = manifest.read_manifest(result.manifest) + self.assertEqual(result.manifest.name, "downloads-moves-2026-09-02_10-20-11.tsv".replace( + "downloads", manifest.slug(self.fixture.root.name))) + self.assertEqual([row["status"] for row in rows], ["MOVED", "MOVED", "CONFLICT"]) + self.assertTrue(rows[0]["restore_method"].startswith('mv "')) + self.assertEqual(rows[0]["kind"], "move") + self.assertEqual(rows[0]["action_id"], actions[0]["id"]) + + records = self.fixture.audit_records() + self.assertEqual(len(records), 3) + self.assertEqual(records[0]["run_id"], "2026-09-02_10-20-11") + self.assertEqual(records[0]["trash_backend"], "finder") + + log = result.review_log.read_text(encoding="utf-8") + self.assertIn("## 2026-09-02", log) + self.assertIn("未读取内容", log) + self.assertIn("未删除任一项", log) + self.assertIn("复查结果：dry-run actions=0。", log) + + def test_audit_paths_are_written_in_home_form(self) -> None: + home = self.fixture.root.parent + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf"), + home=home) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]], home=home) + + report = self.fixture.apply(plan, home=home) + + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(report.results[0]["source"].startswith("$HOME/")) + self.assertTrue(report.results[0]["destination"].startswith("$HOME/")) + self.assertTrue(report.results[0]["manifest"].startswith("$HOME/")) + text = report.manifest.read_text(encoding="utf-8") + self.assertNotIn(str(self.fixture.root), text) + + def test_trash_uses_the_backend_and_never_deletes_on_failure(self) -> None: + target = self.fixture.file("Archive.zip", "zipped") + action = make_action(target, "trash", tier="duplicate", rule="pair") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + def refuse(path, **kw): + raise executor.trash_module.TrashError("Finder said no") + + report = self.fixture.apply(plan, trash_fn=refuse) + self.assertEqual(report.results[0]["status"], "failed") + self.assertIn("Finder said no", report.results[0]["detail"]) + self.assertTrue(target.exists()) + + sent = [] + + def accept(path, **kw): + sent.append((Path(path), kw.get("backend"))) + os.unlink(str(path)) + + plan["approved_action_ids"] = [action["id"]] + report = self.fixture.apply(plan, trash_fn=accept) + self.assertEqual(report.results[0]["status"], "trashed") + self.assertEqual(sent[0][1], "finder") + self.assertFalse(target.exists()) + + def test_empty_approval_list_does_nothing_quietly(self) -> None: + self.fixture.file("report.pdf") + plan = make_plan(self.fixture.root, [], approved=[]) + report = self.fixture.apply(plan) + self.assertEqual(report.results, []) + self.assertFalse(self.fixture.managed.exists()) + + +class CliTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + @staticmethod + def _quiet(argv): + """Run a subcommand without spilling its report into the test output.""" + + from carl_file_organizer import cli + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + code = cli.main(argv) + return code, buffer.getvalue() + + def test_apply_subcommand_keeps_the_approval_file(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + code, output = self._quiet(["apply", str(approval)]) + + self.assertEqual(code, 0) + self.assertIn("moved", output) + self.assertTrue(self.fixture.pdf_dest("report.pdf").exists()) + kept = sorted(self.fixture.managed.glob("approved-*.json")) + self.assertEqual(len(kept), 1) + + def _apply_and_capture(self, plan_hash: str): + """Run ``apply --dry-run`` on a one-move plan and hand back stderr.""" + + from carl_file_organizer import cli + + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + plan["profile"]["hash"] = plan_hash + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = cli.main(["apply", str(approval), "--dry-run"]) + return code, err.getvalue() + + def test_writing_the_settings_file_does_not_look_like_drift(self) -> None: + # The first plan hashes the settings and then writes the settings file, + # which records the language: that is not drift and must not warn. + recorded = self.fixture.config.profile_hash + config_module.save_dotfile(self.fixture.config, only_if_missing=True) + + code, err = self._apply_and_capture(recorded) + + self.assertEqual(code, 0) + self.assertNotIn(executor.text("cli_profile_drift", "zh"), err) + + def test_edited_settings_still_warn_before_applying(self) -> None: + code, err = self._apply_and_capture("0123456789ab") + + self.assertEqual(code, 0) + self.assertIn(executor.text("cli_profile_drift", "zh"), err) + + def test_apply_dry_run_exits_zero_and_writes_nothing(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + code, output = self._quiet(["apply", str(approval), "--dry-run"]) + + self.assertEqual(code, 0) + self.assertIn("dry-run", output) + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.managed.exists()) + + +class PortableOnlyApprovalTests(unittest.TestCase): + """The review page exports $HOME paths only; apply has to cope with that. + + ``home`` is the fixture root itself, so ``portable`` actually produces + ``$HOME/...`` for a temporary folder that does not live under the real home. + """ + + def setUp(self) -> None: + self.fixture = Fixture(self) + self.home = self.fixture.root + + def _approval(self, *, kinds=("move",)): + target = self.fixture.file("report.pdf") + actions = [] + if "move" in kinds: + actions.append( + make_action( + target, + "move", + destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf", + home=self.home, + ) + ) + if "trash" in kinds: + actions.append(make_action(self.fixture.file("old.dmg"), "trash", home=self.home)) + plan = make_plan( + self.fixture.root, + actions, + approved=[action["id"] for action in actions], + home=self.home, + ) + return target, actions, paths.strip_absolute(plan) + + def test_the_exported_shape_really_has_no_absolute_path(self) -> None: + _target, _actions, approval = self._approval(kinds=("move", "trash")) + + self.assertNotIn("source_root", approval) + self.assertNotIn("managed_dir", approval) + self.assertNotIn(str(self.fixture.root), json.dumps(approval, ensure_ascii=False)) + for action in approval["actions"]: + self.assertNotIn("source", action) + self.assertNotIn("destination", action) + + def test_validate_accepts_it_and_rebuilds_the_real_paths(self) -> None: + target, actions, approval = self._approval() + + validated = executor.validate_plan( + approval, allow_permanent_delete=False, home=self.home + ) + + self.assertEqual(validated.root, paths.realpath(self.fixture.root)) + self.assertEqual(validated.home, self.home) + self.assertEqual(Path(validated.actions[0]["source"]), target) + self.assertEqual( + Path(validated.actions[0]["destination"]), self.fixture.pdf_dest("report.pdf") + ) + self.assertEqual(validated.actions[0]["id"], actions[0]["id"]) + + def test_dry_run_passes_preflight_and_touches_nothing(self) -> None: + target, actions, approval = self._approval() + + report = self.fixture.apply(approval, dry_run=True, home=self.home) + + self.assertEqual(report.status_of(actions[0]["id"]), "dry-run") + self.assertEqual(report.counts.get("failed", 0), 0) + self.assertEqual(report.counts.get("refused", 0), 0) + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.pdf_dest("report.pdf").exists()) + + def test_a_real_apply_moves_the_file_and_audits_in_home_form(self) -> None: + target, actions, approval = self._approval() + + report = self.fixture.apply(approval, home=self.home) + + self.assertEqual(report.status_of(actions[0]["id"]), "moved") + self.assertFalse(target.exists()) + self.assertTrue(self.fixture.pdf_dest("report.pdf").exists()) + record = self.fixture.audit_records()[0] + self.assertEqual(record["source"], "$HOME/report.pdf") + self.assertEqual(record["destination"], "$HOME/20_知识库/文档资料/PDF/report.pdf") + + def test_a_tampered_destination_is_still_caught(self) -> None: + _target, _actions, approval = self._approval() + approval["actions"][0]["destination_portable"] = "$HOME/00_收件箱/report.pdf" + + with self.assertRaises(PlanError): + executor.validate_plan(approval, allow_permanent_delete=False, home=self.home) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_fixture_contract.py b/tests/test_cfo_fixture_contract.py new file mode 100644 index 0000000..4aedadd --- /dev/null +++ b/tests/test_cfo_fixture_contract.py @@ -0,0 +1,281 @@ +"""The plan.json schema v2 contract, checked against the hand-written sample. + +Every work package reads and writes this shape, so the sample is the shared +truth: if a change breaks this file, it breaks the review page, the executor +and the agent skill at the same time. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import unittest +from pathlib import Path + +FIXTURE = Path(cfo_path.FIXTURES_DIR) / "plan-v2-sample.json" + +HOLD_TIERS = {"aging", "pinned", "forbidden", "in_use", "referenced"} +REROUTABLE_RULES = {"unknown-ext", "dir:pending", "dir:derivative"} +KINDS = {"move", "trash", "delete", "hold"} +COLORS = {"green", "yellow", "red"} +TIERS = { + "routine", + "sensitive", + "regenerable", + "duplicate", + "cold", + "aging", + "pinned", + "forbidden", + "in_use", + "referenced", +} + + +def _portable_values(node, trail=""): + """Yield every ``*_portable`` string in the document with its path.""" + + if isinstance(node, dict): + for key, value in node.items(): + where = "{0}.{1}".format(trail, key) + if isinstance(value, str) and key.endswith("_portable"): + yield where, value + else: + for item in _portable_values(value, where): + yield item + elif isinstance(node, list): + for index, value in enumerate(node): + for item in _portable_values(value, "{0}[{1}]".format(trail, index)): + yield item + + +class PlanV2ContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.plan = json.loads(FIXTURE.read_text(encoding="utf-8")) + cls.actions = cls.plan["actions"] + cls.by_id = {action["id"]: action for action in cls.actions} + + def test_schema_version_and_top_level_shape(self) -> None: + self.assertEqual(self.plan["schema_version"], 2) + for key in ( + "generator", "created_at", "now", "lang", "lang_source", "profile", + "source_root", "source_root_portable", "managed_dir", "managed_dir_portable", + "names", "capabilities", "scan_policy", "disk", "mess", "summary", + "pinned", "actions", "groups", "notes", "approved_action_ids", + "overrides", "approved_at", "approved_by", + ): + self.assertIn(key, self.plan) + self.assertEqual(self.plan["approved_action_ids"], []) + self.assertEqual(self.plan["overrides"], []) + self.assertIsNone(self.plan["notes"]) + + def test_action_ids_are_unique(self) -> None: + self.assertEqual(len(self.by_id), len(self.actions)) + + def test_every_action_carries_the_frozen_fields(self) -> None: + required = ( + "id", "subject_id", "kind", "subject_kind", "source", "source_portable", + "filename", "destination", "destination_portable", "destination_key", + "size_bytes", "modified_ns", "age_hours", "rule", "reason", "detail", + "confidence", "tier", "group_id", "approvable", "reroutable", + "default_selected", "requires", "reclaims_bytes", "restore_method", + "tags", "hints", "hold_reason", "guard", "dir_stats", "color", + ) + for action in self.actions: + for key in required: + self.assertIn(key, action, action["id"]) + self.assertIn(action["kind"], KINDS) + self.assertIn(action["tier"], TIERS) + self.assertIn(action["color"], COLORS) + self.assertIn(action["subject_kind"], {"file", "dir"}) + self.assertEqual(set(action["reason"]), {"zh", "en"}) + self.assertFalse(action["default_selected"]) + for slot in ("open_by", "referenced_in", "incoming_symlinks", "shape"): + self.assertIn(slot, action["guard"]) + + def test_group_options_map_onto_real_actions(self) -> None: + group_ids = set() + for group in self.plan["groups"]: + group_ids.add(group["group_id"]) + self.assertIn(group["kind"], {"pair", "duplicate", "regenerable"}) + self.assertTrue(group["options"], group["group_id"]) + keys = [option["key"] for option in group["options"]] + self.assertIn(group["default_choice"], keys) + for option in group["options"]: + self.assertTrue(option["action_ids"]) + for action_id in option["action_ids"]: + self.assertIn(action_id, self.by_id, option["key"]) + self.assertEqual(self.by_id[action_id]["group_id"], group["group_id"]) + self.assertTrue(self.by_id[action_id]["approvable"]) + delete_ids = [ + aid for aid in option["action_ids"] if self.by_id[aid]["kind"] == "delete" + ] + if delete_ids: + self.assertIn("allow-permanent-delete", option["requires"]) + member_subjects = set(group["members"]) + for member in member_subjects: + self.assertIn(member, group["member_labels"]) + for action in self.actions: + if action["group_id"] is not None: + self.assertIn(action["group_id"], group_ids) + + def test_group_kinds_present(self) -> None: + kinds = {group["kind"] for group in self.plan["groups"]} + self.assertEqual(kinds, {"pair", "duplicate", "regenerable"}) + pair = next(g for g in self.plan["groups"] if g["kind"] == "pair") + option_keys = {option["key"] for option in pair["options"]} + self.assertIn("keep_all", option_keys) + self.assertEqual( + sum(1 for key in option_keys if key.startswith("keep:")), 3 + ) + duplicate = next(g for g in self.plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(duplicate["evidence"], "sha256") + self.assertIsNotNone(duplicate["sha256"]) + regenerable = next(g for g in self.plan["groups"] if g["kind"] == "regenerable") + self.assertEqual( + {option["key"] for option in regenerable["options"]}, {"trash_all", "delete_all"} + ) + + def test_holds_are_never_approvable(self) -> None: + holds = [action for action in self.actions if action["kind"] == "hold"] + self.assertGreaterEqual(len(holds), 4) + for action in holds: + self.assertFalse(action["approvable"], action["id"]) + self.assertFalse(action["reroutable"], action["id"]) + self.assertIsNone(action["destination"]) + self.assertIsNone(action["destination_portable"]) + self.assertIsNotNone(action["hold_reason"]) + self.assertIn(action["tier"], HOLD_TIERS) + tiers = {action["tier"] for action in holds} + self.assertTrue({"aging", "forbidden", "in_use", "referenced"} <= tiers) + + def test_guard_evidence_on_in_use_and_referenced(self) -> None: + in_use = next(a for a in self.actions if a["tier"] == "in_use") + self.assertTrue(in_use["guard"]["open_by"]) + self.assertIn("pid", in_use["guard"]["open_by"][0]) + referenced = next(a for a in self.actions if a["tier"] == "referenced") + self.assertTrue(referenced["guard"]["referenced_in"]) + first = referenced["guard"]["referenced_in"][0] + self.assertIn("file", first) + self.assertIn("line", first) + + def test_reroutable_actions_stay_in_the_allowed_rule_set(self) -> None: + reroutable = [action for action in self.actions if action["reroutable"]] + self.assertTrue(reroutable) + self.assertTrue(any(action["rule"] == "unknown-ext" for action in reroutable)) + for action in reroutable: + self.assertIn(action["rule"], REROUTABLE_RULES, action["id"]) + self.assertEqual(action["kind"], "move") + self.assertTrue(action["approvable"]) + self.assertIn(action["destination_key"], self.plan["names"]) + + def test_destination_keys_resolve_through_names(self) -> None: + for action in self.actions: + if action["destination_key"] is None: + self.assertIsNone(action["destination"]) + continue + self.assertIn(action["destination_key"], self.plan["names"]) + relative = self.plan["names"][action["destination_key"]] + self.assertIn(relative, action["destination_portable"]) + + def test_delete_actions_require_the_flag(self) -> None: + deletes = [action for action in self.actions if action["kind"] == "delete"] + self.assertTrue(deletes) + for action in deletes: + self.assertIn("allow-permanent-delete", action["requires"]) + self.assertIn(action["tier"], {"regenerable", "duplicate", "cold"}) + self.assertGreater(action["reclaims_bytes"], 0) + + def test_moves_reclaim_nothing_and_carry_a_restore_line(self) -> None: + for action in self.actions: + if action["kind"] == "move": + self.assertEqual(action["reclaims_bytes"], 0) + self.assertIn("mv ", action["restore_method"]["zh"]) + self.assertIn(action["filename"], action["destination_portable"]) + + def test_portable_paths_never_leak_a_user_name(self) -> None: + seen = 0 + for where, value in _portable_values(self.plan): + seen += 1 + self.assertTrue(value.startswith("$HOME/"), "{0}: {1}".format(where, value)) + self.assertGreater(seen, 10) + self.assertEqual(self.plan["source_root_portable"], "$HOME/Downloads") + + def test_summary_matches_the_actions(self) -> None: + summary = self.plan["summary"] + by_kind = {} + by_tier = {} + for action in self.actions: + by_kind[action["kind"]] = by_kind.get(action["kind"], 0) + 1 + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + self.assertEqual(summary["by_kind"], by_kind) + self.assertEqual(summary["by_tier"], by_tier) + self.assertEqual(summary["proposed_reclaimed_bytes"], 0) + self.assertEqual(summary["entries"], summary["files"] + summary["dirs"]) + + def test_every_colour_follows_from_the_decision(self) -> None: + """The buttons follow this field, so it may never be an opinion.""" + + for action in self.actions: + if action["kind"] == "hold": + self.assertEqual(action["color"], "red", action["id"]) + elif action["kind"] == "move": + expected = "yellow" if action["reroutable"] else "green" + self.assertEqual(action["color"], expected, action["id"]) + elif action["tier"] == "regenerable": + self.assertEqual(action["color"], "green", action["id"]) + else: + self.assertEqual(action["color"], "yellow", action["id"]) + colours = {action["color"] for action in self.actions} + self.assertEqual(colours, COLORS) + + def test_no_red_action_can_ever_be_approved(self) -> None: + for action in self.actions: + if action["color"] == "red": + self.assertFalse(action["approvable"], action["id"]) + + def test_every_group_is_a_yellow_question(self) -> None: + self.assertTrue(self.plan["groups"]) + for group in self.plan["groups"]: + self.assertEqual(group["color"], "yellow", group["group_id"]) + + def test_mess_block_describes_the_whole_folder(self) -> None: + mess = self.plan["mess"] + self.assertIn(mess["color"], COLORS) + self.assertEqual(mess["color_source"], "rule") + self.assertTrue(0 <= mess["score"] <= 100) + self.assertEqual( + set(mess["counts"]), {"loose", "aging", "pending", "groups", "top_dirs"} + ) + for key, value in mess["counts"].items(): + self.assertIsInstance(value, int, key) + self.assertGreaterEqual(value, 0, key) + self.assertEqual(set(mess["reason"]), {"zh", "en"}) + self.assertEqual(mess["counts"]["groups"], len(self.plan["groups"])) + self.assertEqual(mess["counts"]["loose"], self.plan["summary"]["entries"]) + self.assertEqual(mess["counts"]["top_dirs"], self.plan["summary"]["dirs"]) + + def test_by_color_counts_every_action_exactly_once(self) -> None: + by_color = self.plan["summary"]["by_color"] + self.assertEqual(set(by_color), COLORS) + self.assertEqual( + sum(slot["actions"] for slot in by_color.values()), len(self.actions) + ) + for colour in COLORS: + wearing = [a for a in self.actions if a["color"] == colour] + self.assertEqual(by_color[colour]["actions"], len(wearing), colour) + self.assertEqual( + by_color[colour]["bytes"], sum(a["size_bytes"] for a in wearing), colour + ) + + def test_sensitive_action_says_contents_were_never_read(self) -> None: + sensitive = next(a for a in self.actions if a["tier"] == "sensitive") + self.assertTrue(sensitive["rule"].startswith("sensitive:")) + self.assertIn("未读取内容", sensitive["reason"]["zh"]) + self.assertIn("60_敏感信息/待转移", sensitive["destination_portable"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_grouping.py b/tests/test_cfo_grouping.py new file mode 100644 index 0000000..4646107 --- /dev/null +++ b/tests/test_cfo_grouping.py @@ -0,0 +1,201 @@ +"""Pairing, duplicates and regenerable build output.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import grouping +from carl_file_organizer.config import resolve_config +from carl_file_organizer.grouping import ( + archive_stem, + copy_base, + dir_stem, + file_sha256, + find_duplicates, + find_regenerable, + pair_archives, +) +from carl_file_organizer.scanner import scan_top_level + +from cfo_helpers import NOW, write # noqa: E402 - the path shim has to run first + + +def scan(root: Path, profile: str = "tiered"): + config = resolve_config(root, profile=profile, lang="zh") + entries, _notes = scan_top_level(root, config, now=NOW, read_finder_tags=False) + return config, entries + + +class NameArithmeticTests(unittest.TestCase): + def test_copy_base(self) -> None: + self.assertEqual(copy_base("foo (1).pdf"), ("foo.pdf", True)) + self.assertEqual(copy_base("foo_1.pdf"), ("foo.pdf", True)) + self.assertEqual(copy_base("Archive 2"), ("Archive", True)) + self.assertEqual(copy_base("foo.pdf"), ("foo.pdf", False)) + + def test_archive_stem_only_answers_for_archives(self) -> None: + self.assertEqual(archive_stem("Archive.zip"), "archive") + self.assertEqual(archive_stem("Archive (1).zip"), "archive") + self.assertEqual(archive_stem("bundle.tar.gz"), "bundle") + self.assertEqual(archive_stem("xxx-v2.zip"), "xxx-v2") + self.assertIsNone(archive_stem("notes.md")) + + def test_dir_stem(self) -> None: + self.assertEqual(dir_stem("Archive (1)"), "archive") + self.assertEqual(dir_stem("Archive 2"), "archive") + self.assertEqual(dir_stem("xxx-v2"), "xxx-v2") + + +class PairTests(unittest.TestCase): + def test_pair_archive_variants(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "z") + write(root / "Archive (1).zip", "z") + (root / "Archive").mkdir() + write(root / "Archive" / "inside.txt") + (root / "Archive 2").mkdir() + write(root / "Archive 2" / "inside.txt") + write(root / "xxx-v2.zip", "z") + (root / "xxx-v2").mkdir() + write(root / "xxx-v2" / "inside.txt") + (root / "xxx").mkdir() + write(root / "xxx" / "inside.txt") + + config, entries = scan(root) + groups = pair_archives(entries, config) + by_stem = {group.stem: group for group in groups} + + self.assertIn("archive", by_stem) + self.assertEqual( + sorted(member.name for member in by_stem["archive"].members), + ["Archive", "Archive (1).zip", "Archive 2", "Archive.zip"], + ) + self.assertIn("xxx-v2", by_stem) + self.assertEqual( + sorted(member.name for member in by_stem["xxx-v2"].members), + ["xxx-v2", "xxx-v2.zip"], + ) + self.assertNotIn("xxx", by_stem) + + def test_a_lone_archive_is_not_a_pair(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "solo.zip", "z") + config, entries = scan(root) + self.assertEqual(pair_archives(entries, config), []) + + +class DuplicateTests(unittest.TestCase): + def test_identical_content_groups_by_sha256(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "identical bytes\n") + write(root / "a (1).pdf", "identical bytes\n") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=True) + + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0].evidence, "sha256") + self.assertIsNotNone(groups[0].sha256) + self.assertEqual([m.name for m in groups[0].members], ["a (1).pdf", "a.pdf"]) + + def test_same_name_shape_but_different_bytes_is_only_a_name_pattern(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "export.png", "AAAA") + write(root / "export_1.png", "BBBB") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=True) + + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0].evidence, "name-pattern") + self.assertIsNone(groups[0].sha256) + + def test_without_hashing_only_names_are_compared(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "identical bytes\n") + write(root / "a (1).pdf", "identical bytes\n") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=False) + self.assertEqual([group.evidence for group in groups], ["name-pattern"]) + + def test_excluded_subjects_never_form_a_duplicate_group(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "same\n") + write(root / "a (1).pdf", "same\n") + _config, entries = scan(root) + everything = [entry.subject_id for entry in entries] + self.assertEqual( + find_duplicates(entries, hash_duplicates=True, exclude=everything), [] + ) + + def test_file_sha256_gives_up_above_the_cap(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "big.bin", "0123456789") + self.assertIsNone(file_sha256(target, max_bytes=4)) + self.assertIsNotNone(file_sha256(target, max_bytes=4096)) + + +class RegenerableTests(unittest.TestCase): + def test_scan_respects_depth_and_never_enters_git(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "package.json", "{}") + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + write(root / "proj" / ".git" / "objects" / "node_modules" / "keep", "x") + write(root / "proj" / "a" / "b" / "c" / "node_modules" / "deep.js", "x") + write(root / "proj" / "a" / "b" / "node_modules" / "ok.js", "x") + + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + relatives = sorted(hit.relative for hit in hits) + + self.assertIn("proj/node_modules", relatives) + self.assertIn("proj/a/b/node_modules", relatives) + self.assertNotIn("proj/a/b/c/node_modules", relatives) + self.assertFalse([r for r in relatives if ".git" in r]) + + def test_hit_carries_a_bounded_size_and_a_subject_id(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "a.js", "0123456789") + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0].dir_stats.files, 1) + self.assertEqual(hits[0].dir_stats.bytes, 10) + self.assertFalse(hits[0].dir_stats.truncated) + self.assertEqual(len(hits[0].subject_id), 16) + + def test_a_matched_folder_is_not_descended_into(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "pkg" / "node_modules" / "x.js", "x") + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + self.assertEqual([hit.relative for hit in hits], ["proj/node_modules"]) + + def test_forbidden_folders_are_left_alone(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "vault" / ".obsidian" / "config", "{}") + write(root / "vault" / "node_modules" / "x.js", "x") + config, entries = scan(root) + self.assertEqual(find_regenerable(entries, config, max_depth=3, now=NOW), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_guard.py b/tests/test_cfo_guard.py new file mode 100644 index 0000000..e7f7d63 --- /dev/null +++ b/tests/test_cfo_guard.py @@ -0,0 +1,273 @@ +"""The safety net, with every outside dependency injected. + +Nothing in this file may consult the real machine: no lsof, no launch agents, +no crontab, no home directory scan. Every check gets its inputs handed to it. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import guard +from carl_file_organizer.config import resolve_config +from carl_file_organizer.guard import ( + config_references, + dir_shape, + file_shape, + incoming_symlinks, + open_handles, + path_forms, + recently_modified, +) +from carl_file_organizer.scanner import make_entry + +from cfo_helpers import NOW, age, fake_lsof, write # noqa: E402 - the path shim runs first + + +def tiered(root: Path): + return resolve_config(root, profile="tiered", lang="zh") + + +class OpenHandleTests(unittest.TestCase): + def test_a_held_file_reports_the_process(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "session.log", "line\n") + with open(str(target), "r", encoding="utf-8"): + result = open_handles( + target, 5.0, lsof_runner=fake_lsof({"session.log": "python3,48213"}) + ) + self.assertTrue(result) + self.assertEqual(result.status, "ok") + self.assertEqual(result.handles[0]["command"], "python3") + self.assertEqual(result.handles[0]["pid"], 48213) + + def test_an_untouched_file_reports_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "quiet.txt") + result = open_handles(target, 5.0, lsof_runner=fake_lsof({})) + self.assertFalse(result) + self.assertEqual(result.status, "ok") + + def test_a_timeout_is_unknown_rather_than_a_crash(self) -> None: + import subprocess + + def runner(argv, timeout_s): + raise subprocess.TimeoutExpired(argv, timeout_s) + + result = open_handles(Path("/nonexistent"), 0.1, lsof_runner=runner) + self.assertEqual(result.status, "unknown") + self.assertFalse(result.handles) + + def test_a_broken_runner_is_unknown_too(self) -> None: + def runner(argv, timeout_s): + raise OSError("no lsof here") + + self.assertEqual(open_handles(Path("/tmp"), 0.1, lsof_runner=runner).status, "unknown") + + def test_directories_are_asked_with_plus_d(self) -> None: + seen = {} + + class _Result(object): + stdout = "" + returncode = 1 + + def runner(argv, timeout_s): + seen["argv"] = list(argv) + return _Result() + + with tempfile.TemporaryDirectory() as temporary: + open_handles(Path(temporary), 1.0, lsof_runner=runner, is_dir=True) + self.assertIn("+d", seen["argv"]) + + def test_spotlight_and_friends_are_not_counted_as_users(self) -> None: + # A file that just landed in Downloads is open in mdworker_shared within + # seconds. Counting that would hold back every fresh download. + self.assertTrue(guard.is_observer("mdworker_shared")) + self.assertTrue(guard.is_observer("quicklookd")) + self.assertFalse(guard.is_observer("tail")) + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "photo.png") + result = open_handles( + target, 5.0, lsof_runner=fake_lsof({"photo.png": "mdworker_shared,900"}) + ) + self.assertFalse(result.handles) + self.assertEqual(result.status, "ok") + + def test_batched_probe_attributes_each_hit_to_its_own_path(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + busy = write(root / "busy.log") + quiet = write(root / "quiet.log") + table = guard.probe_open_handles( + [busy, quiet], [], 1.0, lsof_runner=fake_lsof({"busy.log": "tail,777"}) + ) + self.assertTrue(table[str(busy)].handles) + self.assertFalse(table[str(quiet)].handles) + + +class ReferenceTests(unittest.TestCase): + def test_the_three_spellings_are_all_generated(self) -> None: + home = Path("/home/example") + forms = path_forms(home / "Downloads" / "tool.sh", home) + self.assertEqual( + forms, + [ + "/home/example/Downloads/tool.sh", + "~/Downloads/tool.sh", + "$HOME/Downloads/tool.sh", + ], + ) + + def test_a_launch_agent_reference_reports_the_line_number(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh", "#!/bin/sh\n") + plist = write( + home / "fake-LaunchAgents" / "com.example.backup.plist", + "\nhello\n" + "$HOME/Downloads/backup-tool.sh\n\n", + ) + config = tiered(home / "Downloads") + + found = config_references(target, config, home, reference_files=[plist]) + + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["line"], 3) + self.assertEqual(found[0]["form"], "$HOME/Downloads/backup-tool.sh") + self.assertTrue(found[0]["file"].startswith("$HOME/")) + + def test_a_tilde_spelling_in_a_shell_rc_also_counts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh") + rc = write(home / "fake-zshrc", "\n\nalias b='~/Downloads/backup-tool.sh'\n") + config = tiered(home / "Downloads") + + found = config_references(target, config, home, reference_files=[rc]) + + self.assertEqual(found[0]["line"], 3) + self.assertEqual(found[0]["form"], "~/Downloads/backup-tool.sh") + + def test_an_unrelated_file_matches_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + rc = write(home / "fake-zshrc", "export PATH=$HOME/bin:$PATH\n") + config = tiered(home / "Downloads") + self.assertEqual(config_references(target, config, home, reference_files=[rc]), []) + + def test_a_missing_reference_file_is_skipped_quietly(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + config = tiered(home / "Downloads") + self.assertEqual( + config_references( + target, config, home, reference_files=[home / "nowhere.plist"] + ), + [], + ) + + +class SymlinkTests(unittest.TestCase): + def test_a_symlink_in_a_scan_root_points_back_at_the_entry(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh") + binaries = home / "bin" + binaries.mkdir(parents=True) + (binaries / "backup-tool").symlink_to(target) + config = tiered(home / "Downloads") + + found = incoming_symlinks(target, config, home, scan_roots=[binaries]) + + self.assertEqual(found, ["$HOME/bin/backup-tool"]) + + def test_a_symlink_into_a_folder_counts_for_the_folder(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + inside = write(home / "Downloads" / "proj" / "run.sh") + binaries = home / "bin" + binaries.mkdir(parents=True) + (binaries / "run").symlink_to(inside) + config = tiered(home / "Downloads") + + found = incoming_symlinks( + home / "Downloads" / "proj", config, home, scan_roots=[binaries] + ) + self.assertEqual(found, ["$HOME/bin/run"]) + + def test_no_scan_roots_means_no_findings(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + config = tiered(home / "Downloads") + self.assertEqual(incoming_symlinks(target, config, home, scan_roots=[]), []) + + +class ShapeTests(unittest.TestCase): + def test_a_git_file_is_a_worktree_and_a_git_folder_is_a_repo(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + worktree = root / "topic-worktree" + write(worktree / ".git", "gitdir: /elsewhere/.git/worktrees/topic\n") + repo = root / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + + self.assertIn("git-worktree", dir_shape(worktree).labels) + self.assertIn("git-dir", dir_shape(repo, count_uncommitted=False).labels) + + def test_uncommitted_count_comes_from_the_injected_runner(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + + class _Result(object): + returncode = 0 + stdout = " M a.py\n?? b.py\n" + + shape = dir_shape(repo, git_status_runner=lambda path, timeout: _Result()) + self.assertEqual(shape.uncommitted, 2) + + def test_virtualenv_dotenv_and_node_project_are_flagged(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + write(project / "package.json", "{}") + write(project / ".env", "A=1\n") + (project / ".venv").mkdir(parents=True) + + labels = dir_shape(project).labels + self.assertIn("venv", labels) + self.assertIn("dotenv", labels) + self.assertIn("node-project", labels) + + def test_file_shape_sees_the_shebang_and_the_bit(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + script = write(Path(temporary) / "run.sh", "#!/bin/sh\necho hi\n") + os.chmod(str(script), 0o755) + labels = file_shape(script) + self.assertIn("executable", labels) + self.assertIn("shebang", labels) + + +class RecentTests(unittest.TestCase): + def test_ten_minute_window(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + fresh = age(write(Path(temporary) / "fresh.md"), hours=0.05) + settled = age(write(Path(temporary) / "settled.md"), hours=5) + fresh_entry = make_entry(fresh, now=NOW, read_finder_tags=False) + settled_entry = make_entry(settled, now=NOW, read_finder_tags=False) + + self.assertTrue(recently_modified(fresh_entry, NOW, 10)) + self.assertFalse(recently_modified(settled_entry, NOW, 10)) + self.assertFalse(recently_modified(fresh_entry, NOW, 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_manifest.py b/tests/test_cfo_manifest.py new file mode 100644 index 0000000..5dcb1bb --- /dev/null +++ b/tests/test_cfo_manifest.py @@ -0,0 +1,234 @@ +"""Manifests, tag lists, the audit log and the human review log.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import tempfile +import unittest +from datetime import date, datetime +from pathlib import Path + +from carl_file_organizer import manifest + + +#: The exact header the reference script has been writing since August. +LEGACY_HEADER = ( + "moved_at", + "status", + "original_path", + "target_path", + "size_bytes", + "reason", + "restore_method", +) + +STAMP = datetime(2026, 9, 2, 10, 20, 11) + + +def _row(**kw): + row = { + "moved_at": "2026-09-02T10:20:11+09:00", + "status": "MOVED", + "original_path": "$HOME/Downloads/report.pdf", + "target_path": "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf", + "size_bytes": 2411008, + "reason": "文档按类型归档", + "restore_method": 'mv "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf" "$HOME/Downloads/"', + "kind": "move", + "action_id": "9c1f0a7b2d3e4f55", + } + row.update(kw) + return row + + +class ManifestColumnTests(unittest.TestCase): + def test_manifest_columns_prefix_compatible(self) -> None: + self.assertEqual(manifest.MANIFEST_COLUMNS[:7], LEGACY_HEADER) + self.assertEqual(manifest.MANIFEST_COLUMNS[7:], ("kind", "action_id")) + + def test_manifest_path_names_match_the_reference_layout(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + path = manifest.manifest_path(managed, "Downloads", "moves", STAMP) + self.assertEqual(path.name, "downloads-moves-2026-09-02_10-20-11.tsv") + tags = manifest.tag_list_path(managed, "Downloads", STAMP) + self.assertEqual(tags.name, "downloads-move-tags-2026-09-02_10-20-11.txt") + self.assertEqual(manifest.audit_path(managed).name, "audit.jsonl") + + def test_unknown_series_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(ValueError): + manifest.manifest_path(Path(temporary), "Downloads", "whatever", STAMP) + + def test_write_and_read_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "downloads-moves-2026-09-02_10-20-11.tsv" + manifest.write_manifest(path, [_row(), _row(status="PLANNED")]) + text = path.read_text(encoding="utf-8") + self.assertTrue(text.startswith("\t".join(manifest.MANIFEST_COLUMNS))) + rows = manifest.read_manifest(path) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]["kind"], "move") + self.assertEqual(rows[1]["status"], "PLANNED") + self.assertFalse(list(Path(temporary).glob("*.tmp"))) + + def test_tabs_and_newlines_cannot_break_a_row(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.manifest_path(Path(temporary), "Downloads", "moves", STAMP) + manifest.write_manifest(path, [_row(reason="a\tb\nc")]) + rows = manifest.read_manifest(path) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["reason"], "a b c") + + def test_seven_column_manifest_still_reads(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "downloads-moves-2026-08-30_15-06-43.tsv" + path.write_text( + "\t".join(LEGACY_HEADER) + + "\n" + + "\t".join( + [ + "2026-08-30T15:06:43+09:00", + "MOVED", + "$HOME/Downloads/api-credential.txt", + "$HOME/Downloads/60_敏感信息/待转移/api-credential.txt", + "98", + "敏感命名文件直接移入待转移，不读取内容", + 'mv "$HOME/Downloads/60_敏感信息/待转移/api-credential.txt" "$HOME/Downloads/"', + ] + ) + + "\n", + encoding="utf-8", + ) + rows = manifest.read_manifest(path) + self.assertEqual(rows[0]["status"], "MOVED") + self.assertEqual(rows[0].get("kind", ""), "") + + +class TagListTests(unittest.TestCase): + def test_tag_list_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.tag_list_path(Path(temporary), "Downloads", STAMP) + manifest.write_tag_list(path, ["$HOME/Downloads/a.txt", "$HOME/Downloads/b.txt"]) + self.assertEqual( + manifest.read_tag_list(path), + ["$HOME/Downloads/a.txt", "$HOME/Downloads/b.txt"], + ) + + def test_missing_tag_list_reads_as_empty(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + self.assertEqual(manifest.read_tag_list(Path(temporary) / "nope.txt"), []) + + +class AuditTests(unittest.TestCase): + def test_append_audit_is_one_json_object_per_line(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.audit_path(Path(temporary)) + manifest.append_audit([{"action_id": "a", "status": "moved"}], path) + manifest.append_audit([{"action_id": "b", "status": "skipped"}], path) + lines = path.read_text(encoding="utf-8").strip().split("\n") + self.assertEqual(len(lines), 2) + self.assertEqual(json.loads(lines[1])["action_id"], "b") + records = manifest.read_audit(path) + self.assertEqual([item["status"] for item in records], ["moved", "skipped"]) + + def test_a_broken_line_does_not_stop_the_read(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.audit_path(Path(temporary)) + path.write_text('{"status": "moved"}\nnot json\n\n', encoding="utf-8") + self.assertEqual(len(manifest.read_audit(path)), 1) + + +class ReviewLogTests(unittest.TestCase): + def test_review_log_format(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + first = manifest.append_review_log( + managed, "zh", date(2026, 9, 2), ["移动了 3 项。"] + ) + self.assertEqual(first.name, "人工调整日志.md") + text = first.read_text(encoding="utf-8") + self.assertTrue(text.startswith("# 人工调整日志")) + self.assertIn("## 2026-09-02", text) + self.assertIn("- 移动了 3 项。", text) + + manifest.append_review_log( + managed, "zh", date(2026, 9, 2), ["复查结果：dry-run actions=0。"] + ) + text = first.read_text(encoding="utf-8") + self.assertEqual(text.count("## 2026-09-02"), 1) + self.assertIn("- 复查结果：dry-run actions=0。", text) + self.assertLess( + text.index("- 移动了 3 项。"), text.index("- 复查结果：dry-run actions=0。") + ) + + manifest.append_review_log(managed, "zh", date(2026, 9, 3), ["第二天。"]) + text = first.read_text(encoding="utf-8") + self.assertIn("## 2026-09-03", text) + self.assertLess(text.index("## 2026-09-02"), text.index("## 2026-09-03")) + # A later day must not swallow the earlier day's bullets. + self.assertLess(text.index("- 复查结果"), text.index("## 2026-09-03")) + + def test_english_log_has_its_own_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.append_review_log( + Path(temporary), "en", date(2026, 9, 2), ["Moved 3 items."] + ) + self.assertEqual(path.name, "review-log.md") + self.assertIn("# Review log", path.read_text(encoding="utf-8")) + + def test_no_bullets_writes_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.append_review_log(Path(temporary), "zh", date(2026, 9, 2), []) + self.assertFalse(path.exists()) + + +class LatestRunTests(unittest.TestCase): + def _manifest(self, managed: Path, series: str, stamp: str, rows=1, status="MOVED"): + path = managed / "downloads-{0}-{1}.tsv".format(series, stamp) + manifest.write_manifest( + path, + [ + _row(status=status, moved_at="{0}+09:00".format(stamp.replace("_", "T"))) + for _ in range(rows) + ], + ) + return path + + def test_status_reads_only_manifests(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) / "00_下载目录管理" + managed.mkdir() + # A very fresh folder with no manifest means nobody ever tidied it. + (managed / "plan.json").write_text("{}", encoding="utf-8") + (managed / "notes.md").write_text("hello", encoding="utf-8") + self.assertIsNone(manifest.latest_run(managed)) + + self._manifest(managed, "moves", "2026-08-30_15-06-43", rows=2) + self._manifest(managed, "moves", "2026-09-01_03-11-13", rows=3) + self._manifest(managed, "undo", "2026-09-02_04-00-00", rows=9) + + latest = manifest.latest_run(managed) + self.assertIsNotNone(latest) + self.assertEqual(latest.path.name, "downloads-moves-2026-09-01_03-11-13.tsv") + self.assertEqual(latest.series, "moves") + self.assertEqual(latest.rows, 3) + self.assertEqual(latest.done, 3) + + def test_conflict_moves_is_its_own_series(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + self._manifest(managed, "conflict-moves", "2026-09-01_03-12-04") + latest = manifest.latest_run(managed) + self.assertEqual(latest.series, "conflict-moves") + + def test_missing_folder_is_not_an_error(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + self.assertIsNone(manifest.latest_run(Path(temporary) / "nope")) + self.assertEqual(manifest.list_runs(Path(temporary) / "nope"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_notes.py b/tests/test_cfo_notes.py new file mode 100644 index 0000000..e45e1ad --- /dev/null +++ b/tests/test_cfo_notes.py @@ -0,0 +1,287 @@ +"""The agent explanation layer, and the promise that it decides nothing. + +Every test here is really the same test asked a different way: a notes file is +prose, and prose cannot approve, move, delete, reroute or recolour anything. +The one thing it may change is the folder's own colour, and even then the rule's +answer stays in the file next to it. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import notes as notes_module +from carl_file_organizer.cli import main +from carl_file_organizer.config import resolve_config +from carl_file_organizer.notes import NotesError, merge_notes, validate_notes +from carl_file_organizer.planner import build_plan + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + age, + write, +) + + +def sample_plan(root: Path): + config = resolve_config(root, profile="tiered", lang="zh") + return build_plan( + root, + config=config, + now=NOW, + home=root, + guard_options=GUARD_OFFLINE, + trash_backend="finder", + ) + + +def populated(root: Path) -> None: + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "bundle.zip", "PK\x03\x04"), hours=100) + age(write(root / "bundle" / "inner.txt"), hours=100) + age(root / "bundle", hours=100) + + +class MergeTests(unittest.TestCase): + def setUp(self) -> None: + self._temporary = tempfile.TemporaryDirectory() + self.root = Path(self._temporary.name) + populated(self.root) + self.plan = sample_plan(self.root) + self.action_id = self.plan["actions"][0]["id"] + + def tearDown(self) -> None: + self._temporary.cleanup() + + def test_notes_land_in_their_own_block(self) -> None: + merged = merge_notes( + self.plan, + { + "folder_line": "这个目录基本是上周的下载", + "actions": { + self.action_id: { + "what": "上周的会议纪要", + "why": "扩展名规则命中", + "if_removed": "在备份里还有一份", + } + }, + }, + ) + block = merged["notes"] + self.assertEqual(block["schema"], notes_module.NOTES_SCHEMA) + self.assertEqual(block["folder_line"], "这个目录基本是上周的下载") + self.assertEqual(block["actions"][self.action_id]["what"], "上周的会议纪要") + self.assertEqual(block["warnings"], []) + + def test_the_original_plan_is_untouched(self) -> None: + before = copy.deepcopy(self.plan) + merge_notes(self.plan, {"folder_line": "一句话"}) + self.assertEqual(self.plan, before) + + def test_no_action_field_is_ever_changed(self) -> None: + before = copy.deepcopy(self.plan["actions"]) + merged = merge_notes( + self.plan, + { + "actions": { + self.action_id: { + "what": "x", + "why": "y", + "if_removed": "z", + "color": "green", + "approvable": True, + "destination": "/somewhere/else", + "kind": "delete", + } + } + }, + ) + self.assertEqual(merged["actions"], before) + self.assertEqual( + set(merged["notes"]["actions"][self.action_id]), {"what", "why", "if_removed"} + ) + self.assertTrue( + any("unknown field" in line for line in merged["notes"]["warnings"]) + ) + + def test_an_unknown_id_only_warns(self) -> None: + merged = merge_notes( + self.plan, {"actions": {"0000000000000000": {"what": "谁知道呢"}}} + ) + self.assertEqual(merged["notes"]["actions"], {}) + self.assertTrue( + any("no such id" in line for line in merged["notes"]["warnings"]) + ) + + def test_a_group_note_needs_a_real_group(self) -> None: + group_id = self.plan["groups"][0]["group_id"] + merged = merge_notes( + self.plan, + {"groups": {group_id: {"what": "压缩包和它解出来的目录"}, "nope": {"what": "x"}}}, + ) + self.assertIn(group_id, merged["notes"]["groups"]) + self.assertNotIn("nope", merged["notes"]["groups"]) + + def test_the_folder_colour_can_be_overridden_and_says_so(self) -> None: + rule_colour = self.plan["mess"]["color"] + merged = merge_notes( + self.plan, {"mess": {"color": "red", "line": "我看过了，这里比数字显示的更乱"}} + ) + self.assertEqual(merged["mess"]["color"], "red") + self.assertEqual(merged["mess"]["color_source"], "notes") + self.assertEqual(merged["mess"]["rule_color"], rule_colour) + self.assertEqual(merged["mess"]["line"], "我看过了，这里比数字显示的更乱") + + def test_without_an_override_the_rule_keeps_the_credit(self) -> None: + merged = merge_notes(self.plan, {"folder_line": "一句话"}) + self.assertEqual(merged["mess"]["color_source"], "rule") + self.assertEqual(merged["mess"]["color"], merged["mess"]["rule_color"]) + + def test_an_invented_colour_is_refused_not_stored(self) -> None: + merged = merge_notes(self.plan, {"mess": {"color": "purple"}}) + self.assertEqual(merged["mess"]["color"], self.plan["mess"]["color"]) + self.assertTrue(any("mess.color" in line for line in merged["notes"]["warnings"])) + + def test_merging_twice_replaces_rather_than_piles_up(self) -> None: + once = merge_notes(self.plan, {"folder_line": "第一版"}) + twice = merge_notes(once, {"folder_line": "第二版"}) + self.assertEqual(twice["notes"]["folder_line"], "第二版") + self.assertEqual(twice["mess"]["rule_color"], self.plan["mess"]["color"]) + + def test_empty_and_non_string_sentences_are_dropped(self) -> None: + merged = merge_notes( + self.plan, + {"actions": {self.action_id: {"what": " ", "why": 7, "if_removed": "真的"}}}, + ) + self.assertEqual( + merged["notes"]["actions"][self.action_id], {"if_removed": "真的"} + ) + + def test_a_very_long_sentence_is_trimmed_not_discarded(self) -> None: + merged = merge_notes( + self.plan, {"actions": {self.action_id: {"what": "字" * 5000}}} + ) + self.assertEqual( + len(merged["notes"]["actions"][self.action_id]["what"]), notes_module.MAX_TEXT + ) + + def test_an_unknown_top_level_field_warns(self) -> None: + _cleaned, warnings = validate_notes(self.plan, {"nonsense": 1}) + self.assertTrue(any("unknown top level" in line for line in warnings)) + + def test_a_notes_file_that_is_not_an_object_is_refused(self) -> None: + with self.assertRaises(NotesError): + merge_notes(self.plan, ["not", "an", "object"]) + + +class LoadTests(unittest.TestCase): + def test_broken_json_names_the_file(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "notes.json" + path.write_text("{not json", encoding="utf-8") + with self.assertRaises(NotesError) as caught: + notes_module.load_notes(path) + self.assertIn("notes.json", str(caught.exception)) + + def test_a_missing_file_is_a_notes_error_not_an_oserror(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(NotesError): + notes_module.load_notes(Path(temporary) / "absent.json") + + def test_a_json_list_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "notes.json" + path.write_text("[]", encoding="utf-8") + with self.assertRaises(NotesError): + notes_module.load_notes(path) + + +class BuildCommandTests(unittest.TestCase): + def test_build_writes_the_notes_into_the_plan_on_disk(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text( + json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8" + ) + notes_path = root / "notes.json" + notes_path.write_text( + json.dumps( + { + "folder_line": "上周的下载都在这儿", + "mess": {"color": "yellow", "line": "看着乱，其实好收拾"}, + "actions": {plan["actions"][0]["id"]: {"what": "会议纪要"}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + code = main(["build", str(plan_path), "--notes", str(notes_path)]) + self.assertEqual(code, 0) + + written = json.loads(plan_path.read_text(encoding="utf-8")) + self.assertEqual(written["notes"]["folder_line"], "上周的下载都在这儿") + self.assertEqual(written["mess"]["color"], "yellow") + self.assertEqual(written["mess"]["color_source"], "notes") + + def test_build_can_write_somewhere_else_and_leave_the_plan_alone(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + notes_path = root / "notes.json" + notes_path.write_text('{"folder_line": "一句话"}', encoding="utf-8") + out_path = root / "out" / "annotated.json" + + self.assertEqual( + main( + [ + "build", + str(plan_path), + "--notes", + str(notes_path), + "--output", + str(out_path), + ] + ), + 0, + ) + self.assertIsNone(json.loads(plan_path.read_text(encoding="utf-8"))["notes"]) + self.assertEqual( + json.loads(out_path.read_text(encoding="utf-8"))["notes"]["folder_line"], + "一句话", + ) + + def test_build_without_notes_is_a_no_op_rewrite(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + self.assertEqual(main(["build", str(plan_path)]), 0) + self.assertIsNone(json.loads(plan_path.read_text(encoding="utf-8"))["notes"]) + + def test_build_on_a_broken_plan_exits_two_without_a_traceback(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = Path(temporary) / "plan.json" + plan_path.write_text("not json at all", encoding="utf-8") + self.assertEqual(main(["build", str(plan_path)]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_opener.py b/tests/test_cfo_opener.py new file mode 100644 index 0000000..a5cb735 --- /dev/null +++ b/tests/test_cfo_opener.py @@ -0,0 +1,108 @@ +"""Showing a path to a person, on three platforms, from one of them. + +The command shape is asserted rather than the effect, because the effect is a +window opening on somebody's screen. What matters and what can be checked here +is that no shell is involved and that the path arrives as one argument. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from carl_file_organizer import opener + + +class Launcher(object): + """Stands in for subprocess.Popen and records the call.""" + + def __init__(self, error: bool = False) -> None: + self.calls = [] + self.kwargs = [] + self.error = error + + def __call__(self, argv, **kwargs): + self.calls.append(list(argv)) + self.kwargs.append(dict(kwargs)) + if self.error: + raise OSError("no such file manager") + return object() + + +class CommandShapeTests(unittest.TestCase): + def test_macos_selects_the_item_itself(self) -> None: + argv = opener.reveal_command(Path("/tmp/report.pdf"), platform="darwin") + self.assertEqual(argv[:2], ["open", "-R"]) + self.assertEqual(argv[2], os.path.abspath("/tmp/report.pdf")) + + def test_windows_passes_one_argument_with_the_comma_attached(self) -> None: + argv = opener.reveal_command(Path("C:/Users/someone/report.pdf"), platform="win32") + self.assertEqual(argv[0], "explorer") + self.assertEqual(len(argv), 2) + self.assertTrue(argv[1].startswith("/select,")) + self.assertNotIn(" ", argv[1][:8]) + self.assertIn("report.pdf", argv[1]) + + def test_linux_opens_the_parent_because_xdg_open_cannot_select(self) -> None: + with mock.patch.object(opener.shutil, "which", return_value="/usr/bin/xdg-open"): + argv = opener.reveal_command(Path("/tmp/deep/report.pdf"), platform="linux") + self.assertEqual(argv, ["/usr/bin/xdg-open", os.path.abspath("/tmp/deep")]) + + def test_linux_without_xdg_open_has_no_command_at_all(self) -> None: + with mock.patch.object(opener.shutil, "which", return_value=None): + self.assertIsNone(opener.reveal_command(Path("/tmp/x.pdf"), platform="linux")) + self.assertFalse(opener.supported(platform="linux")) + + def test_a_hostile_filename_stays_one_argument(self) -> None: + name = "$(rm -rf ~) & echo pwned.pdf" + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / name + for platform in ("darwin", "win32"): + argv = opener.reveal_command(target, platform=platform) + self.assertTrue(any(name in part for part in argv), platform) + self.assertEqual(len(argv), 3 if platform == "darwin" else 2, platform) + + +class RevealTests(unittest.TestCase): + def test_reveal_never_goes_through_a_shell(self) -> None: + launcher = Launcher() + self.assertTrue( + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + ) + self.assertIsInstance(launcher.calls[0], list) + self.assertFalse(launcher.kwargs[0].get("shell", False)) + self.assertNotIn("shell", launcher.kwargs[0]) + + def test_reveal_does_not_hold_on_to_the_file_manager_output(self) -> None: + launcher = Launcher() + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + self.assertIn("stdout", launcher.kwargs[0]) + self.assertIn("stderr", launcher.kwargs[0]) + + def test_a_missing_file_manager_is_false_not_an_exception(self) -> None: + launcher = Launcher(error=True) + self.assertFalse( + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + ) + + def test_no_command_means_no_launch_attempt(self) -> None: + launcher = Launcher() + with mock.patch.object(opener.shutil, "which", return_value=None): + self.assertFalse( + opener.reveal(Path("/tmp/x.pdf"), platform="linux", runner=launcher) + ) + self.assertEqual(launcher.calls, []) + + @unittest.skipUnless(sys.platform == "darwin", "the macOS default only exists on macOS") + def test_this_machine_can_show_a_path(self) -> None: + self.assertTrue(opener.supported()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_planner.py b/tests/test_cfo_planner.py new file mode 100644 index 0000000..6979ca1 --- /dev/null +++ b/tests/test_cfo_planner.py @@ -0,0 +1,648 @@ +"""The decision chain end to end, with the clock and the safety net injected.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from datetime import timedelta +from pathlib import Path + +from carl_file_organizer import paths +from carl_file_organizer.config import resolve_config +from carl_file_organizer.planner import SCHEMA_VERSION, build_plan, plan_paths, write_plan + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + action_for, + actions_for, + age, + fake_lsof, + guard_with, + write, +) + + +def plan_for(root: Path, *, lang: str = "zh", profile: str = "tiered", **kwargs): + """Build a plan whose home is the temp root, so portable paths stay $HOME/.""" + + config = kwargs.pop("config", None) or resolve_config(root, profile=profile, lang=lang) + options = kwargs.pop("guard_options", GUARD_OFFLINE) + return build_plan( + root, + config=config, + now=kwargs.pop("now", NOW), + home=kwargs.pop("home", root), + guard_options=options, + trash_backend=kwargs.pop("trash_backend", "finder"), + **kwargs + ) + + +class EnvelopeTests(unittest.TestCase): + def test_schema_and_top_level_shape(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md", "# hi\n"), hours=100) + plan = plan_for(root) + + self.assertEqual(plan["schema_version"], SCHEMA_VERSION) + for key in ( + "generator", "created_at", "now", "lang", "lang_source", "profile", + "source_root", "source_root_portable", "managed_dir", "managed_dir_portable", + "names", "capabilities", "scan_policy", "disk", "summary", "pinned", + "actions", "groups", "approved_action_ids", "overrides", + "approved_at", "approved_by", + ): + self.assertIn(key, plan) + self.assertEqual(plan["approved_action_ids"], []) + self.assertEqual(plan["overrides"], []) + self.assertEqual(plan["now"], NOW.isoformat()) + + def test_every_portable_path_starts_at_home(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "photo.png"), hours=100) + plan = plan_for(root) + + seen = 0 + stack = [plan] + while stack: + node = stack.pop() + if isinstance(node, dict): + for key, value in node.items(): + if isinstance(value, str) and key.endswith("_portable"): + seen += 1 + self.assertTrue(value.startswith("$HOME/"), "{0}={1}".format(key, value)) + else: + stack.append(value) + elif isinstance(node, list): + stack.extend(node) + self.assertGreater(seen, 3) + + def test_absolute_paths_appear_only_where_they_are_allowed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + allowed = {"source", "destination", "source_root", "managed_dir"} + for action in plan["actions"]: + for key, value in action.items(): + if key in allowed or not isinstance(value, str): + continue + self.assertNotIn(str(root), value, key) + + def test_capabilities_are_reported(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + self.assertIn( + plan["capabilities"]["trash_backend"], {"finder", "gio", "none"} + ) + self.assertIn("finder_tags", plan["capabilities"]) + self.assertIn("permanent_delete_offered", plan["capabilities"]) + + def test_summary_counts_match_the_actions(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "fresh.md"), hours=1) + (root / "folder").mkdir() + age(root / "folder", hours=100) + plan = plan_for(root) + + by_kind = {} + by_tier = {} + for action in plan["actions"]: + by_kind[action["kind"]] = by_kind.get(action["kind"], 0) + 1 + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + self.assertEqual(plan["summary"]["by_kind"], by_kind) + self.assertEqual(plan["summary"]["by_tier"], by_tier) + self.assertEqual( + plan["summary"]["entries"], + plan["summary"]["files"] + plan["summary"]["dirs"], + ) + self.assertEqual(plan["summary"]["proposed_reclaimed_bytes"], 0) + + def test_plan_paths_and_write_plan(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + config = resolve_config(root, profile="tiered", lang="zh") + where = plan_paths(config) + self.assertEqual(where.managed.name, "00_下载目录管理") + plan = plan_for(root, config=config) + write_plan(plan, where.plan) + self.assertTrue(where.plan.is_file()) + + +class SettlingTests(unittest.TestCase): + def test_forty_seven_hours_holds_and_forty_nine_moves(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = write(root / "report.pdf", "%PDF\n") + + age(target, hours=47) + plan = plan_for(root) + hold = action_for(plan, "report.pdf", "hold") + self.assertEqual(hold["rule"], "hold:aging") + self.assertEqual(hold["tier"], "aging") + self.assertFalse(hold["approvable"]) + self.assertIsNone(hold["destination"]) + self.assertIn("ready_at", hold["hold_reason"]) + ready = hold["hold_reason"]["ready_at"] + self.assertTrue(ready > NOW.isoformat()) + + age(target, hours=49) + plan = plan_for(root) + move = action_for(plan, "report.pdf", "move") + self.assertEqual(move["rule"], "ext:.pdf") + self.assertEqual(move["tier"], "routine") + self.assertTrue(move["approvable"]) + self.assertIn("20_知识库/文档资料/PDF", move["destination_portable"]) + + def test_archives_wait_a_full_week(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "tool.zip", "PK"), hours=100) + plan = plan_for(root) + self.assertEqual(action_for(plan, "tool.zip", "hold")["rule"], "hold:aging") + + def test_ready_at_is_the_modification_time_plus_the_window(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf"), hours=10) + plan = plan_for(root) + hold = action_for(plan, "report.pdf", "hold") + expected = (NOW - timedelta(hours=10) + timedelta(hours=48)).isoformat() + self.assertEqual(hold["hold_reason"]["ready_at"][:16], expected[:16]) + + +class PriorityTests(unittest.TestCase): + def test_pinned_beats_sensitive(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "api-password-notes.md"), hours=100) + config = resolve_config(root, profile="tiered", lang="zh") + config.profile["pinned"] = list(config.profile["pinned"]) + [ + "api-password-notes.md" + ] + plan = plan_for(root, config=config) + + found = actions_for(plan, "api-password-notes.md") + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["rule"], "hold:pinned") + self.assertEqual(found[0]["tier"], "pinned") + + def test_forbidden_beats_sensitive(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + vault = root / "secret-vault.photoslibrary" + vault.mkdir() + age(vault, hours=100) + plan = plan_for(root) + + found = actions_for(plan, "secret-vault.photoslibrary") + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["rule"], "hold:forbidden") + self.assertIn("forbidden:suffix", found[0]["hold_reason"]["label"]) + + def test_a_sensitive_folder_moves_whole_and_ignores_the_settling_period(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + folder = root / "api-credential-dump" + write(folder / "inside.txt") + age(folder, hours=1) + plan = plan_for(root) + + move = action_for(plan, "api-credential-dump", "move") + self.assertEqual(move["tier"], "sensitive") + self.assertEqual(move["subject_kind"], "dir") + self.assertTrue(move["rule"].startswith("sensitive:")) + self.assertIn("60_敏感信息/待转移/api-credential-dump", move["destination_portable"]) + self.assertIn("未读取内容", move["reason"]["zh"]) + + def test_sensitive_beats_the_extension_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "server.pem"), hours=1) + plan = plan_for(root) + move = action_for(plan, "server.pem", "move") + self.assertEqual(move["rule"], "sensitive:suffix") + + def test_pairing_beats_the_plain_folder_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "PK") + write(root / "Archive" / "inside.txt") + for item in (root / "Archive.zip", root / "Archive"): + age(item, hours=300) + plan = plan_for(root) + + folder = action_for(plan, "Archive", "move") + self.assertEqual(folder["rule"], "pair") + self.assertIn("00_收件箱/待判断/Archive/Archive", folder["destination_portable"]) + self.assertIsNotNone(folder["group_id"]) + + +class FolderTests(unittest.TestCase): + def test_a_folder_moves_whole_and_keeps_its_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "foo" / "inner.txt") + age(root / "foo", hours=100) + plan = plan_for(root) + + move = action_for(plan, "foo", "move") + self.assertEqual(move["rule"], "dir:pending") + self.assertTrue(move["reroutable"]) + self.assertTrue(move["destination_portable"].endswith("00_收件箱/待判断/foo")) + self.assertIsNotNone(move["dir_stats"]) + + def test_a_scratch_copy_only_gets_a_hint(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "project-worktree-topic" / "inner.txt") + age(root / "project-worktree-topic", hours=100) + plan = plan_for(root) + + move = action_for(plan, "project-worktree-topic", "move") + self.assertEqual(move["rule"], "dir:derivative") + self.assertIn("derivative-copy:-worktree-", move["hints"]) + + def test_unknown_extensions_wait_in_pending_and_can_be_rerouted(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "dataset.parquet"), hours=100) + plan = plan_for(root) + + move = action_for(plan, "dataset.parquet", "move") + self.assertEqual(move["rule"], "unknown-ext") + self.assertTrue(move["reroutable"]) + self.assertEqual(move["confidence"], 0.55) + self.assertIn(move["destination_key"], plan["names"]) + + +class BoundaryTests(unittest.TestCase): + def test_partitions_hidden_files_and_symlinks_are_skipped(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + write(root / ".hidden.md") + for name in ("00_收件箱", "20_知识库", "00_下载目录管理"): + (root / name).mkdir() + (root / "link.md").symlink_to(root / "notes.md") + write(root / "carl-file-organizer-approved-1.json", "{}") + + plan = plan_for(root) + names = {action["filename"] for action in plan["actions"]} + + self.assertEqual(names, {"notes.md"}) + self.assertEqual(plan["scan_policy"]["skipped"]["partitions"], 3) + self.assertEqual(plan["scan_policy"]["skipped"]["symlinks"], 1) + self.assertEqual(plan["scan_policy"]["skipped"]["artifacts"], 1) + + def test_a_taken_destination_is_redirected_and_never_overwritten(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf", "new\n"), hours=100) + write(root / "20_知识库" / "文档资料" / "PDF" / "report.pdf", "already here\n") + + plan = plan_for(root) + move = action_for(plan, "report.pdf", "move") + + self.assertEqual(move["rule"], "conflict") + self.assertEqual(move["destination_key"], "inbox.dup") + self.assertEqual( + move["destination_portable"], + "$HOME/00_收件箱/重复待确认/report-2026-09-02_12-00-00.pdf", + ) + self.assertFalse(move["reroutable"]) + self.assertEqual( + (root / "20_知识库" / "文档资料" / "PDF" / "report.pdf").read_text(), "already here\n" + ) + + def test_a_redirected_copy_keeps_a_double_suffix_and_a_folder_keeps_none(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "release.tar.gz", "new\n"), hours=400) + write(root / "90_归档" / "压缩包" / "其他压缩包" / "release.tar.gz", "already here\n") + folder = root / "web-prototype" + age(write(folder / "index.html", "

    hi

    \n"), hours=100) + age(folder, hours=100) + write(root / "00_收件箱" / "待判断" / "web-prototype" / "keep.txt", "taken\n") + + plan = plan_for(root) + archive = action_for(plan, "release.tar.gz", "move") + self.assertEqual( + archive["destination_portable"], + "$HOME/00_收件箱/重复待确认/release-2026-09-02_12-00-00.tar.gz", + ) + + moved_dir = action_for(plan, "web-prototype", "move") + self.assertEqual( + moved_dir["destination_portable"], + "$HOME/00_收件箱/重复待确认/web-prototype-2026-09-02_12-00-00", + ) + + +class LanguageTests(unittest.TestCase): + def test_english_names_are_used_end_to_end(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf"), hours=100) + age(write(root / "dataset.parquet"), hours=100) + plan = plan_for(root, lang="en") + + self.assertEqual(plan["lang"], "en") + self.assertIn( + "20_Library/Documents/PDF", action_for(plan, "report.pdf")["destination_portable"] + ) + self.assertIn( + "00_Inbox/Pending", action_for(plan, "dataset.parquet")["destination_portable"] + ) + self.assertEqual(plan["names"]["library.docs.pdf"], "20_Library/Documents/PDF") + + +class GroupTests(unittest.TestCase): + def test_pair_group_options_all_point_at_real_approvable_actions(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "PK") + write(root / "Archive" / "inside.txt") + for item in (root / "Archive.zip", root / "Archive"): + age(item, hours=300) + plan = plan_for(root) + + group = next(g for g in plan["groups"] if g["kind"] == "pair") + by_id = {action["id"]: action for action in plan["actions"]} + keys = [option["key"] for option in group["options"]] + + self.assertIn("keep_all", keys) + self.assertIn(group["default_choice"], keys) + self.assertTrue(group["default_choice"].startswith("keep:")) + for option in group["options"]: + self.assertTrue(option["action_ids"]) + for action_id in option["action_ids"]: + self.assertIn(action_id, by_id) + self.assertTrue(by_id[action_id]["approvable"]) + self.assertEqual(by_id[action_id]["group_id"], group["group_id"]) + if any(by_id[a]["kind"] == "delete" for a in option["action_ids"]): + self.assertIn("allow-permanent-delete", option["requires"]) + + def test_identical_files_form_a_sha256_group_that_defaults_to_keeping_one(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "guide.pdf", "same bytes\n"), hours=100) + age(write(root / "guide (1).pdf", "same bytes\n"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + group = next(g for g in plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(group["evidence"], "sha256") + self.assertIsNotNone(group["sha256"]) + self.assertTrue(group["default_choice"].startswith("keep:")) + self.assertGreater(group["potential_bytes"], 0) + + def test_lookalike_names_default_to_keeping_everything(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "export.png", "AAAA"), hours=100) + age(write(root / "export_1.png", "BBBB"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + group = next(g for g in plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(group["evidence"], "name-pattern") + self.assertEqual(group["default_choice"], "keep_all") + self.assertEqual(group["potential_bytes"], 0) + self.assertFalse( + [ + action + for action in plan["actions"] + if action["group_id"] == group["group_id"] and action["kind"] == "delete" + ] + ) + + def test_regenerable_output_offers_trash_and_delete_only(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "package.json", "{}") + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root) + + group = next(g for g in plan["groups"] if g["kind"] == "regenerable") + self.assertEqual( + {option["key"] for option in group["options"]}, {"trash_all", "delete_all"} + ) + self.assertEqual(group["default_choice"], "trash_all") + self.assertEqual(group["member_labels"][group["members"][0]], "proj/node_modules") + kinds = { + action["kind"] + for action in plan["actions"] + if action["group_id"] == group["group_id"] + } + self.assertEqual(kinds, {"trash", "delete"}) + + def test_permanent_delete_can_be_turned_off_entirely(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root, offer_permanent_delete=False) + + self.assertFalse([a for a in plan["actions"] if a["kind"] == "delete"]) + self.assertFalse(plan["capabilities"]["permanent_delete_offered"]) + + def test_without_a_trash_backend_nothing_offers_the_trash(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root, trash_backend="none") + + self.assertFalse([a for a in plan["actions"] if a["kind"] == "trash"]) + group = next(g for g in plan["groups"] if g["kind"] == "regenerable") + self.assertEqual([option["key"] for option in group["options"]], ["delete_all"]) + + +class GuardChainTests(unittest.TestCase): + def test_an_open_file_is_held_with_the_process_named(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = age(write(root / "session.log", "line\n"), hours=100) + with open(str(target), "r", encoding="utf-8"): + plan = plan_for( + root, + guard_options=guard_with( + lsof_enabled=True, + lsof_runner=fake_lsof({"session.log": "python3,48213"}), + ), + ) + + hold = action_for(plan, "session.log", "hold") + self.assertEqual(hold["rule"], "hold:in_use") + self.assertEqual(hold["tier"], "in_use") + self.assertFalse(hold["approvable"]) + self.assertEqual(hold["guard"]["open_by"][0]["pid"], 48213) + self.assertIn("python3", hold["reason"]["zh"]) + + def test_a_referenced_script_is_held_with_the_line_number(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + age(write(root / "backup-tool.sh", "#!/bin/sh\n"), hours=100) + plist = write( + home / "com.example.backup.plist", + "\n$HOME/Downloads/backup-tool.sh\n\n", + ) + plan = plan_for( + root, home=home, guard_options=guard_with(reference_files=[plist]) + ) + + hold = action_for(plan, "backup-tool.sh", "hold") + self.assertEqual(hold["rule"], "hold:referenced") + self.assertEqual(hold["tier"], "referenced") + self.assertEqual(hold["guard"]["referenced_in"][0]["line"], 2) + self.assertIn(":2", hold["reason"]["zh"]) + + def test_allow_referenced_downgrades_the_hold_to_a_warning(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + age(write(root / "backup-tool.sh", "#!/bin/sh\n"), hours=100) + plist = write( + home / "com.example.backup.plist", + "\n$HOME/Downloads/backup-tool.sh\n\n", + ) + plan = plan_for( + root, + home=home, + allow_referenced=True, + guard_options=guard_with(reference_files=[plist]), + ) + + move = action_for(plan, "backup-tool.sh", "move") + self.assertTrue(move["approvable"]) + self.assertIn("referenced-warning", move["hints"]) + self.assertTrue(move["guard"]["referenced_in"]) + self.assertIn("改引用", move["detail"]["zh"]) + self.assertTrue(plan["scan_policy"]["allow_referenced"]) + + def test_a_symlink_pointing_in_is_a_reference_too(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + target = age(write(root / "backup-tool.sh"), hours=100) + binaries = home / "bin" + binaries.mkdir() + (binaries / "backup-tool").symlink_to(target) + + plan = plan_for(root, home=home, guard_options=guard_with(scan_roots=[binaries])) + hold = action_for(plan, "backup-tool.sh", "hold") + + self.assertEqual(hold["tier"], "referenced") + self.assertEqual(hold["guard"]["incoming_symlinks"], ["$HOME/bin/backup-tool"]) + + def test_a_git_file_means_a_worktree_and_a_worktree_is_never_moved(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + worktree = root / "topic-worktree-fix" + write(worktree / ".git", "gitdir: /elsewhere/.git/worktrees/topic\n") + age(worktree, hours=100) + + plan = plan_for(root) + hold = action_for(plan, "topic-worktree-fix", "hold") + + self.assertEqual(hold["rule"], "hold:referenced") + self.assertEqual(hold["tier"], "referenced") + self.assertIn("git-worktree", hold["guard"]["shape"]) + self.assertIn("worktree", hold["reason"]["en"]) + + def test_a_git_folder_only_gets_a_hint(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + repo = root / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + write(repo / "package.json", "{}") + age(repo, hours=100) + + config = resolve_config(root, profile="tiered", lang="zh") + + class _Status(object): + returncode = 0 + stdout = " M a.py\n" + + plan = plan_for( + root, + config=config, + guard_options=guard_with(git_status_runner=lambda path, timeout: _Status()), + ) + move = action_for(plan, "repo", "move") + + self.assertTrue(move["approvable"]) + self.assertIn("git-repo", move["hints"]) + self.assertIn("node-project", move["hints"]) + self.assertIn("git-dir", move["guard"]["shape"]) + self.assertEqual(move["guard"]["uncommitted"], 1) + + def test_something_touched_ten_minutes_ago_waits(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "dataset.parquet"), hours=0.05) + plan = plan_for(root) + + hold = action_for(plan, "dataset.parquet", "hold") + self.assertEqual(hold["rule"], "hold:aging") + self.assertEqual(hold["tier"], "aging") + self.assertIn("十分钟", hold["reason"]["zh"]) + + +class ColdTests(unittest.TestCase): + def test_a_large_file_gains_a_disposal_option(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "clip.mp4", "x" * 4096), hours=100) + plan = plan_for(root, large_bytes=1024) + + kinds = {action["kind"] for action in actions_for(plan, "clip.mp4")} + self.assertEqual(kinds, {"move", "trash", "delete"}) + trash = action_for(plan, "clip.mp4", "trash") + self.assertEqual(trash["tier"], "cold") + self.assertEqual(trash["rule"], "cold:large") + self.assertEqual(trash["reclaims_bytes"], 4096) + self.assertEqual(action_for(plan, "clip.mp4", "move")["reclaims_bytes"], 0) + + def test_a_settled_installer_gains_a_disposal_option(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "tool.dmg", "x"), hours=400) + plan = plan_for(root) + + trash = action_for(plan, "tool.dmg", "trash") + self.assertEqual(trash["rule"], "cold:installer") + self.assertEqual(trash["tier"], "cold") + + +class SimpleProfileTests(unittest.TestCase): + def test_the_flat_profile_still_works(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "notes.txt", "hello") + plan = plan_for(root, profile="simple", lang="en") + + move = action_for(plan, "notes.txt", "move") + self.assertEqual(move["destination_key"], "documents") + self.assertTrue(move["destination_portable"].endswith("Documents/notes.txt")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_portability.py b/tests/test_cfo_portability.py new file mode 100644 index 0000000..b1921ca --- /dev/null +++ b/tests/test_cfo_portability.py @@ -0,0 +1,182 @@ +"""Reading a plan that was written on the other kind of machine. + +A plan.json and a report.html both travel: made on a Mac, opened on Windows, or +the reverse. So one token is stored, both tokens are understood, and the token +a person actually sees is decided at render time and nowhere else. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import unittest +from pathlib import Path + +from carl_file_organizer import guard, paths, trash + + +class TokenTests(unittest.TestCase): + def test_only_one_token_is_ever_written(self) -> None: + home = Path("/home/someone") + self.assertEqual( + paths.portable(home / "Downloads" / "a.pdf", home), "$HOME/Downloads/a.pdf" + ) + self.assertTrue(paths.portable(home, home).startswith("$HOME")) + + def test_a_stored_path_always_uses_forward_slashes(self) -> None: + home = Path("/home/someone") + stored = paths.portable(home / "Downloads" / "deep" / "a.pdf", home) + self.assertNotIn("\\", stored) + + def test_both_tokens_expand_to_the_same_place(self) -> None: + home = Path("/home/someone") + expected = home / "Downloads" / "a.pdf" + for text in ( + "$HOME/Downloads/a.pdf", + "%USERPROFILE%/Downloads/a.pdf", + "%USERPROFILE%\\Downloads\\a.pdf", + ): + self.assertEqual(paths.expand_portable(text, home), expected, text) + + def test_a_bare_token_is_the_home_directory(self) -> None: + home = Path("/home/someone") + for text in ("$HOME", "%USERPROFILE%"): + self.assertEqual(paths.expand_portable(text, home), home, text) + + def test_something_that_is_not_a_token_is_left_alone(self) -> None: + home = Path("/home/someone") + self.assertEqual(paths.expand_portable("/etc/hosts", home), Path("/etc/hosts")) + + def test_split_home_token_reports_which_spelling_it_found(self) -> None: + self.assertEqual(paths.split_home_token("$HOME/a/b"), ("$HOME", "a/b")) + self.assertEqual( + paths.split_home_token("%USERPROFILE%\\a\\b"), ("%USERPROFILE%", "a/b") + ) + self.assertIsNone(paths.split_home_token("relative/a")) + self.assertIsNone(paths.split_home_token(None)) + + def test_display_swaps_the_token_only_for_windows_eyes(self) -> None: + self.assertEqual( + paths.display_portable("$HOME/Downloads/a.pdf", platform="darwin"), + "$HOME/Downloads/a.pdf", + ) + self.assertEqual( + paths.display_portable("$HOME/Downloads/a.pdf", platform="win32"), + "%USERPROFILE%\\Downloads\\a.pdf", + ) + self.assertEqual(paths.display_portable("$HOME", platform="win32"), "%USERPROFILE%") + + def test_display_leaves_a_path_it_does_not_recognise_untouched(self) -> None: + self.assertEqual( + paths.display_portable("/etc/hosts", platform="win32"), "/etc/hosts" + ) + + def test_home_token_follows_the_platform(self) -> None: + self.assertEqual(paths.home_token("win32"), "%USERPROFILE%") + self.assertEqual(paths.home_token("darwin"), "$HOME") + self.assertEqual(paths.home_token("linux"), "$HOME") + + +class PlanHomeTests(unittest.TestCase): + def test_a_windows_written_plan_still_gives_up_its_home(self) -> None: + plan = { + "source_root": "/home/someone/Downloads", + "source_root_portable": "%USERPROFILE%\\Downloads", + } + self.assertEqual(paths.plan_home(plan), Path("/home/someone")) + + def test_a_mac_written_plan_gives_up_its_home_too(self) -> None: + plan = { + "source_root": "/home/someone/Downloads", + "source_root_portable": "$HOME/Downloads", + } + self.assertEqual(paths.plan_home(plan), Path("/home/someone")) + + def test_a_stripped_plan_falls_back_to_this_machine(self) -> None: + self.assertEqual(paths.plan_home({}), Path.home()) + + +class WindowsBackendTests(unittest.TestCase): + """The Windows code paths, exercised from whatever machine is running this.""" + + def test_the_flag_word_always_keeps_undo(self) -> None: + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_ALLOWUNDO) + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_NOCONFIRMATION) + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_SILENT) + + def test_the_path_argument_ends_in_two_nuls(self) -> None: + argument = trash.windows_path_argument(Path("C:/Users/someone/a.pdf")) + self.assertTrue(argument.endswith("\0\0")) + self.assertEqual(argument.count("\0"), 2) + self.assertTrue(argument.startswith(os.path.abspath("C:/Users/someone/a.pdf"))) + + def test_a_recycle_bin_call_that_returns_nonzero_is_a_failure(self) -> None: + with self.assertRaises(trash.TrashError) as caught: + trash.move_to_trash( + Path("nowhere.pdf"), + backend=trash.WINDOWS, + win_runner=lambda path: (124, False), + ) + self.assertIn("124", str(caught.exception)) + + def test_an_aborted_call_is_a_failure_too(self) -> None: + with self.assertRaises(trash.TrashError): + trash.move_to_trash( + Path("nowhere.pdf"), + backend=trash.WINDOWS, + win_runner=lambda path: (0, True), + ) + + def test_a_success_that_left_the_file_behind_is_still_a_failure(self) -> None: + here = Path(__file__) + with self.assertRaises(trash.TrashError) as caught: + trash.move_to_trash( + here, backend=trash.WINDOWS, win_runner=lambda path: (0, False), lang="zh" + ) + self.assertIn(here.name, str(caught.exception)) + + def test_a_failed_recycle_bin_call_never_becomes_a_delete(self) -> None: + here = Path(__file__) + for runner in ( + lambda path: (1, False), + lambda path: (0, True), + lambda path: (_ for _ in ()).throw(OSError("shell32 went away")), + ): + with self.assertRaises(trash.TrashError): + trash.move_to_trash(here, backend=trash.WINDOWS, win_runner=runner) + self.assertTrue(here.exists()) + + def test_an_unknown_backend_name_is_refused_outright(self) -> None: + with self.assertRaises(trash.TrashError): + trash.move_to_trash(Path("x"), backend="dumpster") + + @unittest.skipUnless(sys.platform == "win32", "shell32 only exists on Windows") + def test_this_windows_machine_can_reach_the_recycle_bin(self) -> None: + self.assertTrue(trash.windows_available()) + self.assertEqual(trash.detect_backend(), trash.WINDOWS) + + @unittest.skipIf(sys.platform == "win32", "the negative case is for everywhere else") + def test_windows_helpers_are_inert_off_windows(self) -> None: + self.assertFalse(trash.windows_available()) + + +class LsofTests(unittest.TestCase): + def test_windows_never_claims_nobody_has_it_open(self) -> None: + """Unknown is a real answer; a guessed "nobody" would let a file move.""" + + if sys.platform == "win32": + self.assertFalse(guard.lsof_possible()) + else: + self.assertEqual(guard.lsof_possible(), guard.shutil.which("lsof") is not None) + + @unittest.skipUnless(sys.platform == "win32", "only Windows lacks lsof by definition") + def test_open_handles_on_windows_is_unknown(self) -> None: + result = guard.open_handles(Path(__file__)) + self.assertEqual(result.status, "unknown") + self.assertFalse(result.handles) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_report.py b/tests/test_cfo_report.py new file mode 100644 index 0000000..f06e4f7 --- /dev/null +++ b/tests/test_cfo_report.py @@ -0,0 +1,379 @@ +"""Review page: one template, two modes, rendered from the schema v2 sample.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import copy +import json +import re +import unittest +from pathlib import Path + +from carl_file_organizer import paths +from carl_file_organizer.report import TEXT, render_report, write_report + +FIXTURE = Path(cfo_path.FIXTURES_DIR) / "plan-v2-sample.json" +EXPECTED_GROUPS = ["move", "pair", "duplicate", "regenerable", "cold", "hold", "protected", "pending"] + + +def _load_plan() -> dict: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def _groups(html: str) -> list: + return re.findall(r'
    str: + match = re.search( + r'
    '.format(key), html, re.S + ) + assert match, "section {0} missing".format(key) + return match.group(0) + + +def _row(html: str, action_id: str) -> str: + match = re.search(r']*data-ids="[^"]*\b{0}\b[^"]*"[^>]*>.*?'.format(action_id), html, re.S) + assert match, "row for {0} missing".format(action_id) + return match.group(0) + + +def _embedded_plan(html: str) -> dict: + """The JSON the page's own script tag reads, undoing the ', html, re.S + ) + assert match, "gn-plan script missing" + text = ( + match.group(1) + .replace("\\u003c", "<") + .replace("\\u003e", ">") + .replace("\\u0026", "&") + ) + return json.loads(text) + + +def _strip_comments(html: str) -> str: + html = re.sub(r"", "", html, flags=re.S) + return re.sub(r"/\*.*?\*/", "", html, flags=re.S) + + +class RenderModesTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.plan = _load_plan() + cls.static = render_report(cls.plan, mode="static") + cls.serve = render_report(cls.plan, mode="serve", token="test-token-value") + + def test_both_modes_render_the_same_sections(self) -> None: + self.assertEqual(_groups(self.static), EXPECTED_GROUPS) + self.assertEqual(_groups(self.serve), EXPECTED_GROUPS) + + def test_body_carries_the_mode(self) -> None: + self.assertIn('', self.static) + + def test_static_exports_and_serve_posts(self) -> None: + self.assertIn(TEXT["zh"]["btn_export"], self.static) + self.assertIn("carl-file-organizer-approved.json", self.static) + self.assertNotIn("X-GN-Token", self.static) + self.assertNotIn("test-token-value", self.static) + self.assertNotIn("/api/apply", self.static) + + self.assertIn("X-GN-Token", self.serve) + self.assertIn("/api/apply", self.serve) + self.assertIn("history.replaceState", self.serve) + self.assertIn("test-token-value", self.serve) + self.assertIn('
    ' + TEXT["zh"]["serve_note"], self.serve) + self.assertNotIn('
    ', self.static) + self.assertIn(TEXT["zh"]["btn_apply"], self.serve) + + def test_static_token_argument_is_ignored(self) -> None: + html = render_report(self.plan, mode="static", token="should-not-leak") + self.assertNotIn("should-not-leak", html) + + def test_pledge_sentence_is_fixed_at_the_top(self) -> None: + pledge = TEXT["zh"]["pledge"] + self.assertIn(pledge, self.static) + self.assertLess(self.static.index(pledge), self.static.index('data-group="move"')) + + def test_every_approvable_action_is_reachable(self) -> None: + for action in self.plan["actions"]: + if not action["approvable"]: + continue + self.assertIn(action["id"], self.static, "approvable id missing from page: " + action["id"]) + + def test_plan_strings_are_escaped(self) -> None: + plan = copy.deepcopy(self.plan) + plan["actions"][0]["filename"] = '.pdf' + plan["actions"][0]["reason"]["zh"] = "含 x.pdf" + for mode in ("static", "serve"): + html = render_report(plan, mode=mode, token="t") + block = re.search(r'', html, re.S).group(1) + self.assertNotIn("<", block) + self.assertNotIn(">", block) + round_trip = json.loads(block) + self.assertEqual(round_trip["actions"][0]["filename"], "x.pdf") + self.assertEqual(round_trip["schema_version"], 2) + + def test_zh_copy_has_no_straight_double_quotes(self) -> None: + for key, value in TEXT["zh"].items(): + self.assertNotIn('"', value, key) + + +class PrivacyTests(unittest.TestCase): + """A report.html handed to somebody else must not name the user's account.""" + + @classmethod + def setUpClass(cls) -> None: + cls.plan = _load_plan() + cls.static = render_report(cls.plan, mode="static") + cls.serve = render_report(cls.plan, mode="serve", token="test-token-value") + + def test_no_absolute_path_reaches_either_page(self) -> None: + planned_home = str(paths.plan_home(self.plan)) + # the directory accounts live in, taken from the fixture rather than + # spelled out: this repository does not carry absolute home paths + accounts_dir = str(Path(planned_home).parent) + "/" + + for name, html in (("static", self.static), ("serve", self.serve)): + self.assertNotIn(self.plan["source_root"], html, name) + self.assertNotIn(self.plan["managed_dir"], html, name) + self.assertNotIn(planned_home, html, name) + self.assertNotIn(accounts_dir, html, name) + self.assertNotIn(str(Path.home()), html, name) + for action in self.plan["actions"]: + self.assertNotIn(action["source"], html, name) + if action.get("destination"): + self.assertNotIn(action["destination"], html, name) + + def test_the_embedded_plan_keeps_only_the_portable_twins(self) -> None: + embedded = _embedded_plan(self.static) + self.assertNotIn("source_root", embedded) + self.assertNotIn("managed_dir", embedded) + self.assertEqual(embedded["source_root_portable"], "$HOME/Downloads") + self.assertEqual(embedded["managed_dir_portable"], self.plan["managed_dir_portable"]) + for action in embedded["actions"]: + self.assertNotIn("source", action) + self.assertNotIn("destination", action) + self.assertTrue(action["source_portable"].startswith("$HOME/")) + + def test_the_page_still_shows_the_folder_and_every_row(self) -> None: + self.assertIn("$HOME/Downloads", self.static) + for action in self.plan["actions"]: + self.assertIn(action["source_portable"], self.static) + + def test_render_does_not_mutate_the_caller_s_plan(self) -> None: + plan = _load_plan() + render_report(plan, mode="static") + self.assertEqual(plan["source_root"], self.plan["source_root"]) + self.assertEqual(plan["actions"][0]["source"], self.plan["actions"][0]["source"]) + self.assertTrue(Path(plan["source_root"]).is_absolute()) + + def test_referenced_in_form_is_rewritten_too(self) -> None: + # form used to be exempt from the $HOME rewrite because it is the + # literal text found in someone's config file; it no longer is. + plan = copy.deepcopy(self.plan) + home = str(paths.plan_home(plan)) + absolute_form = home + "/Downloads/backup-tool.sh" + plan["actions"][0].setdefault("guard", {})["referenced_in"] = [ + {"file": home + "/.zshrc", "line": 7, "form": absolute_form} + ] + html = render_report(plan, mode="static") + self.assertNotIn(absolute_form, html) + self.assertIn("$HOME/Downloads/backup-tool.sh", html) + + +class LanguageTests(unittest.TestCase): + def test_lang_defaults_to_plan_and_can_be_overridden(self) -> None: + plan = _load_plan() + zh = render_report(plan) + en = render_report(plan, lang="en") + self.assertIn(TEXT["zh"]["sec_move"], zh) + self.assertIn(TEXT["en"]["sec_move"], en) + self.assertIn('', en) + self.assertIn("Document filed by type into", en) + self.assertIn("文档按类型归档到", zh) + self.assertEqual(_groups(zh), _groups(en)) + + def test_write_report_writes_static_page(self) -> None: + import tempfile + + plan = _load_plan() + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "report.html" + write_report(plan, target, lang="en") + html = target.read_text(encoding="utf-8") + self.assertIn(' dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _strip_comments(html: str) -> str: + html = re.sub(r"", "", html, flags=re.S) + return re.sub(r"/\*.*?\*/", "", html, flags=re.S) + + +def _cards(html: str, color: str) -> list: + return re.findall(r'
    '.format(color), html, re.S) + + +def _section(html: str, color: str) -> str: + match = re.search(r'
    '.format(color), html, re.S) + assert match, "section {0} missing".format(color) + return match.group(0) + + +def _segbar_widths(html: str) -> list: + bars = re.findall(r'
    (.*?)
    ', html, re.S) + return [[int(w) for w in re.findall(r'style="width:(\d+)%"', bar)] for bar in bars] + + +def _embedded(html: str, element_id: str) -> dict: + match = re.search(r''.format(element_id), html, re.S) + assert match, "{0} script missing".format(element_id) + text = match.group(1).replace("\\u003c", "<").replace("\\u003e", ">").replace("\\u0026", "&") + return json.loads(text) + + +class RenderBothKindsTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.plan = _load(PLAN) + cls.analysis = _load(ANALYSIS) + cls.pages = { + ("organize", "static"): build_report.render(cls.plan, mode="static"), + ("organize", "serve"): build_report.render(cls.plan, mode="serve", token="test-token-value"), + ("storage", "static"): build_report.render(cls.analysis, mode="static"), + ("storage", "serve"): build_report.render(cls.analysis, mode="serve", token="test-token-value"), + } + + def test_kind_is_detected_and_stamped_on_body(self) -> None: + self.assertEqual(build_report.detect_kind(self.plan), "organize") + self.assertEqual(build_report.detect_kind(self.analysis), "storage") + for (kind, mode), html in self.pages.items(): + self.assertIn('', html) + + def test_no_absolute_paths_no_network_no_gradients(self) -> None: + for (kind, mode), html in self.pages.items(): + bare = _strip_comments(html) + self.assertNotIn("/Users/", bare, "{0}/{1} leaks a home path".format(kind, mode)) + self.assertNotIn("http://", bare, "{0}/{1} references the network".format(kind, mode)) + self.assertNotIn("https://", bare, "{0}/{1} references the network".format(kind, mode)) + self.assertNotIn("linear-gradient", bare) + self.assertNotIn("radial-gradient", bare) + self.assertNotIn(" None: + for html in self.pages.values(): + self.assertNotRegex(html, r"__REPORT_[A-Z]+__") + + def test_token_only_in_serve(self) -> None: + self.assertIn("test-token-value", self.pages[("organize", "serve")]) + self.assertIn("test-token-value", self.pages[("storage", "serve")]) + self.assertNotIn("test-token-value", self.pages[("organize", "static")]) + self.assertNotIn("test-token-value", self.pages[("storage", "static")]) + self.assertEqual(_embedded(self.pages[("organize", "static")], "report-config")["token"], "") + + def test_three_sections_in_order(self) -> None: + for html in self.pages.values(): + self.assertEqual(re.findall(r'
    None: + for (kind, mode), html in self.pages.items(): + bars = _segbar_widths(html) + self.assertTrue(bars, "{0}/{1} has no segbar".format(kind, mode)) + for widths in bars: + self.assertLessEqual(sum(widths), 100) + self.assertTrue(all(w >= 0 for w in widths)) + + def test_red_cards_carry_no_delete_or_trash_controls(self) -> None: + for (kind, mode), html in self.pages.items(): + cards = _cards(html, "red") + self.assertTrue(cards, "{0}/{1} has no red cards".format(kind, mode)) + for card in cards: + self.assertNotIn(" None: + for (kind, mode), html in self.pages.items(): + cards = _cards(html, "yellow") + self.assertTrue(cards, "{0}/{1} has no yellow cards".format(kind, mode)) + for card in cards: + self.assertNotRegex(card, r'data-action="delete"') + self.assertNotRegex(card, r'data-kind="delete"') + self.assertNotIn("needs-permanent", card) + + def test_green_cards_offer_trash_and_permanent_delete(self) -> None: + for mode in ("static", "serve"): + storage = self.pages[("storage", mode)] + cards = _cards(storage, "green") + self.assertTrue(cards) + self.assertTrue(any('data-action="trash"' in c for c in cards)) + self.assertTrue(any('data-action="delete"' in c and "needs-permanent" in c for c in cards)) + self.assertIn('id="gn-permanent"', _section(storage, "green")) + # The planner colours every group yellow; a green regenerable group shows the permanent option. + plan = copy.deepcopy(self.plan) + for group in plan["groups"]: + if group["kind"] == "regenerable": + group["color"] = "green" + html = build_report.render(plan, mode="static") + green = _section(html, "green") + self.assertIn('data-action="delete"', green) + self.assertIn('data-kind="delete"', green) + self.assertIn('id="gn-permanent"', green) + self.assertNotIn('data-action="delete"', _section(html, "yellow")) + + def test_organize_pending_has_destination_select_in_yellow(self) -> None: + for mode in ("static", "serve"): + html = self.pages[("organize", mode)] + yellow = _section(html, "yellow") + self.assertIn(''.format(t("adopt_group")) potential = group.get("potential_bytes") or 0 - badges = ['{0}'.format(_e(str(group.get("kind") or "group")))] + badges = ['{0}'.format(_e(t.badge(group.get("kind") or "group")))] if potential: badges.append('{1}'.format(color, _e(t("potential", bytes=format_bytes(potential))))) reason = _pick(group.get("reason"), lang) @@ -897,7 +946,17 @@ def _organize_longterm(plan: Dict[str, Any], lang: str, t: _T) -> str: continue hold = a.get("hold_reason") if isinstance(a.get("hold_reason"), dict) else {} if hold.get("ready_at"): - aging.append("
  • {0}
    • ".format(_e(t("aging_line", name=a.get("filename"), time=hold.get("ready_at"))))) + aging.append( + "
  • {0}
    • ".format( + _e( + t( + "aging_line", + name=a.get("filename"), + time=format_ready_at(hold.get("ready_at"), lang), + ) + ) + ) + ) aging_html = "" if aging: aging_html = "

    {0}

      {1}
    ".format(t("aging_title"), "".join(aging)) @@ -981,9 +1040,15 @@ def item_color(item: Dict[str, Any]) -> str: def _kind_label(kind: Any, t: _T) -> str: - key = "kind_" + str(kind or "other") + """The badge table first, the older ``kind_*`` keys second, the token last.""" + + token = str(kind or "other") + badge = t.badge(token) + if badge != token: + return badge + key = "kind_" + token label = t(key) - return str(kind) if label == key else label + return token if label == key else label def _card_item(item: Dict[str, Any], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T, *, delete_offered: bool) -> str: @@ -1181,6 +1246,14 @@ def _storage_top5(data: Dict[str, Any], lang: str, t: _T) -> str: if not top: return "" by_id = {str(i.get("id")): i for i in (data.get("items") or []) if isinstance(i, dict)} + + def _size(entry: Dict[str, Any]) -> int: + item = by_id.get(str(entry.get("id")), {}) + return int(entry.get("size_bytes") or item.get("size_bytes") or 0) + + # The written order is whatever the Agent typed; the table promises the five + # largest, so it is sorted here rather than trusted. + top = sorted(top, key=_size, reverse=True) rows = [] for entry in top[:5]: item = by_id.get(str(entry.get("id")), {}) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py index 978c506..2eb45b1 100644 --- a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py @@ -47,16 +47,32 @@ #: a download folder within seconds, so without this list a fresh download is #: always "in use" and the plan proposes nothing at all. These processes read, #: index or preview; none of them breaks when a path changes. +#: +#: Finder is on the list for the same reason: selecting a file in a Finder window +#: hands it to the QuickLook thumbnail agent, and a folder the user is looking at +#: while the plan is built would otherwise come back held open by the very act of +#: looking. Names are matched as prefixes because lsof truncates its COMMAND +#: column somewhere between nine and fifteen characters, so the thumbnail agent +#: arrives as ``com.apple.quicklook.ThumbnailsA`` more often than under its full +#: name. OBSERVER_COMMANDS = ( "mdworker", "mds", + "mds_stores", "mdimport", "mdsync", "mdbulkimport", "quicklookd", "qlmanage", "QuickLookUIService", + "com.apple.quicklook.ThumbnailsAgent", + "com.apple.quicklook.ThumbnailsA", + "Finder", "Spotlight", + "suggestd", + "photoanalysisd", + "cloudd", + "bird", "fseventsd", "backupd", "XProtect", @@ -65,14 +81,22 @@ "distnoted", ) +#: The same list, folded once, so the per-handle check stays a plain lookup. +_OBSERVER_PREFIXES = tuple(sorted({name.casefold() for name in OBSERVER_COMMANDS})) + def is_observer(command: str) -> bool: - """True for a daemon that only reads the file and never owns its path.""" + """True for a daemon that only reads the file and never owns its path. + + The match is on the prefix: lsof gives its COMMAND column a fixed width and + cuts long names off, so a truncated ``com.apple.quicklook.ThumbnailsA`` has + to read the same as the whole name. + """ folded = str(command or "").casefold() if not folded: return False - return any(folded.startswith(name.casefold()) for name in OBSERVER_COMMANDS) + return folded.startswith(_OBSERVER_PREFIXES) MAX_REFERENCE_FILES = 400 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py index 6ddaeb4..862db2e 100644 --- a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py @@ -269,13 +269,15 @@ def run(args: Any) -> int: if bool(getattr(args, "report", False)): try: - from .report import write_report + from . import render report_path = output.parent / "report.html" - write_report(merged, report_path) + report_path.write_text( + render.render(merged, mode="static", lang=lang), encoding="utf-8" + ) except Exception as error: # noqa: BLE001 - the plan is the deliverable print( - i18n.t("cli_report_skipped", lang, module="carl_file_organizer.report") + i18n.t("cli_report_skipped", lang, module="build_report.py") + " ({0})".format(error) ) else: diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py index 7863ad5..0d9ec5f 100644 --- a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py @@ -1413,12 +1413,14 @@ def run(args: Any) -> int: print(i18n.t("cli_plan_written", cfg.lang, path=where.plan)) try: - from .report import write_report + from . import render - write_report(plan, where.report) + where.report.write_text( + render.render(plan, mode="static", lang=cfg.lang), encoding="utf-8" + ) except Exception as error: # noqa: BLE001 - the plan itself is the deliverable print( - i18n.t("cli_report_skipped", cfg.lang, module="carl_file_organizer.report") + i18n.t("cli_report_skipped", cfg.lang, module="build_report.py") + " ({0})".format(error) ) else: diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py deleted file mode 100644 index bf47d5b..0000000 --- a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/report.py +++ /dev/null @@ -1,1266 +0,0 @@ -"""Self-contained HTML review page: one template, two modes. - -``render_report(plan, mode="static"|"serve", token="", lang=None)`` renders the -same controls either way. Only the bottom bar and the submit function differ: -static exports ``carl-file-organizer-approved.json`` through a Blob download, serve -POSTs to ``/api/apply`` on the local review server. - -The page reads nothing but plan.json (schema v2). It never loads an external -resource, so it works from ``file://`` with zero requests. Copy lives in the -``TEXT`` table below; ``i18n.py`` is not touched by this module. - -One thing the page never sees is an absolute path. ``render_report`` runs the -plan through :func:`carl_file_organizer.paths.strip_absolute` first, so both the body -and the embedded ``gn-plan`` JSON carry only ``$HOME/...`` spellings and a page -mailed to a colleague does not spell out the user's account name. The -approved.json exported from the page inherits that, and the executor expands the -twins back at apply time. -""" - -from __future__ import annotations - -import json -from html import escape -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from . import paths - -LANGS = ("zh", "en") -MODES = ("static", "serve") -PERMANENT_FLAG = "allow-permanent-delete" -HOLD_TIER_PROTECTED = "forbidden" - -TEXT: Dict[str, Dict[str, str]] = { - "zh": { - "title": "Carl File Organizer 审核页", - "heading": "整理方案，等你批准", - "lede": "扫描只读，页面上的每一项都要你亲手勾选才会执行。", - "bar_path": "目录", - "bar_time": "扫描时间", - "bar_pending": "待批准", - "bar_disk": "卷使用率", - "bar_reclaim": "可释放估算", - "bar_mode": "模式", - "mode_static": "静态页面", - "mode_serve": "本地服务", - "serve_note": "本页只在这台机器可见，关掉终端即失效。", - "pledge": "移动只改路径不删内容，任何软件找不到文件时，用 undo 或清单里的 mv 命令原路放回。", - "pledge_more": "计划生成后先别改动这些文件和目录，改过的条目执行时会被跳过，重新 plan 即可。", - "disk_note": "同一卷上移动释放 0 字节，只有进废纸篓和永久删除的行会腾出空间。", - "items": "项", - "sec_move": "待批准移动", - "sec_move_desc": "按扩展名和命名规则归到分区，目录整个搬且保留原名。", - "sec_pair": "压缩包与解压目录", - "sec_pair_desc": "成对出现的压缩包和目录，选留谁。默认留目录。", - "sec_duplicate": "重复文件", - "sec_duplicate_desc": "内容一致的副本，选留哪一份。名字像副本但没比对过内容的，默认都保留。", - "sec_regenerable": "可再生产物", - "sec_regenerable_desc": "构建产物和依赖目录，删掉能重新生成。默认进废纸篓。", - "sec_cold": "冷存候选", - "sec_cold_desc": "过了静置期的大文件和安装包。默认进废纸篓，不选就是保留。", - "sec_hold": "静置中、常驻与被占用", - "sec_hold_desc": "这一轮不动。静置期没满、常驻白名单、有进程开着、有配置引用着的都在这里。", - "sec_protected": "禁刀区", - "sec_protected_desc": "系统库和应用数据，只展示，任何情况下都不处理。", - "sec_pending": "待判断", - "sec_pending_desc": "规则表认不出的条目。可以在这里改去向，改动会写进 overrides；拿不准就留在收件箱。", - "th_pick": "批准", - "th_file": "文件", - "th_size": "大小", - "th_dest": "去向", - "th_reason": "理由", - "th_action": "处置", - "th_restore": "恢复方式", - "th_status": "状态", - "th_why": "原因", - "adopt_group": "采纳这组", - "opt_trash": "进废纸篓", - "opt_delete": "永久删除", - "toggle_permanent": "显示永久删除选项", - "permanent_not_enabled": "服务启动时没加 --allow-permanent-delete，永久删除在本页不可用。", - "permanent_static_note": "导出后执行时也要加 --allow-permanent-delete，否则整批拒绝。", - "no_trash_backend": "本机没有废纸篓后端，进废纸篓的选项已隐藏。", - "confirm_delete": "永久删除 {path}（{size}）？删掉就没了，废纸篓里也找不到。", - "confirm_apply": "将执行 {moves} 项移动、{trash} 项进废纸篓、{delete} 项永久删除，预计释放 {bytes}。继续？", - "confirm_dry": "只试运行，不动任何文件。检查这 {n} 项？", - "confirm_shutdown": "停掉本地服务？之后要重新运行 review --serve。", - "nothing_selected": "还没勾选任何项。", - "exported_note": "批准文件已下载，把它交给 carl-file-organizer apply。", - "empty": "本组没有条目。", - "footer_count": "已选 {moves} 项移动 · {trash} 项进废纸篓 · {delete} 项永久删除 · 预计释放 {bytes}", - "btn_export": "导出批准文件", - "btn_apply": "执行全部已选", - "btn_dry": "先试运行", - "btn_section": "执行本组已选", - "btn_select_all": "全选本组", - "btn_clear_all": "清空本组", - "btn_shutdown": "停止服务", - "status_working": "处理中", - "status_moved": "已移动", - "status_trashed": "已进废纸篓", - "status_deleted": "已删除", - "status_dry": "试运行通过", - "status_failed": "失败，看终端", - "status_skipped": "已跳过", - "status_done": "已完成", - "request_failed": "请求失败：{status} {detail}", - "network_failed": "连不上本地服务，终端可能已经关掉。", - "hint": "提示", - "tags": "标签", - "dir_files": "{n} 个文件", - "dir_files_trunc": "至少 {n} 个文件", - "dir_label": "目录", - "sensitive_label": "敏感命名", - "ready_at": "静置到 {time}", - "open_by": "被进程占用", - "referenced_in": "被引用", - "symlinks": "被软链指向", - "shape": "目录形态", - "keep_here": "留在原处", - "potential": "可腾出 {bytes}", - "restore_trash": "废纸篓里选放回原处", - "restore_delete": "不可恢复", - "footer_static_hint": "导出的文件会落到浏览器的下载目录，下次扫描前请交给 apply 或收走。", - "print_hint": "打印版隐藏按钮，只留清单。", - }, - "en": { - "title": "Carl File Organizer review", - "heading": "A tidy-up proposal, waiting for your approval", - "lede": "The scan was read-only. Nothing on this page runs until you tick it yourself.", - "bar_path": "Folder", - "bar_time": "Scanned", - "bar_pending": "Awaiting approval", - "bar_disk": "Volume used", - "bar_reclaim": "Reclaimable estimate", - "bar_mode": "Mode", - "mode_static": "static page", - "mode_serve": "local server", - "serve_note": "This page is only reachable from this machine and stops working when the terminal closes.", - "pledge": "Moves only change the path, never the contents. If any software cannot find a file, run undo or the mv command in the manifest to put it back.", - "pledge_more": "Leave these files and folders alone until you apply the plan; anything changed in between is skipped, and a fresh plan fixes that.", - "disk_note": "Moving files between folders on this volume reclaims 0 bytes; only the trash and permanent-delete rows free up space.", - "items": "items", - "sec_move": "Moves awaiting approval", - "sec_move_desc": "Filed by extension and naming rules. Folders move whole and keep their names.", - "sec_pair": "Archives and their extracted folders", - "sec_pair_desc": "An archive and its folder found together; pick which one stays. The folder is the default.", - "sec_duplicate": "Duplicate files", - "sec_duplicate_desc": "Copies with identical contents; pick the one to keep. Name-only matches default to keeping all.", - "sec_regenerable": "Regenerable output", - "sec_regenerable_desc": "Build output and dependency folders that can be rebuilt. Trash is the default.", - "sec_cold": "Cold storage candidates", - "sec_cold_desc": "Large files and installers past their settling period. Trash is the default; leave unticked to keep.", - "sec_hold": "Settling, pinned and in use", - "sec_hold_desc": "Untouched this round: still settling, pinned by you, held open by a process, or referenced from a config.", - "sec_protected": "No-go zone", - "sec_protected_desc": "System libraries and app data. Shown for context, never handled.", - "sec_pending": "Needs a decision", - "sec_pending_desc": "Entries the rule table does not recognise. Change the destination here and it is recorded in overrides; when unsure, leave it in the inbox.", - "th_pick": "Approve", - "th_file": "File", - "th_size": "Size", - "th_dest": "Destination", - "th_reason": "Why", - "th_action": "Action", - "th_restore": "How to restore", - "th_status": "Status", - "th_why": "Reason", - "adopt_group": "Adopt this group", - "opt_trash": "Move to Trash", - "opt_delete": "Delete permanently", - "toggle_permanent": "Show permanent delete options", - "permanent_not_enabled": "The server was started without --allow-permanent-delete, so permanent deletion is unavailable on this page.", - "permanent_static_note": "Apply the exported file with --allow-permanent-delete, otherwise the whole batch is refused.", - "no_trash_backend": "No trash backend on this machine; the trash options are hidden.", - "confirm_delete": "Permanently delete {path} ({size})? It will not be in the Trash afterwards.", - "confirm_apply": "Run {moves} moves, {trash} to Trash and {delete} permanent deletes, reclaiming about {bytes}. Continue?", - "confirm_dry": "Dry run only, nothing is touched. Check these {n} items?", - "confirm_shutdown": "Stop the local server? You will need to run review --serve again.", - "nothing_selected": "Nothing is ticked yet.", - "exported_note": "The approval file was downloaded. Hand it to carl-file-organizer apply.", - "empty": "Nothing in this group.", - "footer_count": "{moves} moves · {trash} to Trash · {delete} permanent deletes · about {bytes} reclaimed", - "btn_export": "Export approved plan", - "btn_apply": "Apply everything ticked", - "btn_dry": "Dry run first", - "btn_section": "Apply this group", - "btn_select_all": "Select all here", - "btn_clear_all": "Clear all here", - "btn_shutdown": "Stop server", - "status_working": "Working", - "status_moved": "Moved", - "status_trashed": "In Trash", - "status_deleted": "Deleted", - "status_dry": "Dry run passed", - "status_failed": "Failed, see terminal", - "status_skipped": "Skipped", - "status_done": "Done", - "request_failed": "Request failed: {status} {detail}", - "network_failed": "Cannot reach the local server; the terminal may have been closed.", - "hint": "Hint", - "tags": "Tags", - "dir_files": "{n} files", - "dir_files_trunc": "at least {n} files", - "dir_label": "folder", - "sensitive_label": "sensitive name", - "ready_at": "settles at {time}", - "open_by": "Held open by", - "referenced_in": "Referenced in", - "symlinks": "Symlinked from", - "shape": "Folder shape", - "keep_here": "Keep in place", - "potential": "frees {bytes}", - "restore_trash": "Put Back from the Trash", - "restore_delete": "Not recoverable", - "footer_static_hint": "The export lands in the browser download folder; hand it to apply or move it away before the next scan.", - "print_hint": "The print version hides the buttons and keeps the list.", - }, -} - - -# -------------------------------------------------------------------------- -# small helpers -# -------------------------------------------------------------------------- - - -def _format_bytes(value: Union[int, float, None]) -> str: - size = float(value or 0) - for unit in ("B", "KB", "MB", "GB", "TB"): - if size < 1024 or unit == "TB": - return "{0:.1f} {1}".format(size, unit) if unit != "B" else "{0} B".format(int(size)) - size /= 1024 - return "{0:.1f} TB".format(size) - - -def _lang(plan: Dict[str, Any], lang: Optional[str]) -> str: - chosen = lang or plan.get("lang") or "en" - return chosen if chosen in LANGS else "en" - - -def _t(lang: str, key: str, **kw: Any) -> str: - template = TEXT[lang].get(key) or TEXT["en"].get(key) or key - if kw: - try: - return template.format(**kw) - except (KeyError, IndexError, ValueError): - return template - return template - - -def _pick(value: Any, lang: str) -> str: - """A ``{zh, en}`` dict, a plain string or None -> display string.""" - - if isinstance(value, dict): - return str(value.get(lang) or value.get("en") or value.get("zh") or "") - if value is None: - return "" - return str(value) - - -def _e(value: Any) -> str: - return escape("" if value is None else str(value), quote=True) - - -def _embed_json(data: Any) -> str: - """JSON safe inside a '.format(_embed_json(plan)), - "".format( - _embed_json(config) - ), - "".format(JS_COMMON), - "".format(JS_SERVE if mode == "serve" else JS_STATIC), - "", - ] - return "\n".join(parts) - - -def write_report(plan: Dict[str, Any], output: Union[str, Path], *, lang: Optional[str] = None) -> None: - """Write the static page (the serve page never touches disk).""" - - Path(output).write_text(render_report(plan, mode="static", lang=lang), encoding="utf-8") diff --git a/skills/ops/carl-file-organizer/scripts/storage_scan.py b/skills/ops/carl-file-organizer/scripts/storage_scan.py index 61691dd..8bea4d1 100644 --- a/skills/ops/carl-file-organizer/scripts/storage_scan.py +++ b/skills/ops/carl-file-organizer/scripts/storage_scan.py @@ -49,6 +49,32 @@ GB = 1024 ** 3 +# ---------------------------------------------------------------- 卷的取舍 + +#: macOS 的 df 会把一台机器的全部卷都摊出来：只读的系统卷、Preboot、VM、 +#: Recovery、每个挂上来的磁盘镜像。它们跟人要腾的空间没关系，列出来只会把 +#: 真正该看的那一行淹掉。所以磁盘列表只留两种卷：装着用户数据的那一个，和 +#: 真外接上来的盘。其余的收进 system_volumes 里存个底，不进正文。 +PRIMARY_DATA_MOUNT = "/System/Volumes/Data" +PRIMARY_DATA_NAME = "Macintosh HD 数据卷" +EXTERNAL_MOUNT_PREFIX = "/Volumes/" + +#: 名字长这样的都是系统自己的卷或者本地快照，哪怕挂在 /Volumes 下也不算外接盘。 +SYSTEM_VOLUME_NAMES = ( + "com.apple.timemachine", + "recovery", + "preboot", + "vm", + "update", + "xarts", + "iscpreboot", + "hardware", +) + +#: 比这还小的挂载点不是能放东西的盘，是脚本挂上来的临时镜像或者 RAM 盘。 +#: 真的外接盘没有小于 1 GiB 的，所以这条门槛只会误伤假挂载。 +MIN_EXTERNAL_VOLUME_BYTES = GB + # ---------------------------------------------------------------- 分类词表 REGENERABLE_DIR_NAMES = ( @@ -249,6 +275,7 @@ def __init__( self._tick = 0 self._abort = False self.dropped_small = 0 + self._system_volumes = [] # -------------------------------------------------- 路径 @@ -821,8 +848,25 @@ def collect_disks(self): return self.collect_disks_windows() return self.collect_disks_macos() + @staticmethod + def volume_role(mount, total_bytes): + """一个挂载点是 primary（数据卷）、external（外接盘），还是 system。""" + if mount == PRIMARY_DATA_MOUNT: + return "primary" + if not mount.startswith(EXTERNAL_MOUNT_PREFIX): + return "system" + name = mount[len(EXTERNAL_MOUNT_PREFIX):].split("/")[0] + low = name.lower() + for marker in SYSTEM_VOLUME_NAMES: + if low == marker or low.startswith(marker + "."): + return "system" + if total_bytes < MIN_EXTERNAL_VOLUME_BYTES: + return "system" + return "external" + def collect_disks_macos(self): disks = [] + system_volumes = [] output = run_cmd(["df", "-k"]) for line in output.splitlines()[1:]: parts = line.split(None, 8) @@ -839,7 +883,18 @@ def collect_disks_macos(self): free_b = int(avail) * 1024 except ValueError: continue - disks.append(self.make_disk(mount, device, total_b, used_b, free_b, "df")) + role = self.volume_role(mount, total_b) + disk = self.make_disk( + mount, device, total_b, used_b, free_b, "df", + name=PRIMARY_DATA_NAME if role == "primary" else None, + primary=role == "primary", + ) + if role == "system": + system_volumes.append(disk) + else: + disks.append(disk) + disks.sort(key=lambda entry: (not entry["primary"], entry["mount"])) + self._system_volumes = system_volumes if not disks: fallback = self.fallback_disk("/") if fallback: @@ -870,7 +925,8 @@ def fallback_disk(self, root): return None return self.make_disk(root, root, total, used, free, "shutil.disk_usage") - def make_disk(self, mount, device, total, used, free, source): + def make_disk(self, mount, device, total, used, free, source, + name=None, primary=False): capacity = 0.0 if used + free > 0: capacity = round(100.0 * used / float(used + free), 1) @@ -880,6 +936,8 @@ def make_disk(self, mount, device, total, used, free, source): if free < 30 * GB: flags.append("low_free") return { + "name": name or os.path.basename(mount.rstrip("/\\")) or mount, + "primary": bool(primary), "mount": mount, "device": device, "total_bytes": total, @@ -984,6 +1042,7 @@ def run(self): "home_token": self.home_token, "system": system, "disks": disks, + "system_volumes": self._system_volumes, "groups": groups, "top_files": top, "denied": denied, diff --git a/tests/test_cfo_carl_file_organizer.py b/tests/test_cfo_carl_file_organizer.py index 1c22e6b..2332ea7 100644 --- a/tests/test_cfo_carl_file_organizer.py +++ b/tests/test_cfo_carl_file_organizer.py @@ -12,7 +12,7 @@ from carl_file_organizer.config import resolve_config from carl_file_organizer.executor import PlanError, apply_approved_plan from carl_file_organizer.planner import build_plan -from carl_file_organizer.report import render_report +from carl_file_organizer.render import render # The safety net has nowhere to look in these tests: no launch agents, no # symlink scan roots, no lsof. Behaviour must not depend on the machine. @@ -131,7 +131,7 @@ def test_report_embeds_readable_plan_and_export(self) -> None: root = Path(temporary) (root / "paper.pdf").write_text("document", encoding="utf-8") plan = plan_for(root) - html = render_report(plan) + html = render(plan, mode="static") self.assertIn("paper.pdf", html) self.assertIn(json.dumps(plan["schema_version"]), html) diff --git a/tests/test_cfo_dispose.py b/tests/test_cfo_dispose.py index b5f0595..f76cdfb 100644 --- a/tests/test_cfo_dispose.py +++ b/tests/test_cfo_dispose.py @@ -12,6 +12,7 @@ import json import os +import sys import tempfile import unittest from datetime import datetime, timedelta, timezone @@ -247,6 +248,10 @@ def test_a_path_that_climbs_back_out_is_refused(self): self.assertEqual(self.status_of(report, "green-climb"), ["refused"]) self.assertTrue(outside.is_dir()) + @unittest.skipUnless( + sys.platform != "win32", + "creating a symlink on Windows needs administrator rights", + ) def test_a_symlink_segment_is_refused(self): real = self.make_dir("Library/Caches/real-cache") link = self.home / "Library" / "Caches" / "linked" diff --git a/tests/test_cfo_guard.py b/tests/test_cfo_guard.py index e7f7d63..23cfd31 100644 --- a/tests/test_cfo_guard.py +++ b/tests/test_cfo_guard.py @@ -99,6 +99,37 @@ def test_spotlight_and_friends_are_not_counted_as_users(self) -> None: self.assertFalse(result.handles) self.assertEqual(result.status, "ok") + def test_finder_preview_daemons_are_observers_too(self) -> None: + # lsof cuts its COMMAND column off, so the QuickLook thumbnail agent + # usually arrives under a truncated name. Both spellings, and the rest + # of the Finder preview and iCloud crowd, only look at the file. + for command in ( + "com.apple.quicklook.ThumbnailsAgent", + "com.apple.quicklook.ThumbnailsA", + "QuickLookUIService", + "Finder", + "mds_stores", + "suggestd", + "photoanalysisd", + "cloudd", + "bird", + ): + self.assertTrue(guard.is_observer(command), command) + self.assertFalse(guard.is_observer("Preview")) + + def test_a_truncated_quicklook_handle_is_not_a_user(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "poster.png") + result = open_handles( + target, + 5.0, + lsof_runner=fake_lsof( + {"poster.png": "com.apple.quicklook.ThumbnailsA,71204"} + ), + ) + self.assertFalse(result.handles) + self.assertEqual(result.status, "ok") + def test_batched_probe_attributes_each_hit_to_its_own_path(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) diff --git a/tests/test_cfo_planner.py b/tests/test_cfo_planner.py index 6979ca1..dc365d2 100644 --- a/tests/test_cfo_planner.py +++ b/tests/test_cfo_planner.py @@ -492,6 +492,27 @@ def test_an_open_file_is_held_with_the_process_named(self) -> None: self.assertEqual(hold["guard"]["open_by"][0]["pid"], 48213) self.assertIn("python3", hold["reason"]["zh"]) + def test_a_file_the_finder_is_previewing_is_not_held(self) -> None: + # Clicking a file in a Finder window hands it to the QuickLook thumbnail + # agent, and lsof reports that agent under a truncated name. Reading it + # as a user would hold back everything the user happens to be looking at. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "poster.png", "png\n"), hours=100) + plan = plan_for( + root, + guard_options=guard_with( + lsof_enabled=True, + lsof_runner=fake_lsof( + {"poster.png": "com.apple.quicklook.ThumbnailsA,71204"} + ), + ), + ) + + kinds = {a["kind"] for a in plan["actions"] if a["filename"] == "poster.png"} + self.assertNotIn("hold", kinds) + self.assertNotEqual(action_for(plan, "poster.png")["tier"], "in_use") + def test_a_referenced_script_is_held_with_the_line_number(self) -> None: with tempfile.TemporaryDirectory() as temporary: home = Path(temporary) diff --git a/tests/test_cfo_report.py b/tests/test_cfo_report.py deleted file mode 100644 index f06e4f7..0000000 --- a/tests/test_cfo_report.py +++ /dev/null @@ -1,379 +0,0 @@ -"""Review page: one template, two modes, rendered from the schema v2 sample.""" - -from __future__ import annotations - -import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path - -import copy -import json -import re -import unittest -from pathlib import Path - -from carl_file_organizer import paths -from carl_file_organizer.report import TEXT, render_report, write_report - -FIXTURE = Path(cfo_path.FIXTURES_DIR) / "plan-v2-sample.json" -EXPECTED_GROUPS = ["move", "pair", "duplicate", "regenerable", "cold", "hold", "protected", "pending"] - - -def _load_plan() -> dict: - return json.loads(FIXTURE.read_text(encoding="utf-8")) - - -def _groups(html: str) -> list: - return re.findall(r'
    str: - match = re.search( - r'
    '.format(key), html, re.S - ) - assert match, "section {0} missing".format(key) - return match.group(0) - - -def _row(html: str, action_id: str) -> str: - match = re.search(r']*data-ids="[^"]*\b{0}\b[^"]*"[^>]*>.*?'.format(action_id), html, re.S) - assert match, "row for {0} missing".format(action_id) - return match.group(0) - - -def _embedded_plan(html: str) -> dict: - """The JSON the page's own script tag reads, undoing the ', html, re.S - ) - assert match, "gn-plan script missing" - text = ( - match.group(1) - .replace("\\u003c", "<") - .replace("\\u003e", ">") - .replace("\\u0026", "&") - ) - return json.loads(text) - - -def _strip_comments(html: str) -> str: - html = re.sub(r"", "", html, flags=re.S) - return re.sub(r"/\*.*?\*/", "", html, flags=re.S) - - -class RenderModesTests(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.plan = _load_plan() - cls.static = render_report(cls.plan, mode="static") - cls.serve = render_report(cls.plan, mode="serve", token="test-token-value") - - def test_both_modes_render_the_same_sections(self) -> None: - self.assertEqual(_groups(self.static), EXPECTED_GROUPS) - self.assertEqual(_groups(self.serve), EXPECTED_GROUPS) - - def test_body_carries_the_mode(self) -> None: - self.assertIn('', self.static) - - def test_static_exports_and_serve_posts(self) -> None: - self.assertIn(TEXT["zh"]["btn_export"], self.static) - self.assertIn("carl-file-organizer-approved.json", self.static) - self.assertNotIn("X-GN-Token", self.static) - self.assertNotIn("test-token-value", self.static) - self.assertNotIn("/api/apply", self.static) - - self.assertIn("X-GN-Token", self.serve) - self.assertIn("/api/apply", self.serve) - self.assertIn("history.replaceState", self.serve) - self.assertIn("test-token-value", self.serve) - self.assertIn('
    ' + TEXT["zh"]["serve_note"], self.serve) - self.assertNotIn('
    ', self.static) - self.assertIn(TEXT["zh"]["btn_apply"], self.serve) - - def test_static_token_argument_is_ignored(self) -> None: - html = render_report(self.plan, mode="static", token="should-not-leak") - self.assertNotIn("should-not-leak", html) - - def test_pledge_sentence_is_fixed_at_the_top(self) -> None: - pledge = TEXT["zh"]["pledge"] - self.assertIn(pledge, self.static) - self.assertLess(self.static.index(pledge), self.static.index('data-group="move"')) - - def test_every_approvable_action_is_reachable(self) -> None: - for action in self.plan["actions"]: - if not action["approvable"]: - continue - self.assertIn(action["id"], self.static, "approvable id missing from page: " + action["id"]) - - def test_plan_strings_are_escaped(self) -> None: - plan = copy.deepcopy(self.plan) - plan["actions"][0]["filename"] = '.pdf' - plan["actions"][0]["reason"]["zh"] = "含 x.pdf" - for mode in ("static", "serve"): - html = render_report(plan, mode=mode, token="t") - block = re.search(r'', html, re.S).group(1) - self.assertNotIn("<", block) - self.assertNotIn(">", block) - round_trip = json.loads(block) - self.assertEqual(round_trip["actions"][0]["filename"], "x.pdf") - self.assertEqual(round_trip["schema_version"], 2) - - def test_zh_copy_has_no_straight_double_quotes(self) -> None: - for key, value in TEXT["zh"].items(): - self.assertNotIn('"', value, key) - - -class PrivacyTests(unittest.TestCase): - """A report.html handed to somebody else must not name the user's account.""" - - @classmethod - def setUpClass(cls) -> None: - cls.plan = _load_plan() - cls.static = render_report(cls.plan, mode="static") - cls.serve = render_report(cls.plan, mode="serve", token="test-token-value") - - def test_no_absolute_path_reaches_either_page(self) -> None: - planned_home = str(paths.plan_home(self.plan)) - # the directory accounts live in, taken from the fixture rather than - # spelled out: this repository does not carry absolute home paths - accounts_dir = str(Path(planned_home).parent) + "/" - - for name, html in (("static", self.static), ("serve", self.serve)): - self.assertNotIn(self.plan["source_root"], html, name) - self.assertNotIn(self.plan["managed_dir"], html, name) - self.assertNotIn(planned_home, html, name) - self.assertNotIn(accounts_dir, html, name) - self.assertNotIn(str(Path.home()), html, name) - for action in self.plan["actions"]: - self.assertNotIn(action["source"], html, name) - if action.get("destination"): - self.assertNotIn(action["destination"], html, name) - - def test_the_embedded_plan_keeps_only_the_portable_twins(self) -> None: - embedded = _embedded_plan(self.static) - self.assertNotIn("source_root", embedded) - self.assertNotIn("managed_dir", embedded) - self.assertEqual(embedded["source_root_portable"], "$HOME/Downloads") - self.assertEqual(embedded["managed_dir_portable"], self.plan["managed_dir_portable"]) - for action in embedded["actions"]: - self.assertNotIn("source", action) - self.assertNotIn("destination", action) - self.assertTrue(action["source_portable"].startswith("$HOME/")) - - def test_the_page_still_shows_the_folder_and_every_row(self) -> None: - self.assertIn("$HOME/Downloads", self.static) - for action in self.plan["actions"]: - self.assertIn(action["source_portable"], self.static) - - def test_render_does_not_mutate_the_caller_s_plan(self) -> None: - plan = _load_plan() - render_report(plan, mode="static") - self.assertEqual(plan["source_root"], self.plan["source_root"]) - self.assertEqual(plan["actions"][0]["source"], self.plan["actions"][0]["source"]) - self.assertTrue(Path(plan["source_root"]).is_absolute()) - - def test_referenced_in_form_is_rewritten_too(self) -> None: - # form used to be exempt from the $HOME rewrite because it is the - # literal text found in someone's config file; it no longer is. - plan = copy.deepcopy(self.plan) - home = str(paths.plan_home(plan)) - absolute_form = home + "/Downloads/backup-tool.sh" - plan["actions"][0].setdefault("guard", {})["referenced_in"] = [ - {"file": home + "/.zshrc", "line": 7, "form": absolute_form} - ] - html = render_report(plan, mode="static") - self.assertNotIn(absolute_form, html) - self.assertIn("$HOME/Downloads/backup-tool.sh", html) - - -class LanguageTests(unittest.TestCase): - def test_lang_defaults_to_plan_and_can_be_overridden(self) -> None: - plan = _load_plan() - zh = render_report(plan) - en = render_report(plan, lang="en") - self.assertIn(TEXT["zh"]["sec_move"], zh) - self.assertIn(TEXT["en"]["sec_move"], en) - self.assertIn('', en) - self.assertIn("Document filed by type into", en) - self.assertIn("文档按类型归档到", zh) - self.assertEqual(_groups(zh), _groups(en)) - - def test_write_report_writes_static_page(self) -> None: - import tempfile - - plan = _load_plan() - with tempfile.TemporaryDirectory() as tmp: - target = Path(tmp) / "report.html" - write_report(plan, target, lang="en") - html = target.read_text(encoding="utf-8") - self.assertIn(' None: self.assertIn("What it is", body) +class EscapingTests(unittest.TestCase): + """A filename is data. It never becomes markup, in the body or in the JSON.""" + + def setUp(self) -> None: + self.plan = _load(PLAN) + + def test_plan_strings_are_escaped_in_the_body(self) -> None: + plan = copy.deepcopy(self.plan) + plan["actions"][0]["filename"] = '.pdf' + plan["actions"][0]["reason"]["zh"] = "含 x.pdf" + for mode in ("static", "serve"): + html = build_report.render(plan, mode=mode, token="t") + block = re.search( + r'', html, re.S + ).group(1) + self.assertNotIn("<", block) + self.assertNotIn(">", block) + round_trip = json.loads( + block.replace("\\u003c", "<").replace("\\u003e", ">").replace("\\u0026", "&") + ) + self.assertEqual(round_trip["actions"][0]["filename"], "x.pdf") + self.assertEqual(round_trip["schema_version"], 2) + + def test_render_does_not_mutate_the_caller_s_plan(self) -> None: + plan = _load(PLAN) + build_report.render(plan, mode="static") + self.assertEqual(plan["source_root"], self.plan["source_root"]) + self.assertEqual(plan["actions"][0]["source"], self.plan["actions"][0]["source"]) + self.assertTrue(Path(plan["source_root"]).is_absolute()) + + def test_every_approvable_action_is_reachable(self) -> None: + html = build_report.render(self.plan, mode="static") + for action in self.plan["actions"]: + if not action["approvable"]: + continue + self.assertIn(action["id"], html, "approvable id missing: " + action["id"]) + + def test_a_referenced_form_is_rewritten_too(self) -> None: + # form is the literal text found in someone's config file; it used to be + # exempt from the $HOME rewrite and no longer is. + plan = copy.deepcopy(self.plan) + home = str(Path(plan["source_root"]).parent) + absolute_form = home + "/Downloads/backup-tool.sh" + plan["actions"][0].setdefault("guard", {})["referenced_in"] = [ + {"file": home + "/.zshrc", "line": 7, "form": absolute_form} + ] + html = build_report.render(plan, mode="static") + self.assertNotIn(absolute_form, html) + self.assertIn("$HOME/Downloads/backup-tool.sh", html) + + def test_static_carries_no_server_wiring(self) -> None: + # One template serves both modes, so the fetch code is in the file + # either way; what a static page must not carry is a token or a live + # endpoint it could actually post to. + html = build_report.render(self.plan, mode="static", token="should-not-leak") + self.assertNotIn("should-not-leak", html) + self.assertEqual(_embedded(html, "report-config")["token"], "") + self.assertEqual(_embedded(html, "report-config")["mode"], "static") + + +class CopyTests(unittest.TestCase): + def test_the_two_languages_carry_the_same_keys(self) -> None: + text = build_report.template_text() + self.assertEqual(set(text["zh"]), set(text["en"])) + self.assertEqual(set(text["zh"]["badge"]), set(text["en"]["badge"])) + + def test_zh_copy_has_no_straight_double_quotes(self) -> None: + for key, value in build_report.template_text()["zh"].items(): + if isinstance(value, dict): + for token, label in value.items(): + self.assertNotIn('"', label, "{0}.{1}".format(key, token)) + else: + self.assertNotIn('"', value, key) + + +class BadgeTests(unittest.TestCase): + """Cards say 静置中, not aging. A badge is copy, not a token dump.""" + + @classmethod + def setUpClass(cls) -> None: + cls.plan = _load(PLAN) + cls.analysis = _load(ANALYSIS) + cls.zh = build_report.render(cls.plan, mode="static") + cls.storage_zh = build_report.render(cls.analysis, mode="static") + + def _badges(self, html: str) -> list: + return re.findall(r'([^<]*)', html) + + def test_no_english_token_reaches_a_chinese_badge(self) -> None: + for name, html in (("organize", self.zh), ("storage", self.storage_zh)): + found = self._badges(html) + self.assertTrue(found, name) + for label in found: + for token in ( + "aging", "in use", "in_use", "referenced", "forbidden", "pinned", + "dev_cache", "app_data", "build_artifact", "regenerable", + "duplicate", "pair", "unknown", + ): + self.assertNotIn(token, label, "{0}: {1}".format(name, label)) + + def test_the_settling_tier_reads_as_chinese(self) -> None: + self.assertIn(">静置中<", self.zh) + self.assertIn(">被占用<", self.zh) + + def test_english_badges_come_from_the_english_table(self) -> None: + html = build_report.render(self.plan, mode="static", lang="en") + self.assertIn(">settling<", html) + self.assertIn(">in use<", html) + + def test_an_unknown_token_falls_through_unchanged(self) -> None: + t = build_report._T(build_report.template_text(), "zh") + self.assertEqual(t.badge("aging"), "静置中") + self.assertEqual(t.badge("brand_new_tier"), "brand_new_tier") + self.assertEqual(t.badge(""), "") + + +class ReadyAtTests(unittest.TestCase): + def test_an_iso_stamp_becomes_a_readable_time(self) -> None: + self.assertEqual( + build_report.format_ready_at("2026-09-04T11:56:03.412870+09:00", "zh"), + "9 月 4 日 11:56", + ) + self.assertEqual( + build_report.format_ready_at("2026-09-04T11:56:03.412870+09:00", "en"), + "Sep 4, 11:56", + ) + + def test_anything_that_is_not_a_stamp_is_handed_through(self) -> None: + self.assertEqual(build_report.format_ready_at("", "zh"), "") + self.assertEqual(build_report.format_ready_at("soon", "zh"), "soon") + + def test_the_page_shows_the_readable_form_and_not_the_iso_one(self) -> None: + plan = _load(PLAN) + html = build_report.render(plan, mode="static") + self.assertNotIn("2026-09-07T19:15:00", html.split('id="report-data"')[0]) + self.assertIn("9 月 7 日 19:15", html) + english = build_report.render(plan, mode="static", lang="en") + self.assertIn("Sep 7, 19:15", english) + + +class TopFiveTests(unittest.TestCase): + def test_the_table_is_sorted_by_size_whatever_the_analysis_says(self) -> None: + analysis = _load(ANALYSIS) + analysis["top5"] = list(reversed(analysis["top5"])) + html = build_report.render(analysis, mode="static") + sizes = re.findall(r'.*?([^<]+)', html, re.S) + self.assertEqual(len(sizes), 5) + + def as_bytes(text: str) -> float: + number, unit = text.split() + return float(number) * {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3, "TB": 1024 ** 4}[unit] + + values = [as_bytes(x) for x in sizes] + self.assertEqual(values, sorted(values, reverse=True)) + + +class StaticEntranceTests(unittest.TestCase): + """``plan`` and ``build --report`` write the same page the server serves.""" + + def test_build_report_writes_the_shared_template(self) -> None: + import tempfile + + from carl_file_organizer.cli import main + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + plan_path = root / "plan.json" + plan_path.write_text(PLAN.read_text(encoding="utf-8"), encoding="utf-8") + self.assertEqual(main(["build", str(plan_path), "--report"]), 0) + html = (root / "report.html").read_text(encoding="utf-8") + self.assertIn('', html) + self.assertNotIn("/Users/", _strip_comments(html)) + + def test_the_old_report_module_is_gone(self) -> None: + with self.assertRaises(ImportError): + __import__("carl_file_organizer.report") + + class SanitizeTests(unittest.TestCase): def test_home_rewritten_everywhere(self) -> None: data = { diff --git a/tests/test_cfo_storage_scan.py b/tests/test_cfo_storage_scan.py index c820332..ccae919 100644 --- a/tests/test_cfo_storage_scan.py +++ b/tests/test_cfo_storage_scan.py @@ -365,6 +365,94 @@ def test_windows_disks_use_injected_drives(self): self.assertIsInstance(disks, list) +DF_NOISE = """Filesystem 1024-blocks Used Available Capacity iused ifree %iused Mounted on +/dev/disk3s1s1 239362496 15832136 21770672 43% 458734 217706720 0% / +devfs 240 240 0 100% 832 0 100% /dev +/dev/disk3s6 239362496 6291900 21770672 23% 6 217706720 0% /System/Volumes/VM +/dev/disk3s2 239362496 17537476 21770672 45% 2134 217706720 0% /System/Volumes/Preboot +/dev/disk3s4 239362496 751320 21770672 4% 464 217706720 0% /System/Volumes/Update +/dev/disk1s2 512000 6164 494308 2% 1 4943080 0% /System/Volumes/xarts +/dev/disk1s1 512000 6120 494308 2% 37 4943080 0% /System/Volumes/iSCPreboot +/dev/disk1s3 512000 548 494308 1% 74 4943080 0% /System/Volumes/Hardware +/dev/disk3s5 239362496 174542392 21770672 89% 2486382 217706720 1% /System/Volumes/Data +map auto_home 0 0 0 100% 0 0 - /System/Volumes/Data/home +/dev/disk3s1 239362496 15832136 21770672 43% 458732 217706720 0% /System/Volumes/Update/mnt1 +/dev/disk5s1 10236 20 9872 1% 2 98720 0% /Volumes/probe-payload-25539 +/dev/disk7s1 10236 20 9872 1% 2 98720 0% /Volumes/probe-payload-62171 +/dev/disk8s1 41943040 20971520 20971520 50% 9 999999 0% /Volumes/com.apple.TimeMachine.localsnapshots +""" + +DF_WITH_EXTERNAL = DF_NOISE + ( + "/dev/disk9s1 1953458176 900000000 1053458176 47% 42 999999" + " 0% /Volumes/Field Drive\n" +) + + +class MacDiskListTest(unittest.TestCase): + """磁盘列表只留有意义的卷，其余的收进 system_volumes 只做记录。""" + + def setUp(self): + self.real_run_cmd = storage_scan.run_cmd + self.addCleanup(setattr, storage_scan, "run_cmd", self.real_run_cmd) + + def scanner(self, df_output): + storage_scan.run_cmd = lambda args, timeout=None: ( + df_output if list(args)[:1] == ["df"] else "" + ) + return storage_scan.StorageScanner( + home="/tmp/nowhere", platform_name="darwin", with_system=False + ) + + def test_only_the_data_volume_survives_the_noise(self): + scanner = self.scanner(DF_NOISE) + disks = scanner.collect_disks() + self.assertEqual([d["mount"] for d in disks], ["/System/Volumes/Data"]) + self.assertTrue(disks[0]["primary"]) + self.assertEqual(disks[0]["name"], storage_scan.PRIMARY_DATA_NAME) + self.assertEqual(disks[0]["total_bytes"], 239362496 * 1024) + + folded = [d["mount"] for d in scanner._system_volumes] + for mount in ( + "/", + "/System/Volumes/VM", + "/System/Volumes/Preboot", + "/System/Volumes/Update", + "/System/Volumes/xarts", + "/System/Volumes/iSCPreboot", + "/System/Volumes/Hardware", + "/Volumes/probe-payload-25539", + "/Volumes/com.apple.TimeMachine.localsnapshots", + ): + self.assertIn(mount, folded) + + def test_a_real_external_drive_is_kept_after_the_data_volume(self): + scanner = self.scanner(DF_WITH_EXTERNAL) + disks = scanner.collect_disks() + self.assertEqual( + [d["mount"] for d in disks], + ["/System/Volumes/Data", "/Volumes/Field Drive"], + ) + self.assertFalse(disks[1]["primary"]) + self.assertEqual(disks[1]["name"], "Field Drive") + + def test_the_scan_envelope_carries_the_folded_volumes(self): + self.scanner(DF_NOISE) + data = storage_scan.run_scan( + home="/tmp/nowhere", platform_name="darwin", budget_seconds=5, + min_size_bytes=0, with_system=True, + ) + self.assertEqual([d["mount"] for d in data["disks"]], ["/System/Volumes/Data"]) + self.assertTrue(data["system_volumes"]) + + def test_volume_role_rules(self): + role = storage_scan.StorageScanner.volume_role + self.assertEqual(role("/System/Volumes/Data", 1), "primary") + self.assertEqual(role("/", 10 * storage_scan.GB), "system") + self.assertEqual(role("/Volumes/Backup", 0), "system") + self.assertEqual(role("/Volumes/Recovery", 200 * storage_scan.GB), "system") + self.assertEqual(role("/Volumes/Backup", 200 * storage_scan.GB), "external") + + class StorageScanCliTest(unittest.TestCase): """命令行入口：确认能落盘、能读回、只读。""" From 5862bae54502e50963034bd29da7f0947251891a Mon Sep 17 00:00:00 2001 From: LearnPrompt Date: Wed, 2 Sep 2026 15:00:21 +0900 Subject: [PATCH 7/9] =?UTF-8?q?feat(carl-file-organizer):=20=E6=95=B4?= =?UTF-8?q?=E7=90=86=E5=90=8E=E9=A2=84=E8=A7=88=E5=9B=BE=EF=BC=88Canvas=20?= =?UTF-8?q?=E5=8A=9B=E5=AF=BC=E5=90=91=EF=BC=8Caimap=20=E9=A3=8E=E6=A0=BC?= =?UTF-8?q?=EF=BC=8C=E9=9B=B6=E4=BE=9D=E8=B5=96=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../scripts/carl_file_organizer/preview.py | 1368 +++++++++++++++++ tests/test_cfo_preview.py | 368 +++++ 2 files changed, 1736 insertions(+) create mode 100644 skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py create mode 100644 tests/test_cfo_preview.py diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py new file mode 100644 index 0000000..7d10643 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py @@ -0,0 +1,1368 @@ +"""The after picture: one knowledge-graph canvas of where everything lands. + +The report ends with a question a table cannot answer well: if I say yes, what +does the folder look like tomorrow? This module answers it as a picture. + +``build_graph`` turns plan.json (optionally plus a storage analysis) into a tiny +graph: a root, five partitions, the second level of folders under them, one node +per subject, and — when a storage analysis is supplied — a second root for the +machine with the cleanup candidates hanging off it. ``render_preview_html`` +turns that graph into a self-contained ``
    ``: canvas, style and script in +one string, no external resource of any kind, safe to paste at the end of the +report or to open on its own from ``file://``. + +Two layouts are computed for every node. In *now* the files are piled evenly +around the root because that is exactly where they are; in *after* they sit +under the folder they are headed for. The switch at the top interpolates +between the two, which is the whole point of the picture. + +The module is deliberately dependency-free (stdlib only, no intra-package +imports) so it can be tested and rendered in isolation. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from typing import Any, Dict, List, Mapping, Optional, Sequence + +__all__ = [ + "MAX_NODES", + "PARTITION_ORDER", + "PREVIEW_TEXT", + "build_graph", + "render_preview_html", + "render_preview_page", + "main", +] + +#: Hard ceiling on nodes. Past this the picture stops being a picture. +MAX_NODES = 400 + +#: The five partitions, in the order they should ring the root. +PARTITION_ORDER = ("inbox", "work", "library", "sensitive", "archive") + +#: Only these two kinds place a subject on the map, in this order of preference. +_KIND_PRECEDENCE = ("move", "hold") + +#: Kinds that free up space rather than move it. +_RECLAIM_KINDS = ("trash", "delete") + +_HOME_RE = re.compile(r"(?:/Users|/home)/[^/\s\"']+") + +# -------------------------------------------------------------------------- +# copy +# -------------------------------------------------------------------------- + +#: Every string the fragment can show, in both languages. Keys are added, never +#: renamed; the zh column avoids straight double quotes. +PREVIEW_TEXT: Dict[str, Dict[str, str]] = { + "zh": { + "title": "整理后长什么样", + "state_now": "现在", + "state_after": "整理后", + "stat_moving": "项要搬", + "stat_staying": "项留下", + "stat_reclaim": "可腾出", + "legend_green": "绿 · 照做就行", + "legend_yellow": "黄 · 看一眼", + "legend_red": "红 · 先别动", + "shape_partition": "分区", + "shape_folder": "目录", + "shape_file": "文件", + "shape_storage": "待清", + "hint": "拖拽节点 · 滚轮缩放 · 点开关看整理后", + "tip_size": "体积", + "tip_from": "现在", + "tip_to": "去向", + "tip_stay": "留在原地", + "tip_clean": "清掉能腾出空间", + "count_unit": "项", + "empty": "这一轮没有要搬的东西", + "machine": "本机", + "rest": "其余 {k} 项", + }, + "en": { + "title": "What it looks like afterwards", + "state_now": "Now", + "state_after": "After", + "stat_moving": "moving", + "stat_staying": "staying", + "stat_reclaim": "frees up", + "legend_green": "Green · just do it", + "legend_yellow": "Yellow · take a look", + "legend_red": "Red · leave it alone", + "shape_partition": "Partition", + "shape_folder": "Folder", + "shape_file": "File", + "shape_storage": "To clear", + "hint": "Drag nodes · scroll to zoom · flip the switch", + "tip_size": "Size", + "tip_from": "Now", + "tip_to": "Goes to", + "tip_stay": "Stays where it is", + "tip_clean": "Clearing this frees space", + "count_unit": "items", + "empty": "Nothing to move this round", + "machine": "This Mac", + "rest": "{k} more items", + }, +} + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def _scrub(value: Any) -> Any: + """Replace any real home directory with ``~`` before it reaches the page.""" + + if isinstance(value, str): + return _HOME_RE.sub("~", value) + if isinstance(value, dict): + return {key: _scrub(item) for key, item in value.items()} + if isinstance(value, list): + return [_scrub(item) for item in value] + return value + + +def _pick_lang(value: Any, lang: str) -> Optional[str]: + if isinstance(value, Mapping): + text = value.get(lang) or value.get("zh") or value.get("en") + return _scrub(text) if isinstance(text, str) else None + if isinstance(value, str): + return _scrub(value) + return None + + +def _relative_destination(action: Mapping[str, Any], roots: Sequence[str]) -> Optional[str]: + """The action's destination as a path relative to the source root.""" + + for key, root in (("destination_portable", roots[0]), ("destination", roots[1])): + dest = action.get(key) + if not isinstance(dest, str) or not dest: + continue + if root and dest.startswith(root.rstrip("/") + "/"): + return dest[len(root.rstrip("/")) + 1 :] + dest = action.get("destination_portable") or action.get("destination") + if isinstance(dest, str) and dest: + # Unanchored destination: keep the tail so the label still reads. + return dest.lstrip("/") + return None + + +def _partition_names(plan: Mapping[str, Any]) -> List[Dict[str, str]]: + names = plan.get("names") or {} + out: List[Dict[str, str]] = [] + seen = set() + for key in PARTITION_ORDER: + label = names.get(key) + if isinstance(label, str) and label and label not in seen: + seen.add(label) + out.append({"key": key, "label": label}) + for key, label in sorted(names.items()): + if "." in key or key in PARTITION_ORDER: + continue + if isinstance(label, str) and label and label not in seen: + seen.add(label) + out.append({"key": key, "label": label}) + return out + + +def _subject_actions(plan: Mapping[str, Any]) -> List[Mapping[str, Any]]: + """One action per subject: a move wins, otherwise a hold, otherwise nothing.""" + + chosen: Dict[str, Mapping[str, Any]] = {} + for action in plan.get("actions") or []: + if not isinstance(action, Mapping): + continue + kind = action.get("kind") + if kind not in _KIND_PRECEDENCE: + continue + subject = action.get("subject_id") or action.get("id") + if not subject: + continue + current = chosen.get(subject) + if current is None: + chosen[subject] = action + continue + if _KIND_PRECEDENCE.index(kind) < _KIND_PRECEDENCE.index(current.get("kind")): + chosen[subject] = action + ordered: List[Mapping[str, Any]] = [] + seen = set() + for action in plan.get("actions") or []: + if not isinstance(action, Mapping): + continue + subject = action.get("subject_id") or action.get("id") + if subject in chosen and subject not in seen and chosen[subject] is action: + seen.add(subject) + ordered.append(action) + return ordered + + +def _reclaim_bytes(plan: Mapping[str, Any]) -> int: + """Bytes the trash and delete actions would free, counted once per subject.""" + + per_subject: Dict[str, int] = {} + for action in plan.get("actions") or []: + if not isinstance(action, Mapping) or action.get("kind") not in _RECLAIM_KINDS: + continue + subject = action.get("subject_id") or action.get("id") or "" + size = action.get("size_bytes") or action.get("bytes") or 0 + try: + size = int(size) + except (TypeError, ValueError): + size = 0 + per_subject[subject] = max(per_subject.get(subject, 0), size) + return sum(per_subject.values()) + + +# -------------------------------------------------------------------------- +# build_graph +# -------------------------------------------------------------------------- + + +def build_graph(plan: Mapping[str, Any], analysis: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]: + """Turn a plan (and optionally a storage analysis) into the preview graph.""" + + if not isinstance(plan, Mapping): + raise TypeError("plan must be a mapping loaded from plan.json") + + lang = plan.get("lang") if plan.get("lang") in ("zh", "en") else "zh" + root_portable = _scrub(plan.get("source_root_portable") or plan.get("source_root") or "~") + root_label = os.path.basename(root_portable.rstrip("/")) or root_portable + + nodes: List[Dict[str, Any]] = [] + links: List[Dict[str, str]] = [] + index: Dict[str, Dict[str, Any]] = {} + + def add_node(node: Dict[str, Any]) -> Dict[str, Any]: + node.setdefault("color", None) + node.setdefault("size_bytes", 0) + node.setdefault("to", None) + node.setdefault("from", None) + node.setdefault("hint", None) + index[node["id"]] = node + nodes.append(node) + return node + + def add_link(source: str, target: str, kind: str) -> None: + links.append({"source": source, "target": target, "kind": kind}) + + add_node( + { + "id": "root", + "label": root_label, + "type": "root", + "depth": 1, + "to": root_portable, + "from": root_portable, + } + ) + + # -- machine layer ------------------------------------------------------ + storage_leaves: List[Dict[str, Any]] = [] + if isinstance(analysis, Mapping): + machine = analysis.get("machine") or {} + host = machine.get("hostname") if isinstance(machine, Mapping) else None + add_node( + { + "id": "machine", + "label": _scrub(host) or PREVIEW_TEXT[lang]["machine"], + "type": "machine", + "depth": 0, + "hint": _pick_lang((analysis.get("overview") or {}).get("headline"), lang), + } + ) + add_link("root", "machine", "after") + for item in analysis.get("items") or []: + if not isinstance(item, Mapping): + continue + item_id = item.get("id") + if not item_id: + continue + path = _scrub(item.get("path_portable") or item.get("path") or "") + parent = path.rsplit("/", 1)[0] if "/" in path else path + try: + size = int(item.get("size_bytes") or 0) + except (TypeError, ValueError): + size = 0 + color = item.get("color") if item.get("color") in ("green", "yellow", "red") else "yellow" + what = _pick_lang(item.get("what"), lang) or "" + hint = what + if color == "green": + hint = (PREVIEW_TEXT[lang]["tip_clean"] + " · " + what).strip(" ·") + node = { + "id": "s:" + str(item_id), + "label": _scrub(item.get("name") or item_id), + "type": "storage", + "depth": 2, + "color": color, + "size_bytes": size, + "from": parent or None, + "to": None, + "hint": hint or None, + "pending_clean": color == "green", + "parent": "machine", + } + storage_leaves.append(node) + + # -- partitions --------------------------------------------------------- + partitions = _partition_names(plan) + partition_by_label: Dict[str, str] = {} + for entry in partitions: + node_id = "p:" + entry["key"] + partition_by_label[entry["label"]] = node_id + add_node( + { + "id": node_id, + "label": entry["label"], + "type": "partition", + "depth": 2, + "to": root_label + "/" + entry["label"], + } + ) + add_link(node_id, "root", "after") + + # -- files -------------------------------------------------------------- + roots = ( + _scrub(plan.get("source_root_portable") or ""), + plan.get("source_root") or "", + ) + folder_ids: Dict[str, str] = {} + file_leaves: List[Dict[str, Any]] = [] + moving = 0 + staying = 0 + + for action in _subject_actions(plan): + subject = str(action.get("subject_id") or action.get("id")) + kind = action.get("kind") + try: + size = int(action.get("size_bytes") or action.get("bytes") or 0) + except (TypeError, ValueError): + size = 0 + color = action.get("color") if action.get("color") in ("green", "yellow", "red") else "yellow" + label = _scrub(action.get("filename") or subject) + hint_map = action.get("reason") if isinstance(action.get("reason"), Mapping) else None + node: Dict[str, Any] = { + "id": "f:" + subject, + "label": label, + "type": "file", + "depth": 4, + "color": color, + "size_bytes": size, + "kind": "dir" if action.get("subject_kind") == "dir" else "file", + "from": root_label, + "hint": _pick_lang(hint_map, lang), + "hint_i18n": _scrub(dict(hint_map)) if hint_map else None, + "moving": kind == "move", + } + + parent = "root" + relative = _relative_destination(action, roots) if kind == "move" else None + if relative: + segments = [part for part in relative.split("/") if part] + directories = segments[:-1] + if directories: + node["to"] = "/".join(directories) + partition_id = partition_by_label.get(directories[0]) + if partition_id is None: + partition_id = "p:" + directories[0] + if partition_id not in index: + add_node( + { + "id": partition_id, + "label": directories[0], + "type": "partition", + "depth": 2, + "to": root_label + "/" + directories[0], + } + ) + add_link(partition_id, "root", "after") + partition_by_label[directories[0]] = partition_id + parent = partition_id + if len(directories) >= 2: + folder_key = directories[0] + "/" + directories[1] + folder_id = folder_ids.get(folder_key) + if folder_id is None: + folder_id = "d:" + folder_key + folder_ids[folder_key] = folder_id + add_node( + { + "id": folder_id, + "label": directories[1], + "type": "folder", + "depth": 3, + "to": folder_key, + } + ) + add_link(folder_id, partition_id, "after") + parent = folder_id + + if node.get("to") is None: + node["to"] = root_label + node["moving"] = False + + node["parent"] = parent + if node["moving"]: + moving += 1 + else: + staying += 1 + file_leaves.append(node) + + # -- cap ---------------------------------------------------------------- + leaves = file_leaves + storage_leaves + structural = len(nodes) + budget = MAX_NODES - structural - 1 + dropped: List[Dict[str, Any]] = [] + if budget < 0: + budget = 0 + if len(leaves) > budget: + leaves = sorted(leaves, key=lambda item: (-int(item.get("size_bytes") or 0), item["id"])) + dropped = leaves[budget:] + leaves = leaves[:budget] + + kept_ids = {node["id"] for node in leaves} + for node in file_leaves + storage_leaves: + if node["id"] not in kept_ids: + continue + parent = node.pop("parent") + add_node(node) + add_link(node["id"], parent, "after") + if node["type"] == "file": + add_link(node["id"], "root", "now") + + if dropped: + rest_bytes = sum(int(item.get("size_bytes") or 0) for item in dropped) + rest_label = { + code: PREVIEW_TEXT[code]["rest"].format(k=len(dropped)) for code in ("zh", "en") + } + add_node( + { + "id": "agg:rest", + "label": rest_label[lang], + "label_i18n": rest_label, + "type": "file", + "depth": 4, + "color": "yellow", + "size_bytes": rest_bytes, + "kind": "file", + "from": root_label, + "to": root_label, + "moving": False, + "aggregate": len(dropped), + } + ) + add_link("agg:rest", "root", "after") + add_link("agg:rest", "root", "now") + + reclaim = _reclaim_bytes(plan) + if isinstance(analysis, Mapping): + for item in analysis.get("items") or []: + if isinstance(item, Mapping) and item.get("color") == "green": + try: + reclaim += int(item.get("size_bytes") or 0) + except (TypeError, ValueError): + pass + + stats = { + "files": len(file_leaves), + "moving": moving, + "staying": staying, + "partitions": len(partitions), + "reclaim_bytes": reclaim, + "storage_items": len(storage_leaves), + "dropped": len(dropped), + "lang": lang, + "root": root_label, + } + return {"nodes": nodes, "links": links, "stats": stats} + + +# -------------------------------------------------------------------------- +# render +# -------------------------------------------------------------------------- + + +def _js_json(payload: Any) -> str: + """JSON that is safe to sit inline in a `` +
    """ + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +def _load(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog="carl_file_organizer.preview", + description="Render the after picture for a plan as a standalone HTML page.", + ) + parser.add_argument("plan", help="path to plan.json") + parser.add_argument("--analysis", default=None, help="path to a storage analysis JSON") + parser.add_argument("-o", "--output", default=None, help="write the page here (default: stdout)") + parser.add_argument("--lang", default=None, choices=["zh", "en"], help="page language") + parser.add_argument("--height", type=int, default=720, help="canvas height in px") + parser.add_argument("--fragment", action="store_true", help="emit only the
    ") + args = parser.parse_args(list(argv) if argv is not None else None) + + plan = _load(args.plan) + analysis = _load(args.analysis) if args.analysis else None + graph = build_graph(plan, analysis) + lang = args.lang or (plan.get("lang") if plan.get("lang") in ("zh", "en") else "zh") + if args.fragment: + html = render_preview_html(graph, lang=lang, height=args.height) + else: + html = render_preview_page(graph, lang=lang, height=args.height) + + if args.output: + directory = os.path.dirname(os.path.abspath(args.output)) + if directory: + os.makedirs(directory, exist_ok=True) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(html) + sys.stderr.write( + "preview: %d nodes, %d links -> %s\n" + % (len(graph["nodes"]), len(graph["links"]), args.output) + ) + else: + sys.stdout.write(html) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cfo_preview.py b/tests/test_cfo_preview.py new file mode 100644 index 0000000..5a2c332 --- /dev/null +++ b/tests/test_cfo_preview.py @@ -0,0 +1,368 @@ +"""The after picture, and the promise that it is only a picture. + +The preview reads plan.json and draws where things land. It decides nothing, it +fetches nothing, and it never shows anybody's home directory. Every test here +is one of those three promises, or the arithmetic that keeps the drawing honest: +one node per subject, five partitions, hold stays at the root, and a graph too +big to read collapses into an aggregate instead of a hairball. +""" + +from __future__ import annotations + +import os +import sys + +# Runs under both invocations the repo uses: ``unittest discover -s tests`` +# (which puts tests/ on the path for us) and ``unittest tests.test_cfo_preview`` +# (which does not). +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import cfo_path # noqa: E402,F401 - puts the skill's scripts dir on sys.path + +import copy +import json +import re +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import preview +from carl_file_organizer.preview import ( + MAX_NODES, + PREVIEW_TEXT, + build_graph, + render_preview_html, + render_preview_page, +) + +FIXTURES = Path(cfo_path.FIXTURES_DIR) +PLAN_PATH = FIXTURES / "plan-v2-sample.json" +ANALYSIS_PATH = FIXTURES / "storage-analysis-sample.json" + + +def load(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +def index(graph): + return {node["id"]: node for node in graph["nodes"]} + + +class GraphShapeTest(unittest.TestCase): + """The graph mirrors the plan: one node per subject, and the tree above it.""" + + def setUp(self): + self.plan = load(PLAN_PATH) + self.graph = build_graph(self.plan) + + def test_one_node_per_subject_move_beats_hold_and_the_rest(self): + subjects = { + action["subject_id"] + for action in self.plan["actions"] + if action["kind"] in ("move", "hold") + } + files = [node for node in self.graph["nodes"] if node["type"] == "file"] + self.assertEqual(len(files), len(subjects)) + self.assertEqual(len({node["id"] for node in files}), len(files)) + + # Archive.zip carries move + trash + delete; only the move survives. + archive = index(self.graph)["f:3c4d5e6f7a8b9c0d"] + self.assertTrue(archive["moving"]) + self.assertTrue(archive["to"].startswith("00_收件箱/待判断")) + + def test_five_partitions_labelled_by_their_real_directory_names(self): + partitions = [node for node in self.graph["nodes"] if node["type"] == "partition"] + self.assertEqual(len(partitions), 5) + self.assertEqual(self.graph["stats"]["partitions"], 5) + self.assertEqual( + [node["label"] for node in partitions], + [self.plan["names"][key] for key in preview.PARTITION_ORDER], + ) + + def test_second_level_folders_are_deduplicated(self): + folders = [node for node in self.graph["nodes"] if node["type"] == "folder"] + labels = [node["label"] for node in folders] + self.assertEqual(len(labels), len(set(labels)) if len(set(labels)) == len(labels) else len(labels)) + self.assertEqual(len({node["id"] for node in folders}), len(folders)) + # Three PDFs share one 文档资料 folder node. + self.assertEqual(sum(1 for node in folders if node["label"] == "文档资料"), 1) + self.assertIn("待判断", labels) + + def test_hold_files_stay_at_the_root(self): + nodes = index(self.graph) + root_label = nodes["root"]["label"] + holds = [ + action["subject_id"] + for action in self.plan["actions"] + if action["kind"] == "hold" + ] + self.assertTrue(holds) + for subject in holds: + node = nodes["f:" + subject] + self.assertEqual(node["to"], root_label) + self.assertFalse(node["moving"]) + self.assertEqual(node["color"], "red") + + def test_every_link_points_at_a_real_node_and_now_links_cover_every_file(self): + ids = set(index(self.graph)) + for link in self.graph["links"]: + self.assertIn(link["source"], ids) + self.assertIn(link["target"], ids) + self.assertIn(link["kind"], ("now", "after")) + files = [node for node in self.graph["nodes"] if node["type"] == "file"] + now_links = [link for link in self.graph["links"] if link["kind"] == "now"] + self.assertEqual(len(now_links), len(files)) + self.assertTrue(all(link["target"] == "root" for link in now_links)) + # Child -> parent, exactly once per node that has a parent. + after_parents = [link["source"] for link in self.graph["links"] if link["kind"] == "after"] + self.assertEqual(len(after_parents), len(set(after_parents))) + + def test_stats_add_up(self): + stats = self.graph["stats"] + self.assertEqual(stats["files"], stats["moving"] + stats["staying"]) + self.assertGreater(stats["moving"], 0) + self.assertGreater(stats["staying"], 0) + # node_modules is trashed rather than moved, so it only shows up as space. + self.assertGreaterEqual(stats["reclaim_bytes"], 198311744) + + def test_no_home_directory_ever_reaches_the_graph(self): + blob = json.dumps(self.graph, ensure_ascii=False) + self.assertNotIn("/Users/", blob) + self.assertNotIn("/home/", blob) + + +class MachineLayerTest(unittest.TestCase): + """With an analysis in hand, the cleanup map joins the same drawing.""" + + def setUp(self): + self.plan = load(PLAN_PATH) + self.analysis = load(ANALYSIS_PATH) + self.graph = build_graph(self.plan, self.analysis) + + def test_machine_root_carries_every_analysis_item(self): + nodes = index(self.graph) + self.assertEqual(nodes["machine"]["type"], "machine") + self.assertEqual(nodes["machine"]["label"], self.analysis["machine"]["hostname"]) + storage = [node for node in self.graph["nodes"] if node["type"] == "storage"] + self.assertEqual(len(storage), len(self.analysis["items"])) + self.assertEqual(self.graph["stats"]["storage_items"], len(storage)) + parents = { + link["target"] + for link in self.graph["links"] + if link["source"].startswith("s:") + } + self.assertEqual(parents, {"machine"}) + self.assertIn( + {"source": "root", "target": "machine", "kind": "after"}, self.graph["links"] + ) + + def test_green_items_are_flagged_as_pending_clean(self): + nodes = index(self.graph) + derived = nodes["s:st-derived-data"] + self.assertEqual(derived["color"], "green") + self.assertTrue(derived["pending_clean"]) + self.assertIn(PREVIEW_TEXT["zh"]["tip_clean"], derived["hint"]) + chrome = nodes["s:st-chrome-profile"] + self.assertEqual(chrome["color"], "yellow") + self.assertFalse(chrome["pending_clean"]) + + def test_analysis_green_bytes_join_the_reclaim_total(self): + without = build_graph(self.plan)["stats"]["reclaim_bytes"] + green = sum( + item["size_bytes"] for item in self.analysis["items"] if item["color"] == "green" + ) + self.assertEqual(self.graph["stats"]["reclaim_bytes"], without + green) + + def test_storage_paths_stay_portable(self): + blob = json.dumps(self.graph, ensure_ascii=False) + self.assertNotIn("/Users/", blob) + self.assertIn("$HOME", blob) + + +class NodeBudgetTest(unittest.TestCase): + """Past four hundred nodes the picture stops adding dots and starts counting.""" + + def synthetic(self, count): + plan = load(PLAN_PATH) + template = plan["actions"][0] + actions = [] + for i in range(count): + action = copy.deepcopy(template) + action["id"] = "gen%06d" % i + action["subject_id"] = "gen%06d" % i + action["filename"] = "file-%06d.pdf" % i + action["size_bytes"] = 1000 + i + action["destination_portable"] = ( + "$HOME/Downloads/20_知识库/文档资料/PDF/file-%06d.pdf" % i + ) + action["destination"] = None + actions.append(action) + plan["actions"] = actions + return plan + + def test_under_the_cap_nothing_is_aggregated(self): + graph = build_graph(self.synthetic(50)) + self.assertEqual(graph["stats"]["dropped"], 0) + self.assertNotIn("agg:rest", index(graph)) + + def test_over_the_cap_the_tail_becomes_one_node(self): + graph = build_graph(self.synthetic(900)) + self.assertLessEqual(len(graph["nodes"]), MAX_NODES) + aggregate = index(graph)["agg:rest"] + self.assertGreater(aggregate["aggregate"], 0) + self.assertEqual( + aggregate["label"], PREVIEW_TEXT["zh"]["rest"].format(k=aggregate["aggregate"]) + ) + kept = [node for node in graph["nodes"] if node["type"] == "file" and node["id"] != "agg:rest"] + self.assertEqual(len(kept) + aggregate["aggregate"], 900) + # The survivors are the big ones. + self.assertEqual( + min(node["size_bytes"] for node in kept), + max( + node["size_bytes"] for node in kept + ) - (len(kept) - 1), + ) + + def test_the_cap_also_covers_the_machine_layer(self): + graph = build_graph(self.synthetic(900), load(ANALYSIS_PATH)) + self.assertLessEqual(len(graph["nodes"]), MAX_NODES) + + +class FragmentTest(unittest.TestCase): + """A fragment you can paste anywhere: no network, no leaks, no surprises.""" + + def setUp(self): + self.graph = build_graph(load(PLAN_PATH), load(ANALYSIS_PATH)) + self.fragment = render_preview_html(self.graph, lang="zh") + + def test_it_is_one_self_contained_section(self): + self.assertTrue(self.fragment.startswith("
    ")) + self.assertIn("", self.fragment) + self.assertIn("