diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9f00d04 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.12"] + exclude: + # macos-latest is arm64, and setup-python only resolves 3.9 from + # 3.9.13 onward there, sometimes failing to find a build at all. + # The ubuntu 3.9 leg already guards the syntax floor. + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Everything is stdlib on purpose, so there is nothing to install and the + # 3.9 leg is a real floor rather than a formality. + - name: Compile check + run: python -m compileall -q skills tests + - name: Run tests + run: python -m unittest discover -s tests -v + + hygiene: + name: hygiene + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: no private paths, borrowed vocabulary or private product names + run: bash skills/ops/carl-file-organizer/scripts/hygiene.sh diff --git a/README.md b/README.md index 198ea24..8d4c19a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ | ✍️ [**x-article-publisher**](#x-article-publisher) | [![](https://img.shields.io/github/stars/LearnPrompt/x-article-publisher-skill?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/x-article-publisher-skill) | 把飞书或本地Markdown文章发布到X Articles草稿 | [canonical](https://github.com/LearnPrompt/x-article-publisher-skill) | | 🔁 [**skill-sync**](#skill-sync) | [![](https://img.shields.io/github/stars/LearnPrompt/skill-sync?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/skill-sync) | 把多端Agent skills整理成一个可信来源 | [canonical](https://github.com/LearnPrompt/skill-sync) | | 🧭 [**Skill 瘦身**](#skill-slimming) | [![](https://img.shields.io/github/stars/LearnPrompt/carl-skills?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/carl-skills) | 把Agent能力整理成全局、项目和按需触发 | [collection-native](./skills/ops/skill-slimming/SKILL.md) | -| 🗂️ [**文件整理 carl-file-organizer**](#carl-file-organizer) | [![](https://img.shields.io/github/stars/LearnPrompt/carl-file-organizer?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/carl-file-organizer) | 先看后动的下载目录整理器，只读扫出方案，网页勾选批准才移动，删除默认进废纸篓 | [canonical](https://github.com/LearnPrompt/carl-file-organizer) | +| 🗂️ [**文件整理 carl-file-organizer**](#carl-file-organizer) | [![](https://img.shields.io/github/stars/LearnPrompt/carl-skills?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/carl-skills) | 先看后动的文件整理与磁盘盘点，只读扫出方案，三色分级，网页上点头才动 | [collection-native](./skills/ops/carl-file-organizer/SKILL.md) | | 🏮 [**阿福 afu**](#afu-llm-todo) | [![](https://img.shields.io/github/stars/LearnPrompt/afu-llm-todo?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/afu-llm-todo) | Obsidian收件箱管家，Inbox到Wiki到待办到周历一条线 | [canonical](https://github.com/LearnPrompt/afu-llm-todo) | | 📜 [**蔡伦 cailun**](#cailun) | [![](https://img.shields.io/github/stars/LearnPrompt/cailun-skill?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/cailun-skill) | 把对话里聊出来的结论，3秒造成一页能传阅的单文件纸 | [canonical](https://github.com/LearnPrompt/cailun-skill) | | ⛰️ [**愚公 yugong**](#loop-engineering) | [![](https://img.shields.io/github/stars/LearnPrompt/loop-engineering?style=flat&label=%E2%98%85&color=555)](https://github.com/LearnPrompt/loop-engineering) | Loop工程方法论，把模糊目标改造成带验证门的自动循环 | [canonical](https://github.com/LearnPrompt/loop-engineering) | @@ -65,7 +65,7 @@ npx skills add LearnPrompt/carl-skills --skill skill-slimming -g 这条命令会读取仓库目录来发现Skill，但只把`skill-slimming`这一个完整目录安装到Agent，不会把Carl Skills里的其他Skill一起启用。 -Skill 瘦身包含本地复审网页、状态服务和参考合同，不能只下载raw `SKILL.md`。因此它暂不支持Hermes的单文件raw安装；否则能看到提示词，却缺少实际运行层。 +Skill 瘦身包含本地复审网页、状态服务和参考合同，不能只下载raw `SKILL.md`。因此它暂不支持Hermes的单文件raw安装；否则能看到提示词，却缺少实际运行层。文件整理carl-file-organizer也是同一套装法，把`--skill`换成`carl-file-organizer`即可。 ### 装这个目录里的全部可安装skill @@ -495,28 +495,40 @@ npx skills add LearnPrompt/carl-skills --skill skill-slimming -g ### 🗂️ 文件整理 carl-file-organizer -> *"先站好队，再决定谁走，删谁都要你点头。"* +> *整理文件夹和清磁盘这两件事，过去靠忍，或者靠一个看不懂你文件的软件，现在跟Agent说一句就够了。* -它是一个零依赖的Python命令行工具，专门整理下载目录一类越堆越乱的文件夹。plan阶段只读扫描，给出一份整理方案；方案要在网页上逐条勾选批准，apply阶段才真正移动文件。目录按编号分区，00_收件箱、10_工作区、20_知识库、60_敏感信息、90_归档，中英文目录名都能用。 +随口跟Agent说一句「收拾一下Downloads」或者「磁盘满了」，它先只读扫一遍，再打开一份网页报告。整理页上看目录多乱、每个文件往哪走、压缩包和重复副本怎么配对；盘点页上看磁盘去哪了、最大的几项、清理优先级。每一项都写着这是什么、为什么这么判、动了会怎样，你在页面上勾、选、点，动作前都有二次确认，做完能一键撤回。 -判断一份文件该去哪，看的是静置期、敏感命名（只看名字不读内容）、成对压缩包、重复副本、可再生产物这些线索。删除动作默认进废纸篓，要永久删除得显式开关打开。真正动手前会先查一遍进程占用和配置引用，动完的每一步都能undo，Finder标签会打上「文件移动」方便复核。 +传统整理器按扩展名把文件搬走，回头想找一份合同却不知道去了哪；清理软件扫到几个G的目录只写一句用户缓存可删，你不知道里面装的是什么，也不知道删完哪些网站要重新登录。这个skill分两层，脚本先把大小、静置期、压缩包配对、重复副本、进程占用这些能用规则说清的判断做完，Agent再看一眼目录，把每一项翻译成人话。方案先给你看，你点头才动，删除默认进废纸篓，移动过的能原路放回。 + +**三色分级是核心** + +| 颜色 | 能按什么 | +|---|---| +| 🟢 放心归位 / 可清 | 勾上就走，永久删除要你自己额外开开关 | +| 🟡 你看一眼 | 只有打开所在位置和移到废纸篓，都可逆 | +| 🔴 别动 | 只解释为什么不能动，最多打开所在位置 | **适合** -- 下载目录已经乱成泥石流 +- 下载目录已经乱成泥石流，或者磁盘经常报警 - 想让Agent帮忙整理，但不放心它直接删东西 -- 想给每一次整理留一份可审计的记录 +- 想给每一次整理和盘点留一份可审计的记录 **不适合** - 想要全自动、无人值守地清理 -- 想清系统缓存和大盘空间，那不是它管的事 -- Windows用户（目前macOS功能齐全，Linux只有基本功能） +- 只想清系统缓存和大盘空间，那不是它管的事 +- Windows第一次用留个心眼，代码测试过但没在真机上磨过，建议先只读看报告 -[![Repo](https://img.shields.io/badge/GitHub-carl--file--organizer-111827?style=flat-square&logo=github)](https://github.com/LearnPrompt/carl-file-organizer) -[![Install](https://img.shields.io/badge/Install-raw_SKILL.md-10B981?style=flat-square)](https://raw.githubusercontent.com/LearnPrompt/carl-file-organizer/main/skills/carl-file-organizer/SKILL.md) +```bash +npx skills add LearnPrompt/carl-skills --skill carl-file-organizer -g +``` + +[![Repo](https://img.shields.io/badge/GitHub-carl--skills-111827?style=flat-square&logo=github)](https://github.com/LearnPrompt/carl-skills) +[![Install](https://img.shields.io/badge/Install-skill_folder-10B981?style=flat-square)](./skills/ops/carl-file-organizer/SKILL.md) -→ [canonical repo](https://github.com/LearnPrompt/carl-file-organizer) · [raw SKILL.md](https://raw.githubusercontent.com/LearnPrompt/carl-file-organizer/main/skills/carl-file-organizer/SKILL.md) +→ [SKILL.md](./skills/ops/carl-file-organizer/SKILL.md) · [skill 目录](./skills/ops/carl-file-organizer/) diff --git a/registry.json b/registry.json index 9667823..bfb41d4 100644 --- a/registry.json +++ b/registry.json @@ -3,7 +3,7 @@ "description": "Real-world AI workflows for creators.", "homepage": "https://github.com/LearnPrompt/carl-skills", "license": "MIT", - "version": "0.7.0", + "version": "0.8.0", "updated": "2026-09-02", "model": "catalog-first", "install_logic": "Carl Skills is the catalog. Each entry declares either a raw_skill_url or a folder-aware install_mode/install_command; collection-native skills are maintained directly under this repository's skills directory.", @@ -309,13 +309,16 @@ "status": "active", "category": "ops", "catalog_anchor": "carl-file-organizer", - "summary": "先看后动的下载目录整理器:plan只读扫描出方案,网页勾选批准后才apply移动,删除默认进废纸篓。", - "canonical_repo": "https://github.com/LearnPrompt/carl-file-organizer", + "summary": "先看后动的文件整理与磁盘盘点，只读扫出方案，逐项讲人话，三色分级，网页上点头才动，删了默认进废纸篓，动过的能撤。", + "canonical_repo": "https://github.com/LearnPrompt/carl-skills", "canonical_branch": "main", - "canonical_path": "skills/carl-file-organizer/SKILL.md", + "canonical_path": "skills/ops/carl-file-organizer/SKILL.md", + "distribution": "collection-native", "installable": true, - "raw_skill_url": "https://raw.githubusercontent.com/LearnPrompt/carl-file-organizer/main/skills/carl-file-organizer/SKILL.md", - "source_commit": "fdfeb990343c2d81f5f7366bd26c55a37601a91f" + "install_mode": "skill-folder", + "install_command": "npx skills add LearnPrompt/carl-skills --skill carl-file-organizer -g", + "hermes_installable": false, + "hermes_skip_reason": "完整运行时含 scripts/assets/references，raw SKILL.md 装不全" }, { "id": "afu-llm-todo", diff --git a/skills/ops/carl-file-organizer/README.md b/skills/ops/carl-file-organizer/README.md new file mode 100644 index 0000000..b73ab73 --- /dev/null +++ b/skills/ops/carl-file-organizer/README.md @@ -0,0 +1,79 @@ +# 🗂️ 文件整理 carl-file-organizer + +> 整理文件夹和清磁盘这两件事，过去靠忍，或者靠一个看不懂你文件的软件，现在跟 Agent 说一句就够了。 + +随口跟 Agent 说一句「收拾一下 Downloads」或者「磁盘满了」，它先只读扫一遍，然后在浏览器里打开一份报告。就一份：磁盘总览和目录杂乱度并排放在最上面，接着是最大的五项，然后按绿黄红三个区往下排，每个区里先是能清掉的缓存和构建产物，再是要搬去哪儿的文件。每一项都写着这是什么、为什么这么判、动了会怎样。三色区下面是整理后预览，一张图告诉你点完确认之后这个目录长什么样，你勾掉哪项图上那项就变回原地不动。底下只有一个按钮，勾好一次点掉，每次都有二次确认，动过的能一键撤回。 + +## 它跟传统整理器和清理软件的区别 + +传统整理器按扩展名把东西搬走，回头你找不到那份合同去了哪。清理软件扫到一个 3.8G 的目录只会写一句「用户缓存，可删」，你不知道里面到底是什么，也不知道删了哪些网站要重新登录。 + +这个 skill 分两层。脚本负责量大小、算静置期、认压缩包配对和重复副本、查谁还在用这个文件，把八成能用规则说清楚的判断先做完。Agent 负责剩下两成，看过目录之后给每一项写三句人话，这是什么，为什么这么判，删了会怎样。方案先给你看，你点头它才动，删了默认进废纸篓，移动过的能原路放回。 + +## 三色分级是核心 + +三色分两层。目录那层告诉你这个文件夹现在有多乱，值不值得花时间。条目那层告诉你这一条你能按什么按钮。 + +| 目录杂乱度 | 意思 | +|---|---| +| 🟢 整洁 | 没几件散件，没有等你判断的东西 | +| 🟡 有点乱 | 要么量还不大，要么要判断的不多 | +| 🔴 泥石流 | 两条都不满足，该花点时间了 | + +| 条目 | 搬动那半 | 清理那半 | 能按什么 | +|---|---|---|---| +| 🟢 放心归位 / 可清 | 规则明确的移动、敏感命名隔离、可再生构建产物 | 纯缓存和临时文件，删了自动再生 | 勾上就走，永久删除要你自己额外开开关 | +| 🟡 你看一眼 | 待判断、成对压缩包、重复副本、冷存候选 | 含用户数据的目录、安装包、备份、agent 会话日志 | 只有打开所在位置和移到废纸篓，都可逆 | +| 🔴 别动 | 禁刀区、被进程占用、被别处引用、静置期没满 | 系统核心、虚拟机镜像、浏览器登录态、照片图库 | 只解释为什么不能动，最多打开所在位置 | + +颜色由规则算出来。Agent 看过目录之后只能改目录那层的颜色，条目的颜色一个字都改不了，因为按钮权限跟着它走。 + +## 铁律 + +全程只读，直到你在页面上点按钮并在浏览器弹窗里二次确认。Agent 永远不自己 rm、mv、trash，永远不加 `--allow-permanent-delete`，这个开关只能你自己在终端里加。本地服务只绑 127.0.0.1，随机端口加一次性 token，关掉终端就失效。不读文件内容，敏感命名只看名字。拿不准就黄。每次执行都写清单和审计，`undo` 按记录逆序放回。 + +## 怎么触发 + +``` +整理下载目录 +收拾一下 Downloads +下载文件夹太乱了 +帮我归档文件 +磁盘满了 +看看存储 +C 盘满了 +清理一下磁盘 +看下电脑空间 +organize my downloads +tidy this folder +disk is full +storage analysis +``` + +## 安装 + +```bash +npx skills add LearnPrompt/carl-skills --skill carl-file-organizer -g +``` + +零第三方依赖，Python 3.9 以上。装完 Agent 会读到 SKILL.md，脚本在 `scripts/` 下，`python3 scripts/organize.py --help` 能看到全部子命令。 + +## Windows + +回收站走系统的 `SHFileOperationW` 接口，带撤销标志，进回收站的东西能从回收站找回。Finder 标签在 Windows 上没有对应功能，直接跳过不报错。打开所在位置用资源管理器。没有 lsof 时占用检查老实标成未知。C 盘之外的盘符一起盘点，WinSxS、hiberfil、pagefile 这些系统大户只提示正规释放方式，不给删除按钮。macOS 上跑过真实目录和真实磁盘，Windows 的代码走过测试但没在真机上磨过，第一次用建议先只读看报告，留个心眼。 + +## 老师们 + +| 老师 | 教的是 | 在这里变成了什么 | +|---|---|---| +| [John Locke](https://en.wikipedia.org/wiki/Commonplace_book) | commonplace book，能检索比完美分类更重要 | 收件箱先接住一切，待判断不强行归类 | +| [Melvil Dewey](https://en.wikipedia.org/wiki/Dewey_Decimal_Classification) | 用编号分区 | 00/10/20/60/90 这套编号，只到三层就封顶 | +| [S. R. Ranganathan](https://en.wikipedia.org/wiki/S._R._Ranganathan) | 分面分类，一件东西可以有多个机器可读的属性 | `destination_key`、`tier`、`color` 各管一面 | +| Aby Warburg | 好邻居法则，相关的放在一起，哪怕分类学上它们属于不同类 | 压缩包和解压目录成对处理的直觉来源 | +| [Niklas Luhmann](https://en.wikipedia.org/wiki/Niklas_Luhmann) | 卡片盒的价值在关系和理由 | 每条动作都带一句理由，Agent 再补三句人话 | +| [Paul Otlet](https://en.wikipedia.org/wiki/Paul_Otlet) | 给人读的卡片配一份给机器读的索引 | `report.html` 给人，`plan.json` 给程序和 Agent | +| [David Allen](https://en.wikipedia.org/wiki/Getting_Things_Done) | GTD 的 capture、clarify、review、act 四步 | 就是 plan、待判断、review、apply | +| [Andrej Karpathy](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) | LLM Wiki，面向 Agent 可读的知识库分区思路 | `names` 和 `destination_key` 这套机器优先的分区键 | +| [KKKKhazix / storage-analyzer](https://github.com/KKKKhazix/khazix-skills/tree/main/storage-analyzer) | 网页一键处置与三色报告 | 盘点入口的三色分级和页面上直接点按钮的体验 | + +署名 LearnPrompt / 卡尔的AI沃茨。 diff --git a/skills/ops/carl-file-organizer/SKILL.md b/skills/ops/carl-file-organizer/SKILL.md new file mode 100644 index 0000000..bf4d4c2 --- /dev/null +++ b/skills/ops/carl-file-organizer/SKILL.md @@ -0,0 +1,171 @@ +--- +name: carl-file-organizer +description: 文件整理与磁盘盘点。整理下载目录或任意散乱文件夹时，先只读扫出方案，逐项写人话说明，用户在网页上点头才动，动过的能撤。整机盘点时只读扫出空间去向，按绿黄红三色分级，绿的可清，黄的你看一眼，红的别动。用户说「整理下载目录」「收拾一下 Downloads」「下载文件夹太乱了」「帮我归档文件」「整理一下这个文件夹」「磁盘满了」「看看存储」「C 盘满了」「清理一下磁盘」「看下电脑空间」「organize my downloads」「tidy this folder」「clean up my downloads」「disk is full」「storage analysis」「free up space」时使用。全程只读，任何移动、进废纸篓或删除都由用户在页面上点按钮并二次确认，Agent 永远不自己动文件。 +--- + +# 文件整理 carl-file-organizer + +你在这套工具里是解说员和操作员，拿主意的是用户。脚本负责量、算、定色，你负责把每一项讲成人话，然后把报告交给用户，等他在页面上一项一项点。你自己永远不 rm、不 mv、不 trash，永远不给命令加 `--allow-permanent-delete`。 + +下面 `` 指这份 SKILL.md 所在的目录。所有命令用 `python3 /scripts/...` 的形式跑，零依赖，Python 3.9 以上就行。 + +不管用户是嫌目录乱还是嫌磁盘满，流程只有一条，走完是一份报告、一个入口、一份批准文件。 + +``` +scan → 你写 notes.json 与 analysis.json → report → 用户在页面上处理或导出 → apply --dry-run → 用户说执行才真跑 +``` + +两个场景的差别只在扫不扫盘点。用户说的是目录乱、文件多、想归档，加 `--no-storage`，报告里就只有整理这一半。用户说的是磁盘满、空间紧、想清缓存，默认就扫，两半都在。用户只是问某个文件该放哪，不用跑工具，直接答。 + +## 第一步，只读扫描 + +```bash +python3 /scripts/organize.py scan ~/Downloads --lang zh --budget-seconds 60 +``` + +用户没说目录就用 `~/Downloads`，不要自己猜别的路径。用户说英文目录名就把 `--lang` 换成 `en`。目录里已经有一套分区时工具会跟着已有的那套走，不必再传。只整理不盘点就加 `--no-storage`。 + +这一步不动任何文件。产物都落在目标目录下的管理目录里，中文是 `<目录>/00_下载目录管理/`，英文是 `<目录>/00_File_Organizer/`：`plan.json` 是整理这一半，`storage-scan.json` 是盘点这一半。后面你写的 `notes.json`、`analysis.json`，还有渲染出来的 `report.html` 和执行留下的清单，也全在这个目录里。终端里那句「本次只读扫描，没有移动或删除任何文件」原样带给用户。 + +盘点默认六十秒预算，超时的条目标成 `partial`，体积是下限。macOS 上隐私保护会挡住一批目录，一次出现上百个读不到很正常，这句话要写给用户，因为总量因此偏小。`storage-scan.json` 里全是这台机器的绝对路径，它只给本机自己用，别当结论发出去。用户要把结果给别人看，发 `report.html`，那份渲染时已经把家目录换成 `$HOME` 了。 + +## 第二步，读扫描结果，写人话 + +### 读 plan.json，只报数不拍板 + +先看 `summary`，按 `by_color` 报三色各有多少条。绿的是规则已经定好去向的移动、敏感命名隔离和可再生产物。黄的是待判断、成对压缩包、重复副本和冷存候选。红的是这轮碰不了的，禁刀区、被进程占用、被别处引用、静置期没满、常驻白名单都在里面。报红的数目时顺便说清这一轮不会动它们。 + +再看 `mess`，这是目录本身有多乱。`score` 决定进度条长短，`color` 决定它是整洁、有点乱还是泥石流，`counts` 里五个数就是理由。你要是真看过这个目录觉得数字判错了，可以在 notes 里改目录颜色并给一句话。条目的颜色一个字都改不了。 + +`groups` 是几选一的选择题，压缩包和解压目录成对、内容相同的重复副本、构建产物各成一组。每组带 `options`，每个选项自带一份 `action_ids`。你只负责把选项讲清楚，用户在页面上单选。永远不要自己从 `actions` 里挑单条去拼一个组的答案。 + +`actions` 里 `reroutable` 为 true 的那些才是要你判断分区的，也就是未知扩展名、顶层目录、派生副本这三类进了待判断区的移动。其余的一律照规则转述，不建议改动。 + +### 对待判断项按名字判断 + +只看文件名、扩展名、大小、修改时间。不打开文件，不读内容。名字里带密钥、密码、令牌、证书、身份证件这类词的，工具已经按敏感规则处理过了，你不要再往别处搬，拿不准就让它留在待判断区。 + +判断结果写成一句话放进这一条的 `why` 里，去向用 `plan.json` 的 `names` 里已有的分区名，比如「名字像一次 SQL 导出，建议改去 10_工作区/数据库与查询/SQL」。用户在页面的待判断下拉里改去向，页面会把改动写进 `overrides`。用户在聊天里直接确认并让你写批准文件时，你才把 `{action_id, destination_key}` 写进 `overrides`，`destination_key` 只能取自 `names`，不许写字面路径。 + +### 写 notes.json + +给每一条动作和每一个组写三句话。`what` 这是什么，`why` 为什么这么判，`if_removed` 动了会怎样。一两句人话，写给一个不知道这文件是什么的人看，不要复述规则文案。整个目录再写一句 `folder_line`，`mess.line` 写你对杂乱度的一句判断。 + +```json +{ + "folder_line": "基本是上周做提案时下载的东西，外加两个早该归档的老项目", + "mess": {"color": "yellow", "line": "看着乱，其实一半是同一批素材"}, + "actions": { + "9c1f0a7b2d3e4f55": { + "what": "第三方发来的合同扫描件", + "why": "PDF 归文档资料，名字里没有敏感词", + "if_removed": "只是搬家，邮箱里还有原件" + } + }, + "groups": { + "pair-7d2c1e9f": { + "what": "Archive.zip 和它解压出来的目录", + "why": "同名成对，目录里的东西齐全", + "if_removed": "留目录扔压缩包最省事，压缩包进废纸篓还能捞" + } + } +} +``` + +notes 只收这三个键，多写的键会被丢掉并记一条提醒。写不出来的项可以空着，页面会退回规则文案。把这份存成管理目录里的 `notes.json`，`report` 默认就去那儿找。 + +### 读 storage-scan.json，定色，写 analysis.json + +跑了盘点才有这一步，加了 `--no-storage` 就跳过。 + +读扫描结果，对照 `/references/macos.md` 或 `windows.md` 给每一项定色。绿是纯缓存和构建产物，删了会自己长回来。黄是里面有用户数据或者只是暂时没用的，Application Support、Containers、安装包、Backups、agent 的会话日志都算，要用户自己看一眼。红是碰不得的，虚拟机镜像、浏览器的用户数据、钥匙串、邮件和信息的主数据、照片图库、iCloud 本地副本、活跃仓库的 `.git`。脚本给的 `suggested_color` 是起点，你可以改，改了要在 `what` 里说一句为什么。 + +每一项写 `what` 这是什么，`if_removed` 删了会怎样，`disposal` 建议怎么处置，`restore` 怎么回来。写给一个不知道 ms-playwright 是什么的人看，后果说具体，不写没事。 + +`trash_paths` 只填你核实过的具体安全子路径。写 `$HOME/Library/Caches` 这种父目录不合格，要写到具体那个缓存目录。红项的 `trash_paths` 必须是空数组。路径必须落在家目录之内，不能有软链段，不能命中禁刀区。 + +`overview` 里 `headline` 一句话说清这台机器什么状况，第一刀该切哪。`priority` 列现在就能动的，按收益排，这几条会跟整理那半的建议合成报告顶上的一个编号清单。`long_term` 列适合冷存、外置盘、应用内清理的。Windows 上 WinSxS、hiberfil、pagefile 这些只写不扫的东西放进 `long_term`，给正规释放方式，别给删除按钮。 + +把这些写成管理目录里的 `analysis.json`，字段表在 `/docs/storage-schema.md`。路径全部来自扫描结果，不许自己编一条。 + +## 第三步，生成一份报告 + +```bash +python3 /scripts/organize.py report <管理目录> +``` + +一份 `report.html` 写在同一个目录里，清理和搬动在同一页，按绿黄红三个区排，每个区里先清理后搬动。顶上是磁盘条加杂乱度，右边是先做什么，中间是最大的五项，三色区下面是整理后预览，一张图告诉用户搬完长什么样，勾选变了图也跟着变。底下只有一个按钮。 + +告诉用户报告路径，让他双击打开。想在页面上直接点按钮处置，就起本地服务，只绑 127.0.0.1 带一次性 token，关掉终端就失效。 + +```bash +python3 /scripts/organize.py report <管理目录> --serve +``` + +## 第四步，用户批准，你预演 + +用户在页面上勾选、单选、改去向。静态页面底下那个按钮导出一份 `carl-file-organizer-decisions.json`，落在浏览器的下载目录，整理和清理都在里面。serve 模式下按钮直接处置，先搬后清，每次点击浏览器都弹二次确认，这条线不需要你再跑 apply。 + +拿到那份决定清单先预演，把输出原样贴给用户。 + +```bash +python3 /scripts/organize.py apply ~/Downloads/carl-file-organizer-decisions.json --dry-run +``` + +先跑搬动这一段，再跑清理这一段，两段各自写清单和审计，都落在同一个管理目录里。 + +用户在聊天里明确说了执行、apply、动手、可以了，你才去掉 `--dry-run` 跑真的。用户说整理一下、看看、先别动、再想想，都算没授权，继续停在预演。工具拒绝执行或者把某一批标成 refused，把那句话原样转告，不要绕路，不要换个命令重试。 + +绿项可以进废纸篓，也可以永久删除，永久删除必须命令上带 `--allow-permanent-delete`，这个开关只能用户自己加。黄项只能进废纸篓。红项只能打开所在位置。执行前会复查占用。 + +## 事后 + +`status` 看最后一次整理的时间和现在是否又乱了，`undo` 按执行记录逆序把移动过的原路放回，`clear-tags` 清掉页面复查用的 Finder 标签。 + +```bash +python3 /scripts/organize.py status ~/Downloads +python3 /scripts/organize.py undo <管理目录>/audit.jsonl +python3 /scripts/organize.py clear-tags ~/Downloads +``` + +## 拆开跑的那套命令 + +`scan / report / apply` 是把下面这些包起来的三步。要单独跑某一段时才用它们，平时不用提。 + +```bash +python3 /scripts/organize.py plan ~/Downloads --lang zh +python3 /scripts/organize.py build <管理目录>/plan.json --notes notes.json --report +python3 /scripts/organize.py review <管理目录>/plan.json --serve +python3 /scripts/storage_scan.py --out storage-scan.json --budget-seconds 60 +python3 /scripts/organize.py storage-report --serve +python3 /scripts/organize.py dispose --dry-run +``` + +它们出的是只有一半的报告和旧格式的批准文件，`apply` 照样认得。 + +## 铁律 + +全程只读，直到用户在页面上点按钮并在浏览器弹窗里二次确认。你永远不自己 rm、mv、trash，永远不用 osascript 或别的方式绕过工具去动文件。 + +永远不给任何命令加 `--allow-permanent-delete`。这个开关只能由用户自己在终端里加。 + +不读 secrets、密钥文件、浏览器数据库、聊天正文。敏感命名只看名字。 + +拿不准就黄。宁可让用户多看一眼，也不替他猜。 + +工具拒绝的原样转告。不要解释成别的意思，不要换条路再试。 + +用户说「整理一下」「看看」「清一清」都只是让你出报告，算不上授权执行。执行只认用户在聊天里说执行、apply、动手、可以了，或者用户自己在页面上点按钮。 + +## 回报格式 + +每一轮结束按这个说完就停。 + +``` +这一轮做到了哪一步 只读盘点 / 已预演 / 已执行（三选一） +动了多少 移动 N 项，进废纸篓 N 项，永久删除 N 项 +跳过多少 跳过 N 项、被拒绝 N 项，各自为什么 +没碰的范围 禁刀区、静置期没到的、被占用或被引用的 +报告和清单 report.html、plan.json 与 analysis.json、decisions.json、audit.jsonl 的路径，都在同一个管理目录里 +一句提醒 undo 能把这轮移动的原路放回，clear-tags 能清掉复查用的 Finder 标签 +``` diff --git a/skills/ops/carl-file-organizer/agents/openai.yaml b/skills/ops/carl-file-organizer/agents/openai.yaml new file mode 100644 index 0000000..95595a7 --- /dev/null +++ b/skills/ops/carl-file-organizer/agents/openai.yaml @@ -0,0 +1,3 @@ +interface: + display_name: "文件整理 · Carl File Organizer" + short_description: "先只读扫出方案，逐项讲人话，你点头才动，动过的能撤" diff --git a/skills/ops/carl-file-organizer/assets/report_template.html b/skills/ops/carl-file-organizer/assets/report_template.html new file mode 100644 index 0000000..95ffec0 --- /dev/null +++ b/skills/ops/carl-file-organizer/assets/report_template.html @@ -0,0 +1,1068 @@ + + + + + + +__REPORT_TITLE__ + + + + + +__REPORT_BODY__ + + + + + + + diff --git a/skills/ops/carl-file-organizer/docs/plan-schema.md b/skills/ops/carl-file-organizer/docs/plan-schema.md new file mode 100644 index 0000000..6e8f2b0 --- /dev/null +++ b/skills/ops/carl-file-organizer/docs/plan-schema.md @@ -0,0 +1,295 @@ +# plan.json 契约（schema_version 2） + +审核页、本地服务、执行器、undo 和 Agent 都只读这一个文件，谁也不去猜结构。 +样例在仓库根的 `tests/fixtures/carl_file_organizer/plan-v2-sample.json`， +本文每一节都能在那份样例里找到对应的行； +`tests/test_cfo_fixture_contract.py` 按本文的约束逐条断言，改结构就要一起改这三处。 + +一句话记住整份文件：`actions` 是候选清单，`groups` 是几选一的题目， +`approved_action_ids` 是唯一的批准名单，`overrides` 是唯一的改道通道。 + +## 顶层字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `schema_version` | int | 固定 2。执行器见到别的值直接拒绝。 | +| `generator` | object | `{name, version}`，version 取自 `carl_file_organizer.__version__`。 | +| `created_at` | string | 计划时刻，带时区。跟随 `now`，所以同一个 `--now` 跑两遍出来的计划是一样的。 | +| `generated_at` | string | 真实生成时刻（墙钟），带时区。排查问题时看这一个。 | +| `now` | string | 静置期的计算基准。apply 阶段不重算，复现问题时用 `--now` 固定它。 | +| `lang` | string | `zh` 或 `en`，本次输出用的语言。 | +| `lang_source` | string | `flag` / `dotfile` / `env` / `existing-dirs` / `apple-languages` / `posix-locale` / `default`。 | +| `profile` | object | `{name, source, dotfile, hash}`。`source` 是 `builtin` 或 `builtin+dotfile`，`hash` 是合并后 profile 的短摘要，apply 时对不上只提醒不中断。 | +| `source_root` / `source_root_portable` | string | 绝对路径只出现在 plan.json 里；清单、日志、审计一律写 `$HOME/...` 形式。 | +| `managed_dir` / `managed_dir_portable` | string | 可见的管理目录，中文是 `00_下载目录管理`，英文是 `00_File_Organizer`。plan.json 和 report.html 都落在这里。 | +| `names` | object | 本次解析出来的 key → 相对路径，例如 `library.docs.pdf` → `20_知识库/文档资料/PDF`。页面和 Agent 直接显示这里的值，不必回头读 profile。 | +| `capabilities` | object | `{trash_backend, finder_tags, permanent_delete_offered}`。`trash_backend` 是 `finder`（macOS）/ `win32`（Windows 回收站）/ `gio`（Linux）/ `none`，为 none 时废纸篓选项不可批准。 | +| `scan_policy` | object | 本次扫描的边界与上限，写下来是为了让人知道哪些地方根本没看。除设计里那几项，还有 `allow_referenced` 与 `skipped{hidden,symlinks,partitions,artifacts}` 两个计数用的字段。 | +| `disk` | object | 卷容量与目标占用，移动同卷释放 0 字节这句话也写在里面。 | +| `mess` | object | 这个目录现在有多乱，见下。 | +| `summary` | object | 见下。 | +| `pinned` | array | 本次生效的常驻白名单，名字或 glob。 | +| `actions` | array | 候选动作，见下。 | +| `groups` | array | 成组的选择题，见下。 | +| `notes` | object 或 null | Agent 写的人话说明，见下。生成时是 null。 | +| `approved_action_ids` | array | 页面、服务和 Agent 唯一需要填的字段。 | +| `overrides` | array | 待判断项的改道，见下。 | +| `approved_at` | string 或 null | 批准时刻。 | +| `approved_by` | string 或 null | `html-static` / `serve` / `agent:<名字>`。 | + +`summary` 里 `entries` / `files` / `dirs` / `bytes` 数的是扫描到的顶层条目， +`by_tier` 和 `by_kind` 数的是 action 条数（同一个文件可能同时有移动和处置两条候选，所以两组数不相等是正常的）。 +`potential_reclaimable_bytes` 按 subject 去重后累加，只是个信息量，不代表建议； +`proposed_reclaimed_bytes` 在生成时永远是 0，执行完才按实际写进审计。 + +`by_color` 是 `{green|yellow|red: {actions, bytes}}`。`actions` 按条数算， +所有颜色加起来等于 `len(actions)`；`bytes` 是这些 action 的 `size_bytes` 之和， +跟 `by_tier` 一样会重复计入同一个文件（一个文件可以同时有一条绿色移动和一条黄色处置）， +它描述的是形状不是总量。 + +## mess：这个目录有多乱 + +```json +"mess": { + "score": 59, + "color": "yellow", + "color_source": "rule", + "counts": {"loose": 12, "aging": 1, "pending": 2, "groups": 3, "top_dirs": 3}, + "reason": {"zh": "...", "en": "..."} +} +``` + +| 字段 | 说明 | +|---|---| +| `score` | 0 到 100，越大越乱。只决定进度条长度，不决定颜色。 | +| `color` | `green` / `yellow` / `red`。 | +| `color_source` | `rule` 或 `notes`，谁定的这个颜色。 | +| `rule_color` | 只在被 notes 覆盖后出现，保存规则原本算出来的颜色。 | +| `line` | 只在 notes 给了一句话时出现。 | +| `counts` | `loose` 顶层散件数、`aging` 静置中、`pending` 待判断（按 subject 去重）、`groups` 组数、`top_dirs` 顶层目录数。 | +| `reason` | `{zh, en}`，规则给的一句话。 | + +阈值与分数公式在 `references/tiers.md`，代码里对应 `planner.MESS_THRESHOLDS`。 +Agent 可以在 notes.json 里覆盖 `color`，覆盖后 `color_source` 变成 `notes`， +规则的答案退到 `rule_color`。 + +## action 字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `id` | string | `sha256(subject_id \0 kind \0 destination_portable)[:16]`。同一个文件的移动、废纸篓、永久删除是三条不同 id。 | +| `subject_id` | string | `sha256(realpath \0 subject_kind \0 size \0 mtime_ns)[:16]`。执行前实时重算，对不上就跳过。 | +| `kind` | string | `move` / `trash` / `delete` / `hold`。 | +| `subject_kind` | string | `file` 或 `dir`。目录整个搬，不拆内部结构。 | +| `source` / `source_portable` | string | 源路径。 | +| `filename` | string | 条目名，目录移动后保持原名。 | +| `destination` / `destination_portable` | string 或 null | 目标路径；`trash` / `delete` / `hold` 为 null。 | +| `destination_key` | string 或 null | `names` 里的 key，页面按 key 显示分区名。 | +| `size_bytes` / `modified_ns` / `age_hours` | number | 体积、修改时间、已静置小时数。 | +| `rule` | string | 判定依据，取值见下表。 | +| `reason` | object | `{zh, en}`，一句人话说清为什么。 | +| `detail` | object 或 null | `{zh, en}`，补充证据，例如静置了多久。 | +| `confidence` | number | 0 到 1。未知扩展名是 0.55，扩展名规则命中是 0.98。 | +| `tier` | string | `routine` / `sensitive` / `regenerable` / `duplicate` / `cold` / `aging` / `pinned` / `forbidden` / `in_use` / `referenced`。 | +| `group_id` | string 或 null | 属于某个组时填组 id。 | +| `approvable` | bool | `hold` 恒为 false。执行器见到不可批准的 id 出现在名单里直接抛错。 | +| `reroutable` | bool | 只有 rule 属于 `unknown-ext` / `dir:pending` / `dir:derivative` 的移动才为 true，也只有它们能被 `overrides` 改道。 | +| `default_selected` | bool | 生成时一律 false，勾选由人来做。 | +| `requires` | array | `delete` 是 `["allow-permanent-delete"]`。 | +| `reclaims_bytes` | number | 移动是 0，废纸篓和永久删除是条目体积。 | +| `restore_method` | object 或 null | `{zh, en}`。移动给一条可直接执行的 mv，废纸篓写放回原处，永久删除写清楚不可恢复。 | +| `tags` | array | 扫描到的非临时 Finder 标签。 | +| `hints` | array | 只提示不改判定，例如 `git-repo`、`derivative-copy:-worktree-`。 | +| `color` | string | `green` / `yellow` / `red`。页面的按钮跟着它走，所以它由 kind、tier 与 reroutable 三个字段算出来，永远不手写。判定表在 `references/tiers.md`。 | +| `hold_reason` | object 或 null | hold 时必填，`{zh, en}`，静置类还带 `ready_at`，禁刀区带 `label`，常驻带 `pattern`（命中的那条白名单）。 | +| `guard` | object | 保底层的取证结果，见下。 | +| `dir_stats` | object 或 null | 目录才有，`{files, bytes, truncated}`，`truncated` 为 true 表示统计到上限就停了。 | + +### rule 取值 + +`ext:<后缀>`、`sensitive:exact` / `sensitive:suffix` / `sensitive:marker` / `sensitive:manual`、 +`dir:pending`、`dir:derivative`、`pair`、`tag:<标签>`、`name:<规则 id>`、`conflict`、 +`unknown-ext`、`dup:sha256`、`dup:name`、`regenerable:<目录名>`、`cold:large`、`cold:installer`、 +`hold:aging`、`hold:pinned`、`hold:forbidden`、`hold:in_use`、`hold:referenced`。 + +`conflict` 是目标位置已经被占了，条目转进重复待确认区并带上时间戳改名。 +时间戳插在扩展名前面，所以改完还是能双击打开： +`quarterly-report.pdf` → `00_收件箱/重复待确认/quarterly-report-2026-09-02_12-00-00.pdf`， +`release.tar.gz` → `release-2026-09-02_12-00-00.tar.gz`。 +目录和没有扩展名的条目保持 `<原名>-<时间戳>`。 + +### guard 字段 + +保底层查的是谁还在用这个路径，命中就把条目变成 hold，让人自己去改引用。 + +| 键 | 内容 | +|---|---| +| `open_by` | `[{command, pid}]`，来自 lsof。命中即 tier `in_use`，不可降级。 | +| `referenced_in` | `[{file, line, form}]`，来自 launchd、cron、shell 配置和用户自己加的引用文件。命中即 tier `referenced`，`plan --allow-referenced` 可降级成黄字提示。 | +| `incoming_symlinks` | 指进这个条目的软链路径。 | +| `shape` | 目录形态，例如 `git-dir`、`git-worktree`、`venv`、`dotenv`、`node-project`；文件是 `executable`、`shebang`。worktree 直接进 hold，其余只加提示。 | + +除这四个必填键，`guard` 还可能带两个可选键：`open_by_status` 为 `unknown` 表示这台机器上没有 lsof 或者 lsof 超时了， +「没人开着它」这句话只是查不出来而不是查过；`uncommitted` 是 git 仓库未提交的条目数。读的人按有则用、无则忽略处理。 + +lsof 报出来的 Spotlight 一类只读守护进程（`mdworker`、`quicklookd`、`fseventsd` 等）不算占用。 +刚下载完的文件几秒内就会被索引，把它们算进去等于整个下载目录永远动不了。 + +## group 字段 + +| 字段 | 说明 | +|---|---| +| `group_id` | 组 id，成员 action 的 `group_id` 指向它。 | +| `kind` | `pair`（压缩包与解压目录）、`duplicate`（重复文件）、`regenerable`（构建产物）。 | +| `tier` | pair 与 duplicate 都是 `duplicate`，regenerable 是 `regenerable`。 | +| `stem` | 归组用的词干。 | +| `evidence` | pair 是 `name`；duplicate 是 `sha256` 或 `name-pattern`；regenerable 是 `basename`。 | +| `sha256` | 只有内容验证过的重复组才有值。 | +| `members` | 成员的 `subject_id` 列表。 | +| `member_labels` | subject_id → 给人看的名字。 | +| `reason` | `{zh, en}`。 | +| `options` | 选项数组，见下。 | +| `color` | `regenerable` 组是 `green`，`pair` 与 `duplicate` 组是 `yellow`。组是一道选择题，不管选哪个都会有东西被处置，所以默认要人看一眼；构建产物是例外，它的两个选项本身都是绿的，问一遍就够了，不用再要一次确认。 | +| `default_choice` | 单选框的默认位置。压缩包配对默认留目录；名字像副本但内容没比对过的重复组默认都保留。 | +| `potential_bytes` | 采纳默认选项能腾出的字节数。 | + +`options[]` 里每项是 `{key, label{zh,en}, action_ids[], requires[]}`。 +`action_ids` 是组选择映射到批准名单的唯一通道，页面和 Agent 都不许自己推导； +里面出现的每个 id 都在 `actions` 里，而且 `approvable` 一定为 true。 +只要选项里包含永久删除，`requires` 就带 `allow-permanent-delete`。 + +压缩包配对给四个选项：都保留、留目录、留压缩包、留目录并永久删除压缩包。 +重复组给每个成员一个留它的选项外加一个都保留。 +构建产物组只有全部进废纸篓和全部永久删除两项，不选就等于保留。 + +## notes：Agent 写的人话 + +规则给的理由是每条规则一句固定话，读起来像通知。真正有用的是「这是什么、 +为什么在这儿、删了会怎样」，那要一个看过这个目录的 Agent 来写。 +notes 就是这一层，它只加文字，不改任何决策。 + +Agent 写一份 notes.json： + +```json +{ + "folder_line": "这个目录基本是上周做提案时下载的东西", + "mess": {"color": "yellow", "line": "看着乱，其实一半是同一批素材"}, + "actions": { + "0112233445566a7b": { + "what": "第三方给的合同 PDF 扫描件", + "why": "扩展名规则命中，归到文档资料", + "if_removed": "邮箱里还有原件，丢了能要回来" + } + }, + "groups": { + "dup-3ba91c04": {"what": "同一份素材导了两次", "why": "内容哈希一致", "if_removed": "留一份就够"} + } +} +``` + +然后跑： + +```bash +python3 scripts/organize.py build --notes notes.json +``` + +合并结果写回 `plan.notes`： + +| 字段 | 说明 | +|---|---| +| `schema` | 固定 1。 | +| `folder_line` | 整个目录一句话，或 null。 | +| `mess` | `{color?, line?}`，清理过的那份。 | +| `actions` | `action_id` → `{what?, why?, if_removed?}`，只留这三个键。 | +| `groups` | `group_id` → 同上。 | +| `warnings` | 合并时发现的问题，每条一句话。 | + +规矩就三条： + +1. **只收三个句子。** `what`、`why`、`if_removed` 之外的键一律丢掉并记一条 warning， + 所以 notes 里写 `"approvable": true` 或 `"destination": "..."` 不会有任何效果。 +2. **不认识的 id 只提醒。** 上一次扫描留下的 notes 拿来配这次的计划，多出来的条目 + 会被跳过并记一条 warning，不会让整份报告生不出来。 +3. **唯一能改的是目录颜色。** `mess.color` 会覆盖 `plan.mess.color`，规则的答案退到 + `rule_color`，`color_source` 记成 `notes`。条目的 `color` 一个字都不能改， + 因为页面的按钮跟着它走，那是安全边界。 + +单句超过 600 字会被截断，不会被丢掉。合并两次是替换不是叠加， +改完一句重跑一遍不会留下上一版。 + +Python 里直接用： + +```python +from carl_file_organizer.notes import merge_notes +plan = merge_notes(plan, notes) # 返回新的一份，原来的不动 +``` + +报告怎么渲染是 `build_report.py` 的事，这里只保证 plan.json 里有一份干净的 `notes`。 +没有 notes 时报告退回规则文案，一样能出。 + +## overrides + +```json +{"action_id": "0112233445566a7b", "destination_key": "work.db.sql"} +``` + +只对 `reroutable` 为 true 的 action 生效，`destination_key` 必须是 `names` 里已有的 key。 +执行器拿到 override 后自己重算目标路径并校验仍在 root 之内，任何写字面路径的尝试都会被拒。 +待判断区的下拉框和 Agent 的语义判断都走这一条路，别的改道方式一律没有。 + +## report.html 与绝对路径 + +plan.json 留着绝对路径，report.html 一个都不留。渲染前整份计划先过一遍 +`paths.strip_absolute()`：`source_root`、`managed_dir`、每条 action 的 `source` 与 +`destination` 这四类字段直接删掉，只留 `_portable` 双胞胎；`guard.referenced_in[].file`、 +`guard.referenced_in[].form` 与 `guard.incoming_symlinks` 都改写成 `$HOME` 形式；剩下任何 +还带着家目录前缀的字符串一律一起改写，没有例外，免得以后新加的字段又把用户名漏出去。 + +这条规则的理由很实在：审核页是能直接发给别人看的一页 HTML，绝对路径里带着账号名。 +serve 模式的 `/api/plan` 走同一条路，服务进程内存里的计划仍然是完整的，`/api/apply` +用的是内存那份，不受影响。 + +页面导出的 approved.json 因此也只有 `$HOME` 形式。apply 一侧用 +`paths.hydrate_absolute()` 按当前机器的家目录把四类字段展开回来，再做原有的全部校验， +所以 id 重算、越界检查、篡改检测都不用知道文件是从哪来的。 + +落盘的 token 只有 `$HOME` 一种，Windows 上也是。计划和报告是会跨机器传的东西， +一种拼法就只有一个解析器和一条隐私检查（整份文档一次 `startswith`）。 +`paths.expand_portable()` 两种都认，`%USERPROFILE%\Downloads` 和 +`$HOME/Downloads` 落到同一个地方；Windows 上给人看的时候由 +`paths.display_portable()` 换成 `%USERPROFILE%\...`，那只是渲染，不进文件。 + +## approved.json 与 plan.json 的关系 + +approved.json 就是 plan.json 本身，只多填三个字段：`approved_action_ids` 填上勾选的 id， +`overrides` 填上改道，`approved_at` 与 `approved_by` 记下谁在什么时候批的。 +`notes` 可以带着，执行器完全不读它。 +其余字段一个都不能改，执行器会按 `id` 重算并比对，改过 destination 的文件会被当成篡改直接拒绝。 + +执行前的校验按顺序是：schema 必须为 2；root 存在且不是软链； +批准的 id 必须都在已知清单里；同一个 subject 只能批准一条； +出现不可批准的 id 直接抛错；名单里有永久删除而命令行没加 `--allow-permanent-delete` 时整批拒绝，不是跳过。 + +apply 成功之后，用户传进来的 approved.json 会被复制一份到管理目录，叫 `approved-<时间戳>.json`。 +浏览器通常把导出文件下载到下载目录根部，不收走的话下次扫描就会被当成散件。 + +## 审计记录 + +`managed_dir/audit.jsonl` 每行一条，路径一律 `$HOME` 形式。 + +| 字段 | 说明 | +|---|---| +| `timestamp` | 这一条执行完的时刻。 | +| `run_id` | 一次 apply 的批次号，形如 `2026-09-02_10-20-11`。 | +| `action_id` / `subject_id` | 对应 plan 里的两个 id。 | +| `kind` | `move` / `trash` / `delete`。 | +| `source` / `destination` | `$HOME` 形式。 | +| `status` | `moved` / `trashed` / `deleted` / `dry-run` / `skipped` / `conflict` / `refused` / `failed`。 | +| `detail` | 状态的一句话解释。 | +| `size_bytes` | 条目体积。 | +| `manifest` | 这一批对应的 TSV 清单路径。 | +| `tagged` | 有没有成功打上临时标签。打标签失败只提醒，不影响这条动作的状态。 | +| `trash_backend` | 本次用的废纸篓后端。 | + +崩溃时以 audit 为准。清单会先落一行 PLANNED 再重写终态， +中途断电就停在 PLANNED，undo 和 status 都只信 audit，清单留给人读和查最后整理时间。 diff --git a/skills/ops/carl-file-organizer/docs/review-page.md b/skills/ops/carl-file-organizer/docs/review-page.md new file mode 100644 index 0000000..18035b5 --- /dev/null +++ b/skills/ops/carl-file-organizer/docs/review-page.md @@ -0,0 +1,153 @@ +# 报告页视觉规范（report.html，v0.3 三色版） + +报告页是整个 skill 唯一给人看的界面。一份模板三种形态，由 `data-kind` 决定：`combined` 是人真正看到的那一份，清理和搬动写在同一页，末尾带整理后预览；`organize` 只有整理这一半，`storage` 只有盘点这一半，两者是分步命令留下的旧形态。页面不加载任何外部资源，双击打开 file:// 也是零请求；同一份模板在 static 与 serve 两种模式下长得一样，只有底栏按钮的动作不同。 + +三个文件对应本文：`assets/report_template.html` 是模板（样式、脚本、顶部的 TEXT 文案表），`scripts/build_report.py` 负责把数据脱敏后渲染成正文并注入模板，仓库根 `tests/test_cfo_report_template.py` 逐条断言本文的约束。改样式先改模板，改文案只改 TEXT。 + +## combined：一份报告，一个入口，一份批准文件 + +`render(data)` 收到 `{"plan": plan, "analysis": analysis}` 就渲染这一份。两半都在最好，只给一半也能渲染，页面上就少一块。两半分别脱敏，因为整理那半能从 `source_root` 反推自己的家目录，盘点那半没有这一对字段，只能退回当前账号的家目录。 + +``` +顶栏 目录 · 机器 · 扫描时间 · 模式 +页面标题 + 一句引导 +总览双卡 左：磁盘条（盘点）+ 杂乱度大字与三色分段（整理）+ 一句话 + 右：先做什么，analysis.overview.priority 与整理三条建议合成一个编号清单 +最大的五项 只有带 analysis 时才有 +三色三个区 green → yellow → red，每区先「清理」后「搬动」两个小标题 +整理后预览 标题「整理后长这样」，下面是 preview.py 渲染的自足片段 +长期建议 盘点的 overview.long_term，加整理的静置到期时间与固定说明 +底栏 一个主按钮 +``` + +小标题是 `

`，空的那一组不渲染标题也不占位。三个区永远都在，整区空了才写一句本组没有条目。 + +预览片段来自 `carl_file_organizer.preview`：`build_graph(plan, analysis)` 出图，`render_preview_html(graph, lang=, height=720)` 出片段，整块包在 `
` 里。片段自带样式和脚本，不加载任何外部资源，全页只出现一次。页面上勾选或取消任何一项时，`refresh()` 会 `window.dispatchEvent(new CustomEvent('cfo:selection', {detail: {action_ids, item_ids}}))`，预览里监听这个事件，把没勾中的「要搬」节点画成空心，并改写面板上的要搬与留下计数。首次渲染不发这个事件，所以打开页面看到的仍是完整的整理后状态。 + +## combined 的批准文件 + +static 模式底栏只有一个按钮，导出一份 `carl-file-organizer-decisions.json`： + +```json +{ + "schema": "carl-file-organizer/decisions", + "schema_version": 1, + "decided_at": "2026-09-02T12:00:00+09:00", + "decided_by": "html-static", + "plan": { + "approved_action_ids": ["9c1f0a7b2d3e4f55"], + "overrides": [{"action_id": "0112233445566a7b", "destination_key": "work.data"}], + "document": { "...": "渲染时用的 plan.json，已脱敏" } + }, + "storage": { + "item_ids": ["st-pip-cache"], + "actions": {"st-pip-cache": "trash"}, + "document": { "...": "渲染时用的 analysis.json，已脱敏" } + } +} +``` + +`document` 两块是页面顺手带上的来源文件，所以 `apply` 只要这一个参数就够。两块都能缺，缺了就用 `--plan`、`--analysis` 或 `--managed-dir` 指过去。`apply` 先跑整理那段再跑盘点那段，两段各自写清单和审计，最后打一行合计。 + +serve 模式底栏是「执行全部已选」，前端按类型拆开：先 POST `/api/apply`（body `{action_ids, overrides, dry_run}`），再 POST `/api/dispose`（body `{item_ids, action}`，废纸篓一次永久删除一次），逐项回显状态。同一个服务两个端点都接，`/api/plan` 返回 `{plan, analysis, executed_ids}`，两半共用一份 `executed_ids`。 + +## 一句话原则 + +三色是核心，别的都让路。绿是放心做，黄是你看一眼，红是别动。强调色用墨色，页面里没有蓝，没有渐变，没有深色模式。危险动作靠文字说清楚，按钮权限跟颜色走。 + +## Token 表 + +| token | 值 | 用途 | +|---|---|---| +| `--bg` | `#f6f7f9` | 页面底、命令块底 | +| `--card` | `#ffffff` | 卡片底 | +| `--ink` | `#1d2129` | 正文、主按钮底、勾选框选中色，也是唯一的强调色 | +| `--muted` | `#86909c` | 次要文字、表头、路径 | +| `--line` | `#e5e6eb` | 1px 分隔线、次按钮描边 | +| `--green` / `--green-bg` | `#00b42a` / `#e8ffea` | 放心归位、可清 | +| `--yellow` / `--yellow-bg` | `#ff7d00` / `#fff7e8` | 你看一眼 | +| `--red` / `--red-bg` | `#f53f3f` / `#ffece8` | 别动 | +| `--other` | `#c9cdd4` | 磁盘条里的其他占用、无色的点 | +| `--radius` | `14px` | 卡片圆角；命令块、按钮、下拉用 `--radius-s` 8px | +| `--shadow` | `0 1px 3px rgba(0,0,0,.04), 0 8px 24px rgba(0,0,0,.04)` | 所有卡片同一个浅阴影 | + +字体：正文 `-apple-system, "PingFang SC", "Segoe UI", "Noto Sans CJK SC", "Helvetica Neue", Arial, sans-serif`，15px 行高 1.6；路径、命令、id 用 `ui-monospace, "SF Mono", Menlo, Consolas, "Noto Sans Mono CJK SC", monospace`，13px。标题只有页面标题 26px/700 和分区标题 18px/700，小节标题 13px 灰色大写字距。没有 web font。 + +## 布局（从上到下，organize 与 storage 两种旧形态） + +combined 的布局见开头那一节，下面这份是分步命令产出的单半页面。 + +``` +顶栏（sticky top，白底 92% 透明加轻微模糊，底边 1px） + 标题 · 目录或机器 · 扫描时间 · 模式；serve 时多一行「本页只在这台机器可见」 +页面标题 + 一句引导 +总览网格（两列，窄屏单列） + 整理页左：杂乱度大字 + 三色 pill + 计数小字 + 三色分段条 + 三色图例 + 一句话 + 盘点页左：每块磁盘一条（已用拆成 绿/黄/红/其他 四段加可用）+ 三色占比条 + 一句话 + 系统信息网格 + 右：执行建议卡「先做什么」，整理页按三色计数各一句，盘点页用 overview.priority；有绿项时给「勾选全部绿项」按钮 +Top 5 表（只有盘点页）：色点 · 大小 · 类型 · 名字 · 路径 · 一句话 +三色分区 green → yellow → red，每区一个
+ 分区头：色点 + 标题 + 计数 + 一句说明 + 工具（全部展开 / 本组全选 / serve 的执行本组 / 绿区的永久删除开关） + 每项一张
，左侧 4px 色条 +长期建议卡：盘点页 overview.long_term；整理页列静置中的到期时间，再放 undo 提示与三句固定说明 +底栏（sticky bottom）：已选计数 + 主按钮（static 导出，serve 执行） +``` + +三个分区永远都渲染，空的写一句本组没有条目。分段条的每一段用 `style="width:NN%"`，NN 按数量或字节向下取整，所以宽度之和永远不超过 100。 + +## 卡片 + +卡片头：勾选框（有权限才有）、名字、徽章（目录、敏感命名、待判断、去向、类型、组类型、可腾出）、大小、状态、折叠箭头。点头部空白处展开，点控件不展开。 + +卡片体：路径（等宽）、目录统计与标签、`这是什么 / 为什么这么判 / 动了会怎样 / 怎么恢复` 四行、命令块带复制按钮、控件行。 + +四行文案的来源：Agent 写了 notes 就用 notes（`notes.actions[id]`、`notes.groups[id]`、`notes.items[id]` 里的 `what / why / if_removed`），没写就回落到脚本的 `reason / detail / hold_reason / restore_method`（整理页）或条目自带的 `what / why / if_removed / restore`（盘点页）。空行不渲染。 + +## 按钮权限跟色走 + +| 颜色 | 整理页 | 盘点页 | +|---|---|---| +| green | 勾选执行；trash/delete 候选给单选，永久删除折叠在开关后并弹 confirm | 勾选；废纸篓单选默认选中，永久删除折叠在开关后 | +| yellow | 勾选移动或进废纸篓（都可逆）；配对与重复组的单选；待判断的去向下拉写进 overrides | 只有「移到废纸篓（可逆）」，且仅当 `trash_paths` 非空；serve 时多「打开所在位置」 | +| red | 没有任何 input，没有 `data-action="trash|delete"`；serve 时只有「打开所在位置」 | 同左 | + +判色顺序：数据里带 `color` 就用它；没有则整理页按 kind/tier 推（hold 或 forbidden/in_use/referenced/aging/pinned 是红，reroutable 或 duplicate/cold 是黄，其余绿；组里 regenerable 绿、pair/duplicate 黄），盘点页按 tier 和 `open_by`、`trash_paths` 推。 + +带永久删除的选项加 `needs-permanent` 类，`body[data-permanent="off"]` 时整体隐藏并 disabled；打开绿区的开关才显示。选中时弹 `confirm()`，文案含路径与大小，取消就退回默认选项。yellow 区永远不渲染永久删除选项，即便数据里有。 + +## 控件 data 属性 + +- 勾选框 `input.gn-pick`：整理页移动带 `data-action-id`，候选行带 `data-choice`；盘点页带 `data-item-id` 与 `data-choice`。 +- 单选 `input.gn-choice`：`data-action="trash|delete"`，永久删除再带 `data-kind="delete"`、`data-path`、`data-size`。 +- 组：`article[data-group-id]` 内 `input.gn-adopt` 与 `input.gn-option[data-action-ids]`，页面只按 `options[].action_ids` 映射，不自己推导。 +- 待判断下拉 `select.gn-dest`，`data-original` 记原去向，改了才写进 overrides。 +- 按钮 `data-action="copy"` 复制同块 ``；`data-action="reveal"` 带 `data-path`，只在 serve 渲染。 +- 状态回写按 `[data-ids]` 匹配 id，`.gn-status` 显示文字，`done` 时禁用该卡片全部控件。 + +## static 与 serve + +| | static | serve | +|---|---|---| +| 主按钮 | combined 导出一份 `carl-file-organizer-decisions.json`（格式见上）；整理页导出 `carl-file-organizer-approved.json`（plan 本身加 `approved_action_ids / overrides / approved_at / approved_by`）；盘点页导出 `carl-file-organizer-decisions.json` 的旧格式（`item_ids` 加每个 id 的 `actions` 映射） | combined 先 POST `/api/apply` 再 POST `/api/dispose`；整理页只 POST `/api/apply`，body `{action_ids, overrides, dry_run}`；盘点页只 POST `/api/dispose`，按动作分两次，body `{item_ids, action: "trash"\|"delete"}` | +| 其他按钮 | 无 | 停止服务（POST `/api/shutdown`）；带整理那半时多「先预演」；分区头多「执行本组已选」 | +| 打开所在位置 | 不渲染 | POST `/api/reveal`，body `{path_portable}` | +| token | 不嵌 | 嵌进 config，也接受 `?t=` 后立即从地址栏抹掉；请求头 `X-GN-Token` | +| 启动时 | 无请求 | 带整理那半时 GET `/api/plan` 同步 `executed_ids` 与 `permanent_delete_enabled` | + +服务端响应 `{results: [{action_id 或 item_id, status, detail}]}`，status 取 `moved / trashed / deleted / dry-run / skipped / refused / failed`。 + +## 脱敏 + +页面和内嵌 JSON 里一个绝对路径都没有。渲染前整份数据过 `build_report.sanitize()`，combined 的两半各过一遍：`source_root`、`managed_dir`、每条 action 的 `source` 与 `destination`、每个 item 的 `path` 直接删掉，只留 `_portable` 双胞胎；其余任何字符串里出现家目录（按 plan 反推的 home，以及 `/Users/x`、`/home/x`、`C:\Users\x` 三种形态）都改写成 `$HOME`。测试扫整页不许出现 `/Users/`。 + +## 文案 + +所有固定文案都在模板顶部 `', html, re.S) + if not match: + raise ValueError("report_template.html has no report-text block") + return json.loads(match.group(1)) + + +def _lang_of(data: Dict[str, Any], lang: Optional[str]) -> str: + chosen = lang or data.get("lang") or "zh" + return chosen if chosen in LANGS else "zh" + + +class _T(object): + """``t("key", n=3)`` with fallback to English and then the key itself.""" + + def __init__(self, text: Dict[str, Dict[str, Any]], lang: str) -> None: + self.text = text + self.lang = lang + + def badge(self, token: Any) -> str: + """The reader's word for a tier or a kind, from the template's badge table. + + Cards used to print the raw token, so a page in Chinese still said + ``aging``, ``in use`` and ``build_artifact`` on its own badges. A token + the table does not know falls through unchanged, which is the only way a + plan written by a newer planner still renders. + """ + + key = str(token or "") + if not key: + return "" + for lang in (self.lang, "en"): + table = (self.text.get(lang) or {}).get("badge") + if isinstance(table, dict): + label = table.get(key) + if label: + return str(label) + return key + + def __call__(self, key: str, **kw: Any) -> str: + table = self.text.get(self.lang) or {} + template = table.get(key) + if template is None: + template = (self.text.get("en") or {}).get(key) + if template is None: + template = key + if kw: + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + return template + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + + +def format_bytes(value: Union[int, float, None]) -> str: + size = float(value or 0) + for unit in ("B", "KB", "MB", "GB", "TB"): + if size < 1024 or unit == "TB": + return "{0:.1f} {1}".format(size, unit) if unit != "B" else "{0} B".format(int(size)) + size /= 1024 + return "{0:.1f} TB".format(size) + + +def _pick(value: Any, lang: str) -> str: + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or value.get("zh") or "") + if value is None: + return "" + return str(value) + + +def _e(value: Any) -> str: + return escape("" if value is None else str(value), quote=True) + + +def _embed_json(data: Any) -> str: + text = json.dumps(data, ensure_ascii=False) + return ( + text.replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("
", "\\u2028") + .replace("
", "\\u2029") + ) + + +def _display_time(value: Any) -> str: + text = str(value or "") + short = text + if len(text) >= 16 and text[10] == "T": + short = text[:10] + " " + text[11:16] + return '{1}'.format(_e(text), _e(short)) + + +#: ``2026-09-04T11:56:03.412870+09:00`` -> the year, month, day, hour, minute. +_ISO_HEAD = re.compile(r"^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})") + +_MONTHS_EN = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + + +def format_ready_at(value: Any, lang: str) -> str: + """A settling deadline the way a person says it: ``9 月 4 日 11:56``. + + The planner stores an ISO timestamp with microseconds and an offset, which + is the right thing to store and the wrong thing to read. The template does + no date arithmetic; the whole conversion happens here, and anything that is + not an ISO stamp is handed through untouched. + """ + + text = str(value or "") + match = _ISO_HEAD.match(text) + if not match: + return text + _year, month, day, hour, minute = match.groups() + if lang == "zh": + return "{0} 月 {1} 日 {2}:{3}".format(int(month), int(day), hour, minute) + return "{0} {1}, {2}:{3}".format(_MONTHS_EN[int(month) - 1], int(day), hour, minute) + + +def _color(value: Any, fallback: str) -> str: + return value if value in COLORS else fallback + + +def _widths(parts: List[Tuple[str, float]]) -> List[Tuple[str, int]]: + """Integer percentages, floored, so they never add up past 100.""" + + total = sum(max(0.0, float(v or 0)) for _, v in parts) + if total <= 0: + return [(k, 0) for k, _ in parts] + return [(k, int(max(0.0, float(v or 0)) * 100 // total)) for k, v in parts] + + +def _segbar(parts: List[Tuple[str, float]]) -> str: + segs = [] + for key, width in _widths(parts): + if width <= 0: + continue + segs.append(''.format(_e(key), width)) + return '
{0}
'.format("".join(segs)) + + +# -------------------------------------------------------------------------- +# kind detection, sanitising, notes +# -------------------------------------------------------------------------- + + +def detect_kind(data: Dict[str, Any]) -> str: + if isinstance(data.get("plan"), dict) or isinstance(data.get("analysis"), dict): + return "combined" + schema = str(data.get("schema") or "") + if "storage" in schema or "analysis" in schema: + return "storage" + if "actions" in data and data.get("schema_version") == 2: + return "organize" + if "items" in data or "disks" in data: + return "storage" + if "actions" in data: + return "organize" + raise ValueError("cannot tell whether this is a plan.json or an analysis.json") + + +def _home_of(data: Dict[str, Any], home: Optional[Union[str, Path]]) -> str: + if home is not None: + return str(home) + root = str(data.get("source_root") or "") + portable = str(data.get("source_root_portable") or "") + if root and portable == HOME_TOKEN: + return root + prefix = HOME_TOKEN + "/" + if root and portable.startswith(prefix): + relative = Path(portable[len(prefix):]).parts + whole = Path(root).parts + if len(whole) > len(relative) and whole[-len(relative):] == relative: + return str(Path(*whole[: -len(relative)])) + return str(Path.home()) + + +def _portable_text(text: str, base: str) -> str: + if base and base != "/": + if text == base: + text = HOME_TOKEN + else: + text = text.replace(base + "/", HOME_TOKEN + "/").replace(base + "\\", HOME_TOKEN + "\\") + for pattern in _HOME_PATTERNS: + text = pattern.sub(HOME_TOKEN, text) + return text + + +def _scrub(value: Any, base: str) -> Any: + if isinstance(value, dict): + return {k: _scrub(v, base) for k, v in value.items()} + if isinstance(value, list): + return [_scrub(item, base) for item in value] + if isinstance(value, str): + return _portable_text(value, base) + return value + + +def sanitize(data: Any, home: Optional[Union[str, Path]] = None) -> Any: + """A copy carrying no absolute path: the ``paths.strip_absolute`` rule, re-stated. + + ``source_root``, ``managed_dir`` and every action's ``source`` and + ``destination`` are dropped (their ``_portable`` twins stay); every other + string that starts with the home directory, or looks like one on macOS, + Linux or Windows, is rewritten to ``$HOME``. + """ + + if not isinstance(data, dict): + return data + base = _home_of(data, home) + out = copy.deepcopy(data) + for field in PLAN_ABSOLUTE_FIELDS: + out.pop(field, None) + for action in out.get("actions") or []: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + action.pop(field, None) + for item in out.get("items") or []: + if isinstance(item, dict): + item.pop("path", None) + return _scrub(out, base) + + +def merge_notes(data: Dict[str, Any], notes: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """``notes.json`` on top of whatever ``data["notes"]`` already carries.""" + + merged: Dict[str, Any] = copy.deepcopy(data.get("notes") or {}) if isinstance(data.get("notes"), dict) else {} + for key, value in (notes or {}).items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + inner = dict(merged[key]) + for k2, v2 in value.items(): + if isinstance(v2, dict) and isinstance(inner.get(k2), dict): + tmp = dict(inner[k2]) + tmp.update(v2) + inner[k2] = tmp + else: + inner[k2] = v2 + merged[key] = inner + else: + merged[key] = value + out = dict(data) + out["notes"] = merged + return out + + +def _note_for(notes: Dict[str, Any], table: str, ids: List[str], lang: str) -> Dict[str, str]: + bucket = notes.get(table) or {} + for i in ids: + entry = bucket.get(i) + if isinstance(entry, dict): + return {k: _pick(v, lang) for k, v in entry.items() if isinstance(v, (str, dict))} + return {} + + +# -------------------------------------------------------------------------- +# colours for a plan that predates them +# -------------------------------------------------------------------------- + + +def action_color(action: Dict[str, Any]) -> str: + given = action.get("color") + if given in COLORS: + return given + kind = action.get("kind", "move") + tier = action.get("tier", "routine") + if kind == "hold" or tier in RED_TIERS: + return "red" + if action.get("reroutable") or tier in YELLOW_TIERS: + return "yellow" + return "green" + + +def group_color(group: Dict[str, Any]) -> str: + given = group.get("color") + if given in COLORS: + return given + return "green" if group.get("kind") == "regenerable" else "yellow" + + +def _mess_fallback(plan: Dict[str, Any], t: _T) -> Dict[str, Any]: + actions = [a for a in (plan.get("actions") or []) if isinstance(a, dict)] + groups = plan.get("groups") or [] + summary = plan.get("summary") or {} + entries = int(summary.get("entries") or len({a.get("subject_id") for a in actions})) + pending = len([a for a in actions if a.get("kind") == "move" and a.get("reroutable")]) + holds = len([a for a in actions if a.get("kind") == "hold"]) + aging = len([a for a in actions if a.get("kind") == "hold" and a.get("tier") == "aging"]) + top_dirs = len({a.get("subject_id") for a in actions if a.get("subject_kind") == "dir" and a.get("kind") != "hold"}) + score = int(min(100, round(entries * 1.5 + pending * 5 + len(groups) * 6 + holds * 2))) + color = "green" if score < 35 else "yellow" if score < 70 else "red" + return { + "score": score, + "color": color, + "counts": {"scattered": entries, "pending": pending, "duplicate": len(groups), "aging": aging, "top_dirs": top_dirs}, + "reason": t("mess_fallback", entries=entries, pending=pending, groups=len(groups), holds=holds), + "source": "fallback", + } + + +def _by_color_fallback(plan: Dict[str, Any]) -> Dict[str, Dict[str, int]]: + out = {c: {"count": 0, "bytes": 0} for c in COLORS} + seen = set() + for a in plan.get("actions") or []: + if not isinstance(a, dict): + continue + sid = str(a.get("subject_id") or a.get("id")) + if sid in seen: + continue + seen.add(sid) + c = action_color(a) + out[c]["count"] += 1 + out[c]["bytes"] += int(a.get("size_bytes") or 0) + return out + + +def _normalise_by_color(raw: Any, fallback: Dict[str, Dict[str, int]]) -> Dict[str, Dict[str, int]]: + if not isinstance(raw, dict): + return fallback + out = {} + for c in COLORS: + v = raw.get(c) + if isinstance(v, dict): + count = v.get("count") + if count is None: + count = v.get("actions", v.get("items", 0)) + out[c] = {"count": int(count or 0), "bytes": int(v.get("bytes") or 0)} + elif isinstance(v, (int, float)): + out[c] = {"count": int(v), "bytes": fallback.get(c, {}).get("bytes", 0)} + else: + out[c] = {"count": 0, "bytes": 0} + return out + + +# -------------------------------------------------------------------------- +# shared fragments +# -------------------------------------------------------------------------- + + +def _copy_block(command: str, t: _T) -> str: + return ( + '
{0}' + '
' + ).format(_e(command), t("copy")) + + +def _reveal_button(path: str, mode: str, t: _T) -> str: + if mode != "serve" or not path: + return "" + return ''.format( + _e(path), t("reveal") + ) + + +def _dl(rows: List[Tuple[str, str]]) -> str: + body = "".join("
{0}
{1}
".format(_e(k), v) for k, v in rows if v) + return "
{0}
".format(body) if body else "" + + +def _card( + *, + ids: List[str], + subject: str, + color: str, + name: str, + badges: List[str], + size: Any, + head_extra: str = "", + pick: str = "", + body: str, + open_by_default: bool = False, + extra_attrs: str = "", +) -> str: + return ( + '
' + '
' + '{pick}' + '{name}{badges}{head_extra}' + '{size}' + '' + '' + "
" + '
{body}
' + "
" + ).format( + open=" open" if open_by_default else "", + color=_e(color), + ids=_e(" ".join(i for i in ids if i)), + sid=_e(subject), + extra=extra_attrs, + pick=pick, + name=_e(name), + badges="".join(badges), + head_extra=head_extra, + size=format_bytes(size), + body=body, + ) + + +def _section( + key: str, + color: str, + title: str, + desc: str, + cards: List[str], + *, + mode: str, + t: _T, + selectable: bool, + tools_extra: str = "", + subgroups: Optional[List[Tuple[str, List[str]]]] = None, +) -> str: + """One coloured band. ``subgroups`` splits it into labelled runs of cards. + + The combined page needs each band to say cleanup first and moves second + without growing a second section markup; everything else -- the head, the + tools, the empty line -- is the same either way, so the two shapes share + this function and differ only in how the body is assembled. + """ + + if subgroups is not None: + cards = [card for _, group in subgroups for card in group] + tools = [] + if cards: + tools.append( + ''.format( + t("expand_all"), t("collapse_all") + ) + ) + if selectable and cards: + tools.append( + ''.format( + t("btn_select_all"), t("btn_clear_all") + ) + ) + if mode == "serve": + tools.append(''.format(t("btn_section"))) + tools.append(tools_extra) + if subgroups is not None: + parts = [] + for label, group in subgroups: + if not group: + continue + parts.append('

{0}{1}

'.format(_e(label), len(group))) + parts.extend(group) + body = "".join(parts) or '

{0}

'.format(t("empty")) + else: + body = "".join(cards) or '

{0}

'.format(t("empty")) + return ( + '
' + '

{title}' + '{count}

{desc}

' + '
{tools}
' + "{body}
" + ).format(color=_e(color), key=_e(key), title=_e(title), count=len(cards), desc=_e(desc), tools="".join(tools), body=body) + + +# -------------------------------------------------------------------------- +# organize page +# -------------------------------------------------------------------------- + + +def _guard_lines(action: Dict[str, Any], lang: str, t: _T) -> List[str]: + guard = action.get("guard") or {} + lines: List[str] = [] + hold = action.get("hold_reason") + if isinstance(hold, dict) and hold.get("ready_at"): + lines.append(_e(t("ready_at", time=format_ready_at(hold.get("ready_at"), lang)))) + if isinstance(hold, dict) and hold.get("label"): + lines.append('{0}'.format(_e(hold.get("label")))) + open_by = guard.get("open_by") or [] + if open_by: + procs = ", ".join( + "{0} (pid {1})".format(_e(p.get("command")), _e(p.get("pid"))) if isinstance(p, dict) else _e(p) for p in open_by + ) + lines.append("{0}: {1}".format(t("open_by"), procs)) + refs = guard.get("referenced_in") or [] + if refs: + items = [] + for ref in refs: + if isinstance(ref, dict): + where = "{0}:{1}".format(ref.get("file", ""), ref.get("line", "")) + form = ref.get("form") + items.append(_e(where) + (" ({0})".format(_e(form)) if form else "")) + else: + items.append(_e(ref)) + lines.append("{0}: {1}".format(t("referenced_in"), "; ".join(items))) + links = guard.get("incoming_symlinks") or [] + if links: + lines.append("{0}: {1}".format(t("symlinks"), _e("; ".join(str(x) for x in links)))) + shape = guard.get("shape") or [] + if shape: + lines.append("{0}: {1}".format(t("shape"), _e(" · ".join(str(s) for s in shape)))) + return lines + + +def _action_badges(action: Dict[str, Any], t: _T) -> List[str]: + badges = [] + if action.get("subject_kind") == "dir": + badges.append('{0}'.format(_e(t.badge("dir")))) + if action.get("tier") == "sensitive": + badges.append('{0}'.format(_e(t.badge("sensitive")))) + return badges + + +def _action_extras(action: Dict[str, Any], t: _T) -> str: + bits = [] + stats = action.get("dir_stats") + if isinstance(stats, dict): + key = "dir_files_trunc" if stats.get("truncated") else "dir_files" + bits.append('
{0}
'.format(_e(t(key, n=stats.get("files", 0))))) + tags = [x for x in (action.get("tags") or []) if x] + if tags: + bits.append('
{0}: {1}
'.format(t("tags"), _e(" · ".join(str(x) for x in tags)))) + hints = [str(h) for h in (action.get("hints") or []) if h] + if hints: + bits.append('
{0}: {1}
'.format(t("hint"), _e(" · ".join(hints)))) + return "".join(bits) + + +def _explain(action: Dict[str, Any], note: Dict[str, str], lang: str, t: _T, *, what_default: str = "") -> str: + """what / why / if_removed: the note when the Agent wrote one, the rule text otherwise.""" + + what = note.get("what") or what_default or _pick(action.get("reason"), lang) + why = note.get("why") or _pick(action.get("detail"), lang) + hold = _pick(action.get("hold_reason"), lang) + if not why and hold and hold != what: + why = hold + restore = _pick(action.get("restore_method"), lang) + if_removed = note.get("if_removed") or "" + if not if_removed and action.get("kind") == "trash": + if_removed = t("restore_trash") + if not if_removed and action.get("kind") == "delete": + if_removed = t("restore_delete") + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(if_removed))] + if restore and action.get("kind") != "move": + rows.append((t("restore"), _e(restore))) + return _dl(rows) + + +def _path_line(path: str, t: _T) -> str: + return '
{0}
'.format(_e(path)) if path else "" + + +def _card_move(action: Dict[str, Any], color: str, names: Dict[str, str], notes: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + aid = str(action.get("id", "")) + note = _note_for(notes, "actions", [aid], lang) + dest = names.get(str(action.get("destination_key") or ""), "") + if not dest: + dest_path = action.get("destination_portable") or "" + dest = str(Path(str(dest_path)).parent) if dest_path else "" + path = str(action.get("source_portable") or "") + pending = bool(action.get("reroutable")) + controls_allowed = color != "red" + pick = "" + if controls_allowed: + pick = ''.format( + _e(aid), _e(action.get("filename")) + ) + badges = _action_badges(action, t) + if pending: + badges.append('{0}'.format(t("pending_label"))) + if dest: + badges.append('→ {0}'.format(_e(dest))) + body = [_path_line(path, t), _action_extras(action, t), _explain(action, note, lang, t)] + restore = _pick(action.get("restore_method"), lang) + if restore: + body.append('
{0}
'.format(t("restore"))) + body.append(_copy_block(restore, t)) + controls = [] + if pending and controls_allowed: + current = str(action.get("destination_key") or "") + options = [] + for key, value in sorted(names.items(), key=lambda kv: (str(kv[1]), kv[0])): + options.append(''.format(_e(key), " selected" if key == current else "", _e(value))) + controls.append( + ''.format( + t("change_dest"), _e(aid), _e(current), "".join(options) + ) + ) + controls.append(_reveal_button(path, mode, t)) + if any(controls): + body.append('
{0}
'.format("".join(controls))) + return _card( + ids=[aid], subject=str(action.get("subject_id", "")), color=color, name=str(action.get("filename") or ""), + badges=badges, size=action.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _card_candidate( + trash: Optional[Dict[str, Any]], delete: Optional[Dict[str, Any]], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T, + *, trash_available: bool, delete_offered: bool, +) -> str: + action = trash or delete or {} + sid = str(action.get("subject_id", "")) + ids = [str(a["id"]) for a in (trash, delete) if a and a.get("id")] + note = _note_for(notes, "actions", ids, lang) + name = "cand-" + (sid or "-".join(ids)) + path = str(action.get("source_portable") or "") + choices: List[str] = [] + if color != "red": + if trash is not None and trash_available and trash.get("approvable", True): + choices.append( + ''.format( + n=_e(name), v=_e(trash.get("id")), label=t("opt_trash") + ) + ) + if color == "green" and delete is not None and delete_offered and delete.get("approvable", True): + choices.append( + ''.format( + n=_e(name), v=_e(delete.get("id")), p=_e(path or delete.get("filename")), + s=_e(format_bytes(delete.get("size_bytes"))), label=t("opt_delete"), + ) + ) + pick = "" + if choices: + pick = ''.format(_e(name), _e(action.get("filename"))) + badges = _action_badges(action, t) + badges.append('{1}'.format(color, t("opt_trash") if trash is not None else t("opt_delete"))) + body = [_path_line(path, t), _action_extras(action, t), _explain(action, note, lang, t)] + controls = choices + [_reveal_button(path, mode, t)] + if any(controls): + body.append('
{0}
'.format("".join(controls))) + elif color != "red": + body.append('
{0}
'.format(t("keep_here"))) + return _card( + ids=ids, subject=sid, color=color, name=str(action.get("filename") or ""), badges=badges, + size=action.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _card_hold(action: Dict[str, Any], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + aid = str(action.get("id", "")) + note = _note_for(notes, "actions", [aid], lang) + path = str(action.get("source_portable") or "") + hold = _pick(action.get("hold_reason"), lang) + reason = _pick(action.get("reason"), lang) + what = note.get("what") or hold or reason + why = note.get("why") or (reason if reason != what else "") + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(note.get("if_removed") or ""))] + body = [_path_line(path, t), _action_extras(action, t), _dl(rows)] + guard = _guard_lines(action, lang, t) + if guard: + body.append("".join('
{0}
'.format(line) for line in guard)) + reveal = _reveal_button(path, mode, t) + if reveal: + body.append('
{0}
'.format(reveal)) + badges = _action_badges(action, t) + tier = str(action.get("tier") or "") + if tier: + badges.append('{0}'.format(_e(t.badge(tier)))) + head_extra = '{0}'.format(_e(what)) if what else "" + return _card( + ids=[aid], subject=str(action.get("subject_id", "")), color=color, name=str(action.get("filename") or ""), + badges=badges, size=action.get("size_bytes"), head_extra=head_extra, body="".join(body), + ) + + +def _card_group( + group: Dict[str, Any], color: str, by_subject: Dict[str, Dict[str, Any]], by_id: Dict[str, Dict[str, Any]], + notes: Dict[str, Any], lang: str, mode: str, t: _T, *, trash_available: bool, delete_offered: bool, +) -> str: + gid = str(group.get("group_id", "")) + note = _note_for(notes, "groups", [gid], lang) + labels = group.get("member_labels") or {} + members = [str(m) for m in (group.get("members") or [])] + member_rows = [] + total = 0 + for sid in members: + action = by_subject.get(sid) or {} + label = labels.get(sid) or action.get("filename") or sid + source = action.get("source_portable") or "" + total += int(action.get("size_bytes") or 0) + member_rows.append( + '
  • {0}{1}{2}{3}
    • '.format( + _e(label), format_bytes(action.get("size_bytes")), + '{0}'.format(_e(source)) if source else "", + _reveal_button(str(source), mode, t), + ) + ) + all_ids: List[str] = [] + options_html = [] + default = group.get("default_choice") + radio_name = "grp-" + gid + for option in group.get("options") or []: + key = str(option.get("key", "")) + ids = [str(i) for i in (option.get("action_ids") or [])] + for i in ids: + if i not in all_ids: + all_ids.append(i) + needs_permanent = PERMANENT_FLAG in (option.get("requires") or []) or any( + by_id.get(i, {}).get("kind") == "delete" for i in ids + ) + if color == "red": + continue + if needs_permanent and (not delete_offered or color != "green"): + continue + if not trash_available and any(by_id.get(i, {}).get("kind") == "trash" for i in ids): + continue + attrs = [ + 'type="radio"', 'class="gn-option"', 'name="{0}"'.format(_e(radio_name)), 'value="{0}"'.format(_e(key)), + 'data-action-ids="{0}"'.format(_e(" ".join(ids))), + ] + if key == default: + attrs.append("checked") + attrs.append('data-default="1"') + if needs_permanent: + attrs.append('data-action="delete"') + attrs.append('data-kind="delete"') + attrs.append('data-path="{0}"'.format(_e(" / ".join(str(labels.get(m, m)) for m in members)))) + attrs.append('data-size="{0}"'.format(_e(format_bytes(group.get("potential_bytes"))))) + attrs.append("disabled") + else: + attrs.append('data-action="{0}"'.format("trash" if any(by_id.get(i, {}).get("kind") == "trash" for i in ids) else "move")) + options_html.append( + ''.format( + cls=" needs-permanent" if needs_permanent else "", attrs=" ".join(attrs), label=_e(_pick(option.get("label"), lang)) + ) + ) + pick = "" + if options_html: + pick = ''.format(t("adopt_group")) + potential = group.get("potential_bytes") or 0 + badges = ['{0}'.format(_e(t.badge(group.get("kind") or "group")))] + if potential: + badges.append('{1}'.format(color, _e(t("potential", bytes=format_bytes(potential))))) + reason = _pick(group.get("reason"), lang) + rows = [ + (t("what"), _e(note.get("what") or reason)), + (t("why"), _e(note.get("why") or (_e(group.get("evidence") or "") if note.get("what") else ""))), + (t("if_removed"), _e(note.get("if_removed") or "")), + ] + body = ['
      {0}
    '.format("".join(member_rows)), _dl(rows)] + if options_html: + body.append('
    {0}
    '.format("".join(options_html))) + else: + body.append('
    {0}
    '.format(t("keep_here"))) + title = " · ".join(str(labels.get(m, m)) for m in members) + return _card( + ids=all_ids, subject=gid, color=color, name=title, badges=badges, size=total, pick=pick, body="".join(body), + extra_attrs=' data-group-id="{0}"'.format(_e(gid)), + ) + + +def _permanent_switch(data: Dict[str, Any], mode: str, t: _T) -> str: + """The green section's permanent-delete toggle, or nothing when it is off.""" + + capabilities = data.get("capabilities") or {} + if not bool(capabilities.get("permanent_delete_offered", True)): + return "" + enabled = capabilities.get("permanent_delete_enabled", True) + disabled = " disabled" if mode == "serve" and not enabled else "" + note = t("permanent_not_enabled") if (mode == "serve" and not enabled) else t("permanent_static_note") + return ''.format( + disabled, t("toggle_permanent"), _e(note) + ) + + +def _organize_buckets(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> Dict[str, List[str]]: + """Every tidy-up card, already sorted, keyed by colour.""" + + actions: List[Dict[str, Any]] = [a for a in (plan.get("actions") or []) if isinstance(a, dict)] + groups: List[Dict[str, Any]] = [g for g in (plan.get("groups") or []) if isinstance(g, dict)] + names: Dict[str, str] = {str(k): str(v) for k, v in (plan.get("names") or {}).items()} + notes: Dict[str, Any] = plan.get("notes") or {} + capabilities = plan.get("capabilities") or {} + trash_available = capabilities.get("trash_backend", "finder") != "none" + delete_offered = bool(capabilities.get("permanent_delete_offered", True)) + + by_id = {str(a.get("id")): a for a in actions} + by_subject: Dict[str, Dict[str, Any]] = {} + for a in actions: + by_subject.setdefault(str(a.get("subject_id", a.get("id"))), a) + grouped_ids = set() + for g in groups: + for option in g.get("options") or []: + for i in option.get("action_ids") or []: + grouped_ids.add(str(i)) + + buckets: Dict[str, List[Tuple[int, str]]] = {c: [] for c in COLORS} + candidates: Dict[str, Dict[str, Optional[Dict[str, Any]]]] = {} + order: List[str] = [] + + for a in actions: + kind = a.get("kind", "move") + aid = str(a.get("id")) + color = action_color(a) + if kind == "hold": + buckets[color].append((4, _card_hold(a, color, notes, lang, mode, t))) + continue + if aid in grouped_ids: + continue + if kind == "move": + rank = 1 if a.get("reroutable") else 0 + buckets[color].append((rank, _card_move(a, color, names, notes, lang, mode, t))) + continue + if kind in ("trash", "delete"): + sid = str(a.get("subject_id", aid)) + if sid not in candidates: + candidates[sid] = {"trash": None, "delete": None} + order.append(sid) + candidates[sid][kind] = a + continue + buckets[color].append((0, _card_move(a, color, names, notes, lang, mode, t))) + + for g in groups: + color = group_color(g) + buckets[color].append( + (2, _card_group(g, color, by_subject, by_id, notes, lang, mode, t, trash_available=trash_available, delete_offered=delete_offered)) + ) + for sid in order: + trash, delete = candidates[sid]["trash"], candidates[sid]["delete"] + color = action_color(trash or delete or {}) + buckets[color].append( + (3, _card_candidate(trash, delete, color, notes, lang, mode, t, trash_available=trash_available, delete_offered=delete_offered)) + ) + + return {c: [h for _, h in sorted(buckets[c], key=lambda x: x[0])] for c in COLORS} + + +def _organize_sections(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> Tuple[str, Dict[str, int]]: + buckets = _organize_buckets(plan, lang, mode, t) + permanent = _permanent_switch(plan, mode, t) + counts = {c: len(buckets[c]) for c in COLORS} + html = "".join( + [ + _section("green", "green", t("sec_green"), t("sec_green_desc"), buckets["green"], mode=mode, t=t, selectable=True, tools_extra=permanent), + _section("yellow", "yellow", t("sec_yellow"), t("sec_yellow_desc"), buckets["yellow"], mode=mode, t=t, selectable=True), + _section("red", "red", t("sec_red"), t("sec_red_desc"), buckets["red"], mode=mode, t=t, selectable=False), + ] + ) + return html, counts + + +def _mess_inner(plan: Dict[str, Any], lang: str, t: _T) -> str: + """The mess score, the pill, the counts, the three-colour bar and one line. + + Both the tidy-up page and the combined page open with this block, so it + lives on its own rather than inside either overview. + """ + + notes = plan.get("notes") or {} + mess = plan.get("mess") if isinstance(plan.get("mess"), dict) else _mess_fallback(plan, t) + mess_color = _color(mess.get("color"), "yellow") + score = mess.get("score") + by_color = _normalise_by_color((plan.get("summary") or {}).get("by_color"), _by_color_fallback(plan)) + line = _pick((notes.get("mess") or {}).get("line"), lang) if isinstance(notes.get("mess"), dict) else "" + if isinstance(notes.get("mess"), dict) and notes["mess"].get("color") in COLORS: + mess_color = notes["mess"]["color"] + if not line: + line = _pick(mess.get("reason"), lang) + chips = [] + for key, value in (mess.get("counts") or {}).items(): + label = t("count_" + str(key)) + chips.append("{0} {1}".format(_e(label if label != "count_" + str(key) else key), _e(value))) + legend = "".join( + '{1} {2} · {3}'.format( + c, t("legend_" + c), by_color[c]["count"], format_bytes(by_color[c]["bytes"]) + ) + for c in COLORS + ) + big = '{0}/100'.format(_e(score)) if isinstance(score, (int, float)) else "" + return ( + '

    {title}

    ' + '
    {big}{label}
    ' + '
    {chips}
    ' + "{bar}" + '
    {legend}
    ' + '

    {line}

    ' + ).format( + title=t("mess_title"), big=big, color=mess_color, label=t("mess_" + mess_color), chips="".join(chips), + bar=_segbar([(c, by_color[c]["count"]) for c in COLORS]), legend=legend, line=_e(line), + ) + + +def _advice_items(counts: Dict[str, int], t: _T) -> List[str]: + """One line per colour that actually has cards, in colour order.""" + + items = [] + for color in COLORS: + if counts.get(color): + items.append("
  • {0}
    • ".format(_e(t("advice_" + color, n=counts[color])))) + return items + + +def _pick_green_button(counts: Dict[str, int], t: _T) -> str: + if not counts.get("green"): + return "" + return '
    '.format( + t("btn_pick_green") + ) + + +def _organize_overview(plan: Dict[str, Any], lang: str, mode: str, t: _T, counts: Dict[str, int]) -> str: + notes = plan.get("notes") or {} + overview = '
    {0}
    '.format(_mess_inner(plan, lang, t)) + folder_line = _pick(notes.get("folder_line"), lang) + advice_items = _advice_items(counts, t) + if not advice_items: + advice_items.append("
  • {0}
    • ".format(_e(t("advice_none")))) + tools = _pick_green_button(counts, t) + advice = ( + '

    {title}

    {folder}
      {items}
    {tools}
    ' + ).format( + title=t("advice_title"), folder='

    {0}

    '.format(_e(folder_line)) if folder_line else "", + items="".join(advice_items), tools=tools, + ) + return '
    {0}{1}
    '.format(overview, advice) + + +def _organize_longterm(plan: Dict[str, Any], lang: str, t: _T) -> str: + aging = [] + for a in plan.get("actions") or []: + if not isinstance(a, dict) or a.get("kind") != "hold": + continue + hold = a.get("hold_reason") if isinstance(a.get("hold_reason"), dict) else {} + if hold.get("ready_at"): + aging.append( + "
  • {0}
    • ".format( + _e( + t( + "aging_line", + name=a.get("filename"), + time=format_ready_at(hold.get("ready_at"), lang), + ) + ) + ) + ) + aging_html = "" + if aging: + aging_html = "

    {0}

      {1}
    ".format(t("aging_title"), "".join(aging)) + capabilities = plan.get("capabilities") or {} + notes = [t("undo_hint"), t("pledge"), t("pledge_more"), t("disk_note")] + if capabilities.get("trash_backend", "finder") == "none": + notes.append(t("no_trash_backend")) + return '
    {0}

    {1}

    {2}
    '.format( + aging_html, t("longterm_title"), "".join("

    {0}

    ".format(_e(n)) for n in notes) + ) + + +def _organize_bar(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + cells = [ + (t("bar_dir"), '{0}'.format(_e(plan.get("source_root_portable") or ""))), + (t("bar_time"), '{0}'.format(_display_time(plan.get("created_at") or plan.get("now") or ""))), + (t("bar_mode"), '{0}'.format(t("mode_" + mode))), + ] + return _bar(t("title_organize"), cells, mode, t) + + +def _bar(brand: str, cells: List[Tuple[str, str]], mode: str, t: _T) -> str: + items = "".join('{0}{1}'.format(_e(k), v) for k, v in cells) + serve = '
    {0}
    '.format(t("serve_note")) if mode == "serve" else "" + return '
    {0}{1}
    {2}
    '.format( + _e(brand), items, serve + ) + + +def _footer(kind: str, mode: str, t: _T) -> str: + if mode == "serve": + buttons = [''.format(t("btn_shutdown"))] + if kind in ("organize", "combined"): + buttons.append(''.format(t("btn_dry"))) + label = t("btn_apply_all") if kind == "combined" else t("btn_apply") + buttons.append(''.format(label)) + hint = "" + else: + label = t("btn_export") if kind == "organize" else t("btn_export_decisions") + buttons = [''.format(label)] + hint = '
    {0}
    '.format(t("footer_static_hint")) + return ( + '
    ' + '
    {hint}
    ' + '
    {buttons}
    ' + "
    " + ).format(hint=hint, buttons="".join(buttons)) + + +def render_organize_body(plan: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + sections, counts = _organize_sections(plan, lang, mode, t) + return "".join( + [ + _organize_bar(plan, lang, mode, t), + '
    ', + '

    {0}

    {1}

    '.format(t("heading_organize"), t("lede_organize")), + _organize_overview(plan, lang, mode, t, counts), + sections, + _organize_longterm(plan, lang, t), + '

    {0}

    '.format(t("print_hint")), + "
    ", + _footer("organize", mode, t), + ] + ) + + +# -------------------------------------------------------------------------- +# storage page +# -------------------------------------------------------------------------- + + +def item_color(item: Dict[str, Any]) -> str: + given = item.get("color") + if given in COLORS: + return given + tier = str(item.get("tier") or "") + if tier in RED_TIERS or item.get("open_by"): + return "red" + if tier in YELLOW_TIERS or not item.get("trash_paths"): + return "yellow" + return "green" + + +def _kind_label(kind: Any, t: _T) -> str: + """The badge table first, the older ``kind_*`` keys second, the token last.""" + + token = str(kind or "other") + badge = t.badge(token) + if badge != token: + return badge + key = "kind_" + token + label = t(key) + return token if label == key else label + + +def _card_item(item: Dict[str, Any], color: str, notes: Dict[str, Any], lang: str, mode: str, t: _T, *, delete_offered: bool) -> str: + iid = str(item.get("id", "")) + note = _note_for(notes, "items", [iid], lang) + path = str(item.get("path_portable") or "") + trash_paths = [str(p) for p in (item.get("trash_paths") or []) if p] + action = str(item.get("action") or ("trash" if trash_paths else "keep")) + name = "item-" + iid + choices: List[str] = [] + if color != "red" and trash_paths and action in ("trash", "delete"): + choices.append( + ''.format( + n=_e(name), v=_e(iid), label=t("opt_trash") + ) + ) + if color == "green" and delete_offered: + choices.append( + ''.format( + n=_e(name), v=_e(iid), p=_e(path), s=_e(format_bytes(item.get("size_bytes"))), label=t("opt_delete") + ) + ) + pick = "" + if choices: + pick = ''.format( + _e(name), _e(iid), _e(item.get("name")) + ) + badges = ['{0}'.format(_e(_kind_label(item.get("kind"), t)))] + if item.get("open_by"): + badges.append('{0}'.format(t("open_by"))) + what = note.get("what") or _pick(item.get("what"), lang) or _pick(item.get("note"), lang) + why = note.get("why") or _pick(item.get("why"), lang) + if_removed = note.get("if_removed") or _pick(item.get("if_removed"), lang) + restore = _pick(item.get("restore"), lang) + rows = [(t("what"), _e(what)), (t("why"), _e(why)), (t("if_removed"), _e(if_removed)), (t("restore"), _e(restore))] + body = [_path_line(path, t), _dl(rows)] + extra_note = _pick(item.get("note"), lang) + if extra_note and extra_note != what: + body.append('
    {0}: {1}
    '.format(t("hint"), _e(extra_note))) + if trash_paths and (len(trash_paths) > 1 or trash_paths[0] != path): + body.append( + '
    {0}
      {1}
    '.format( + t("trash_paths"), "".join('
  • {0}
    • '.format(_e(p)) for p in trash_paths) + ) + ) + kill = [str(p) for p in (item.get("kill_processes") or []) if p] + if kill: + body.append('
    {0}: {1}
    '.format(t("kill_processes"), _e(" · ".join(kill)))) + open_by = item.get("open_by") or [] + if open_by: + procs = ", ".join( + "{0} (pid {1})".format(_e(p.get("command")), _e(p.get("pid"))) if isinstance(p, dict) else _e(p) for p in open_by + ) + body.append('
    {0}: {1}
    '.format(t("open_by"), procs)) + commands = [str(c) for c in (item.get("commands") or []) if c] + if commands: + body.append('
    {0}
    '.format(t("commands"))) + body.extend(_copy_block(c, t) for c in commands) + controls = choices + [_reveal_button(path, mode, t)] + if any(controls): + body.append('
    {0}
    '.format("".join(controls))) + return _card( + ids=[iid], subject=iid, color=color, name=str(item.get("name") or path), badges=badges, + size=item.get("size_bytes"), pick=pick, body="".join(body), + ) + + +def _storage_buckets(data: Dict[str, Any], lang: str, mode: str, t: _T) -> Dict[str, List[str]]: + """Every whole-machine card, keyed by colour.""" + + items = [i for i in (data.get("items") or []) if isinstance(i, dict)] + notes = data.get("notes") or {} + capabilities = data.get("capabilities") or {} + delete_offered = bool(capabilities.get("permanent_delete_offered", True)) + buckets: Dict[str, List[str]] = {c: [] for c in COLORS} + for item in items: + color = item_color(item) + buckets[color].append(_card_item(item, color, notes, lang, mode, t, delete_offered=delete_offered)) + return buckets + + +def _storage_sections(data: Dict[str, Any], lang: str, mode: str, t: _T) -> Tuple[str, Dict[str, int]]: + buckets = _storage_buckets(data, lang, mode, t) + permanent = _permanent_switch(data, mode, t) + counts = {c: len(buckets[c]) for c in COLORS} + html = "".join( + [ + _section("green", "green", t("st_sec_green"), t("st_sec_green_desc"), buckets["green"], mode=mode, t=t, selectable=True, tools_extra=permanent), + _section("yellow", "yellow", t("st_sec_yellow"), t("st_sec_yellow_desc"), buckets["yellow"], mode=mode, t=t, selectable=True), + _section("red", "red", t("st_sec_red"), t("st_sec_red_desc"), buckets["red"], mode=mode, t=t, selectable=False), + ] + ) + return html, counts + + +def _disk_segments(disk: Dict[str, Any], items: List[Dict[str, Any]], primary: bool) -> List[Tuple[str, float]]: + mount = str(disk.get("mount") or "") + colored = {c: 0.0 for c in COLORS} + for item in items: + where = str(item.get("disk") or "") + if where == mount or (not where and primary): + colored[item_color(item)] += float(item.get("size_bytes") or 0) + used = float(disk.get("used_bytes") or 0) + total = float(disk.get("total_bytes") or 0) + free = float(disk.get("free_bytes") or max(0.0, total - used)) + other = max(0.0, used - sum(colored.values())) + return [("green", colored["green"]), ("yellow", colored["yellow"]), ("red", colored["red"]), ("other", other), ("free", free)] + + +def _priority_items(data: Dict[str, Any], lang: str) -> List[str]: + """``overview.priority`` as list items; the Agent's own order is kept.""" + + out = [] + for entry in (data.get("overview") or {}).get("priority") or []: + text = _pick(entry.get("text") if isinstance(entry, dict) else entry, lang) + if text: + out.append("
  • {0}
    • ".format(_e(text))) + return out + + +def _disk_bars(data: Dict[str, Any], t: _T) -> str: + """One bar per volume, used dividing into green / yellow / red / other / free.""" + + items = [i for i in (data.get("items") or []) if isinstance(i, dict)] + disks = [d for d in (data.get("disks") or []) if isinstance(d, dict)] + disk_html = [] + for index, disk in enumerate(disks): + primary = bool(disk.get("primary")) or (index == 0 and not any(d.get("primary") for d in disks)) + used = disk.get("used_bytes") + pct = disk.get("used_percent") + pct_text = "{0:.0f}%".format(float(pct)) if isinstance(pct, (int, float)) else "" + disk_html.append( + '
    {name} {mount}' + '{used} · {pct}
    {bar}' + '
    {lg}{ly}' + '{lr}{lo}{lf}
    '.format( + name=_e(disk.get("name") or disk.get("mount")), mount=_e(disk.get("mount") or ""), + used=_e(t("disk_used", used=format_bytes(used), total=format_bytes(disk.get("total_bytes")))), pct=pct_text, + bar=_segbar(_disk_segments(disk, items, primary)), + lg=t("legend_green"), ly=t("legend_yellow"), lr=t("legend_red"), lo=t("seg_other"), + lf=_e(t("disk_free", free=format_bytes(disk.get("free_bytes")))), + ) + ) + return "".join(disk_html) + + +def _storage_overview(data: Dict[str, Any], lang: str, mode: str, t: _T, counts: Dict[str, int]) -> str: + items = [i for i in (data.get("items") or []) if isinstance(i, dict)] + machine = data.get("machine") or {} + summary = data.get("summary") or {} + scan = data.get("scan") or {} + overview = data.get("overview") or {} + notes = data.get("notes") or {} + by_color = _normalise_by_color(summary.get("by_color"), _by_color_items(items)) + + sys_rows = [ + (t("sys_host"), machine.get("hostname")), + (t("sys_os"), " ".join(str(x) for x in (machine.get("os"), machine.get("arch")) if x)), + (t("sys_cpu"), machine.get("cpu")), + (t("sys_mem"), format_bytes(machine.get("memory_bytes")) if machine.get("memory_bytes") else None), + (t("sys_uptime"), t("sys_uptime_days", n=machine.get("uptime_days")) if machine.get("uptime_days") is not None else None), + (t("sys_scanned"), format_bytes(summary.get("scanned_bytes")) if summary.get("scanned_bytes") else None), + (t("sys_reclaim"), format_bytes(summary.get("reclaimable_bytes")) if summary.get("reclaimable_bytes") else None), + (t("sys_elapsed"), "{0:.0f} s".format(float(scan.get("elapsed_s"))) if isinstance(scan.get("elapsed_s"), (int, float)) else None), + (t("sys_denied"), ", ".join(str(x) for x in (scan.get("denied") or [])) or None), + ] + sys_html = "".join( + '
    {0}{1}
    '.format(_e(k), _e(v)) for k, v in sys_rows if v + ) + legend = "".join( + '{1} {2} · {3}'.format( + c, t("legend_" + c), by_color[c]["count"], format_bytes(by_color[c]["bytes"]) + ) + for c in COLORS + ) + headline = _pick(notes.get("headline"), lang) or _pick(overview.get("headline"), lang) + left = ( + '

    {dt}

    {disks}' + '

    {lt}

    {bar}
    {legend}
    {line}' + '

    {st}

    {sys}
    ' + ).format( + dt=t("disk_title"), disks=_disk_bars(data, t), lt=t("color_split"), bar=_segbar([(c, by_color[c]["bytes"]) for c in COLORS]), + legend=legend, line='

    {0}

    '.format(_e(headline)) if headline else "", st=t("sys_title"), sys=sys_html, + ) + li = _priority_items(data, lang) or _advice_items(counts, t) + right = '

    {0}

      {1}
    {2}
    '.format( + t("advice_title"), "".join(li), _pick_green_button(counts, t) + ) + return '
    {0}{1}
    '.format(left, right) + + +def _by_color_items(items: List[Dict[str, Any]]) -> Dict[str, Dict[str, int]]: + out = {c: {"count": 0, "bytes": 0} for c in COLORS} + for item in items: + c = item_color(item) + out[c]["count"] += 1 + out[c]["bytes"] += int(item.get("size_bytes") or 0) + return out + + +def _storage_top5(data: Dict[str, Any], lang: str, t: _T) -> str: + top = [x for x in (data.get("top5") or data.get("top") or []) if isinstance(x, dict)] + if not top: + return "" + by_id = {str(i.get("id")): i for i in (data.get("items") or []) if isinstance(i, dict)} + + def _size(entry: Dict[str, Any]) -> int: + item = by_id.get(str(entry.get("id")), {}) + return int(entry.get("size_bytes") or item.get("size_bytes") or 0) + + # The written order is whatever the Agent typed; the table promises the five + # largest, so it is sorted here rather than trusted. + top = sorted(top, key=_size, reverse=True) + rows = [] + for entry in top[:5]: + item = by_id.get(str(entry.get("id")), {}) + color = _color(entry.get("color"), item_color(item) if item else "yellow") + note = _pick(entry.get("note"), lang) or _pick(item.get("what"), lang) + rows.append( + '{size}{kind}' + '{name}{path}{note}'.format( + c=color, size=format_bytes(entry.get("size_bytes") or item.get("size_bytes")), + kind=_e(_kind_label(entry.get("kind") or item.get("kind"), t)), name=_e(entry.get("name") or item.get("name")), + path=_e(entry.get("path_portable") or item.get("path_portable") or ""), note=_e(note), + ) + ) + head = "".join("{0}".format(_e(t(k))) for k in ("th_color", "th_size", "th_kind", "th_name", "th_path", "th_note")) + return '

    {0}

    {1}{2}
    '.format( + t("top5_title"), head, "".join(rows) + ) + + +def _storage_longterm(data: Dict[str, Any], lang: str, t: _T) -> str: + overview = data.get("overview") or {} + notes = data.get("notes") or {} + lines = notes.get("long_term") if isinstance(notes.get("long_term"), list) else overview.get("long_term") or [] + li = "".join("
  • {0}
    • ".format(_e(_pick(x, lang))) for x in lines if _pick(x, lang)) + if not li: + return "" + return '

    {0}

      {1}
    '.format(t("longterm_title"), li) + + +def _storage_bar(data: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + machine = data.get("machine") or {} + cells = [ + (t("bar_machine"), '{0}'.format(_e(" · ".join(str(x) for x in (machine.get("hostname"), machine.get("os")) if x)))), + (t("bar_time"), '{0}'.format(_display_time(data.get("created_at") or ""))), + (t("bar_mode"), '{0}'.format(t("mode_" + mode))), + ] + return _bar(t("title_storage"), cells, mode, t) + + +def render_storage_body(data: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + sections, counts = _storage_sections(data, lang, mode, t) + return "".join( + [ + _storage_bar(data, lang, mode, t), + '
    ', + '

    {0}

    {1}

    '.format(t("heading_storage"), t("lede_storage")), + _storage_overview(data, lang, mode, t, counts), + _storage_top5(data, lang, t), + sections, + _storage_longterm(data, lang, t), + '

    {0}

    '.format(t("print_hint")), + "
    ", + _footer("storage", mode, t), + ] + ) + + +# -------------------------------------------------------------------------- +# combined page: one report, one entrance, one approval file +# -------------------------------------------------------------------------- + + +def _clean_combined( + data: Dict[str, Any], notes: Optional[Dict[str, Any]], home: Optional[Union[str, Path]] +) -> Dict[str, Any]: + """Redact the two halves separately, then put them back in one envelope. + + They cannot share a pass: the tidy-up half knows its own home directory + from ``source_root``/``source_root_portable``, the whole-machine half has no + such pair and falls back to this account's home. The Agent's notes are laid + over both, and each half reads only the tables it knows. + """ + + plan, analysis = split_combined(data) + out: Dict[str, Any] = {} + if plan: + out["plan"] = sanitize(merge_notes(plan, notes), home) + if analysis: + out["analysis"] = sanitize(merge_notes(analysis, notes), home) + lang = data.get("lang") or (plan.get("lang") if plan else None) or (analysis.get("lang") if analysis else None) + if lang: + out["lang"] = lang + capabilities: Dict[str, Any] = {} + for half in (plan, analysis): + capabilities.update(half.get("capabilities") or {}) + if capabilities: + out["capabilities"] = capabilities + executed = data.get("executed_ids") + out["executed_ids"] = list(executed) if isinstance(executed, list) else [] + return out + + +def split_combined(data: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """``{"plan": ..., "analysis": ...}`` -> the two halves, either of them empty. + + A caller may hand over only one half; the page then simply has one block + fewer. Anything that is not a dict is treated as absent rather than as an + error, because a half-written analysis.json should still let the tidy-up + half render. + """ + + plan = data.get("plan") + analysis = data.get("analysis") + return (plan if isinstance(plan, dict) else {}), (analysis if isinstance(analysis, dict) else {}) + + +def _combined_bar(plan: Dict[str, Any], analysis: Dict[str, Any], mode: str, t: _T) -> str: + machine = analysis.get("machine") or {} + cells = [] + if plan: + cells.append((t("bar_dir"), '{0}'.format(_e(plan.get("source_root_portable") or "")))) + if analysis: + cells.append( + ( + t("bar_machine"), + '{0}'.format( + _e(" · ".join(str(x) for x in (machine.get("hostname"), machine.get("os")) if x)) + ), + ) + ) + when = plan.get("created_at") or plan.get("now") or analysis.get("created_at") or "" + cells.append((t("bar_time"), '{0}'.format(_display_time(when)))) + cells.append((t("bar_mode"), '{0}'.format(t("mode_" + mode)))) + return _bar(t("title_combined"), cells, mode, t) + + +def _combined_overview( + plan: Dict[str, Any], analysis: Dict[str, Any], lang: str, mode: str, t: _T, counts: Dict[str, int] +) -> str: + """Left: the disks and the mess score. Right: one numbered list of what to do first.""" + + left_parts = [] + if analysis.get("disks"): + left_parts.append("

    {0}

    {1}".format(t("disk_title"), _disk_bars(analysis, t))) + if plan: + left_parts.append( + '
    {0}
    '.format(_mess_inner(plan, lang, t)) + if left_parts + else _mess_inner(plan, lang, t) + ) + headline = "" + if analysis: + notes = analysis.get("notes") or {} + headline = _pick(notes.get("headline"), lang) or _pick((analysis.get("overview") or {}).get("headline"), lang) + if headline: + left_parts.append('

    {0}

    '.format(_e(headline))) + left = '
    {0}
    '.format("".join(left_parts)) + + items = _priority_items(analysis, lang) if analysis else [] + items.extend(_advice_items(counts, t)) + if not items: + items.append("
  • {0}
    • ".format(_e(t("advice_none")))) + folder_line = _pick((plan.get("notes") or {}).get("folder_line"), lang) if plan else "" + right = '

    {title}

    {folder}
      {items}
    {tools}
    '.format( + title=t("advice_title"), + folder='

    {0}

    '.format(_e(folder_line)) if folder_line else "", + items="".join(items), + tools=_pick_green_button(counts, t), + ) + return '
    {0}{1}
    '.format(left, right) + + +def _combined_sections( + plan: Dict[str, Any], analysis: Dict[str, Any], lang: str, mode: str, t: _T +) -> Tuple[str, Dict[str, int]]: + """Three bands, each one cleanup first and moves second.""" + + clean = _storage_buckets(analysis, lang, mode, t) if analysis else {c: [] for c in COLORS} + move = _organize_buckets(plan, lang, mode, t) if plan else {c: [] for c in COLORS} + permanent = _permanent_switch(analysis or plan, mode, t) + counts = {c: len(clean[c]) + len(move[c]) for c in COLORS} + titles = {"green": "cb_green", "yellow": "cb_yellow", "red": "cb_red"} + html = [] + for color in COLORS: + html.append( + _section( + color, + color, + t(titles[color]), + t(titles[color] + "_desc"), + [], + mode=mode, + t=t, + selectable=color != "red", + tools_extra=permanent if color == "green" else "", + subgroups=[(t("sub_clean"), clean[color]), (t("sub_move"), move[color])], + ) + ) + return "".join(html), counts + + +def _preview_block(plan: Dict[str, Any], analysis: Dict[str, Any], lang: str, t: _T) -> str: + """The after picture, rendered by ``carl_file_organizer.preview``. + + The fragment carries its own style and script and loads nothing, so it is + dropped in verbatim. A preview that cannot be built is not worth failing a + report over, so a broken one leaves the section out entirely. + """ + + try: + if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + from carl_file_organizer import preview as preview_module + except Exception: # noqa: BLE001 - a missing preview costs the page one block + return "" + try: + graph = preview_module.build_graph(plan or {}, analysis or None) + fragment = preview_module.render_preview_html(graph, lang=lang, height=720) + except Exception: # noqa: BLE001 - same reasoning + return "" + return ( + '
    ' + '

    {0}

    {1}

    ' + "{2}
    " + ).format(_e(t("preview_title")), _e(t("preview_note")), fragment) + + +def _combined_longterm(plan: Dict[str, Any], analysis: Dict[str, Any], lang: str, t: _T) -> str: + blocks = [] + if analysis: + blocks.append(_storage_longterm(analysis, lang, t)) + if plan: + blocks.append(_organize_longterm(plan, lang, t)) + return "".join(b for b in blocks if b) + + +def render_combined_body(data: Dict[str, Any], lang: str, mode: str, t: _T) -> str: + plan, analysis = split_combined(data) + sections, counts = _combined_sections(plan, analysis, lang, mode, t) + return "".join( + [ + _combined_bar(plan, analysis, mode, t), + '
    ', + '

    {0}

    {1}

    '.format( + t("heading_combined"), t("lede_combined") + ), + _combined_overview(plan, analysis, lang, mode, t, counts), + _storage_top5(analysis, lang, t) if analysis else "", + sections, + _preview_block(plan, analysis, lang, t), + _combined_longterm(plan, analysis, lang, t), + '

    {0}

    '.format(t("print_hint")), + "
    ", + _footer("combined", mode, t), + ] + ) + + +# -------------------------------------------------------------------------- +# public API +# -------------------------------------------------------------------------- + + +def render( + data: Dict[str, Any], + *, + mode: str = "static", + token: str = "", + lang: Optional[str] = None, + notes: Optional[Dict[str, Any]] = None, + kind: Optional[str] = None, + template_path: Optional[Union[str, Path]] = None, + home: Optional[Union[str, Path]] = None, +) -> str: + """The finished page for ``data`` (a plan.json or an analysis.json dict).""" + + if mode not in MODES: + raise ValueError("mode must be one of {0}".format(", ".join(MODES))) + kind = kind or detect_kind(data) + if kind not in KINDS: + raise ValueError("kind must be one of {0}".format(", ".join(KINDS))) + template = load_template(template_path) + text = template_text(template) + if kind == "combined": + clean = _clean_combined(data, notes, home) + else: + clean = sanitize(merge_notes(data, notes), home) + lang = _lang_of(clean, lang) + t = _T(text, lang) + if mode != "serve": + token = "" + capabilities = dict(clean.get("capabilities") or {}) + config = { + "kind": kind, + "mode": mode, + "token": token, + "lang": lang, + "capabilities": capabilities, + "generator": "build_report.py {0}".format(__version__), + } + if kind == "combined": + body = render_combined_body(clean, lang, mode, t) + elif kind == "organize": + body = render_organize_body(clean, lang, mode, t) + else: + body = render_storage_body(clean, lang, mode, t) + mapping = { + "LANG": "zh-CN" if lang == "zh" else "en", + "TITLE": _e(t("title_" + kind)), + "KIND": kind, + "MODE": mode, + "DATA": _embed_json(clean), + "CONFIG": _embed_json(config), + "BODY": body, + } + return re.sub(r"__REPORT_(LANG|TITLE|KIND|MODE|DATA|CONFIG|BODY)__", lambda m: mapping[m.group(1)], template) + + +def _read_json(path: Union[str, Path]) -> Dict[str, Any]: + with open(str(path), "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Render the three-colour report page.") + parser.add_argument("input", help="plan.json or analysis.json") + parser.add_argument("--notes", help="notes.json written by the Agent (what / why / if_removed per id)") + parser.add_argument("-o", "--output", help="where to write report.html (default: next to the input)") + parser.add_argument("--mode", choices=MODES, default="static") + parser.add_argument("--token", default="", help="serve-mode token embedded in the page") + parser.add_argument("--lang", choices=LANGS, default=None) + parser.add_argument("--kind", choices=KINDS, default=None, help="override the auto-detected page kind") + args = parser.parse_args(argv) + + data = _read_json(args.input) + notes = _read_json(args.notes) if args.notes else None + html = render(data, mode=args.mode, token=args.token, lang=args.lang, notes=notes, kind=args.kind) + output = Path(args.output) if args.output else Path(args.input).resolve().parent / "report.html" + output.write_text(html, encoding="utf-8") + sys.stdout.write("{0}\n".format(output)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py new file mode 100644 index 0000000..a4bf366 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__init__.py @@ -0,0 +1,3 @@ +"""Carl File Organizer: human-first file organization.""" + +__version__ = "0.3.0" diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py new file mode 100644 index 0000000..a049ad7 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py new file mode 100644 index 0000000..31f7ec9 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/classifier.py @@ -0,0 +1,298 @@ +"""Rule matching: extensions, sensitive naming, pinned names, derived copies. + +Two eras live side by side here. ``CATEGORY_EXTENSIONS`` and :func:`classify` +are the original small taxonomy and stay exactly as they were, because the +``simple`` profile and the first test file are built on them. Everything else +is profile driven and reads its data out of a :class:`~carl_file_organizer.config.Config`. + +The sensitive check is the part worth reading twice. It never opens a file; it +only looks at the name, in four passes of decreasing certainty: + + exact_names the whole name, compared case-insensitively + exact_stems the name minus its suffix, compared as a whole word + suffixes the suffix alone + markers substring, and only for long or non-ASCII words + +Short words are never substring matched. That is the fix for the day a rule +containing ``r2`` swallowed ``pixverse-r2.md``. +""" + +from __future__ import annotations + +import fnmatch +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Set + +from .scanner import split_suffix + + +CATEGORY_EXTENSIONS: Dict[str, Set[str]] = { + "Documents": { + ".doc", + ".docx", + ".epub", + ".key", + ".md", + ".numbers", + ".pages", + ".pdf", + ".ppt", + ".pptx", + ".rtf", + ".txt", + ".xls", + ".xlsx", + }, + "Images": { + ".avif", + ".gif", + ".heic", + ".jpeg", + ".jpg", + ".png", + ".svg", + ".tif", + ".tiff", + ".webp", + }, + "Video": {".avi", ".m4v", ".mkv", ".mov", ".mp4", ".webm"}, + "Audio": {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".wav"}, + "Archives": { + ".7z", + ".bz2", + ".gz", + ".rar", + ".tar", + ".tgz", + ".xz", + ".zip", + }, + "Installers": {".apk", ".dmg", ".exe", ".iso", ".msi", ".pkg"}, + "Code & Data": { + ".c", + ".cpp", + ".css", + ".csv", + ".go", + ".html", + ".ipynb", + ".java", + ".js", + ".json", + ".jsx", + ".py", + ".rb", + ".rs", + ".sql", + ".swift", + ".toml", + ".ts", + ".tsx", + ".xml", + ".yaml", + ".yml", + }, +} + + +def classify(path: Path) -> tuple: + """Return a human-readable category, reason, and confidence.""" + suffix = path.suffix.lower() + for category, extensions in CATEGORY_EXTENSIONS.items(): + if suffix in extensions: + return category, "{0} file".format(suffix or "extensionless"), 0.98 + return "Other", "file type is not in the small default taxonomy", 0.55 + + +# --------------------------------------------------------------------------- +# profile driven matching +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Rule: + key: str # the suffix that matched, e.g. ".pdf" + dest_key: str + min_age_hours: int + reason_key: str + label: Optional[Dict[str, str]] = None + + +@dataclass(frozen=True) +class SensitiveHit: + layer: str # "exact" | "suffix" | "marker" -> becomes rule "sensitive:" + needle: str + where: str # "exact_names" | "exact_stems" | "suffixes" | "markers" + + +@dataclass(frozen=True) +class NamePattern: + id: str + match: str # "contains" | "prefix" | "glob" + needle: str + dest: Optional[str] # literal path relative to root + dest_key: Optional[str] # or a key in profile names + + +def _profile(config: Any) -> Mapping[str, Any]: + return getattr(config, "profile", config) + + +def _section(config: Any, name: str) -> Mapping[str, Any]: + section = _profile(config).get(name) + return section if isinstance(section, Mapping) else {} + + +def _list(config: Any, section: str, key: str) -> Sequence[Any]: + values = _section(config, section).get(key) + return values if isinstance(values, (list, tuple)) else () + + +def match_rule(suffix: str, config: Any) -> Optional[Rule]: + """Look a suffix up in the profile rule table.""" + + rules = _section(config, "rules") + folded = str(suffix).casefold() + entry = rules.get(folded) + if not isinstance(entry, Mapping): + return None + aging = _section(config, "aging") + default_hours = int(aging.get("default_hours", 48) or 0) + minimum = entry.get("min_age_hours") + return Rule( + key=folded, + dest_key=str(entry.get("dest")), + min_age_hours=int(default_hours if minimum is None else minimum), + reason_key=str(entry.get("reason") or "ext_other"), + label=entry.get("label") if isinstance(entry.get("label"), Mapping) else None, + ) + + +def is_sensitive(name: str, config: Any) -> Optional[SensitiveHit]: + """Name-only credential detection. Contents are never read.""" + + text = str(name) + folded = text.casefold() + stem, suffix = split_suffix(text) + folded_stem = stem.casefold() + + for candidate in _list(config, "sensitive", "exact_names"): + if folded == str(candidate).casefold(): + return SensitiveHit("exact", str(candidate), "exact_names") + + for candidate in _list(config, "sensitive", "exact_stems"): + if folded_stem == str(candidate).casefold(): + return SensitiveHit("exact", str(candidate), "exact_stems") + + for candidate in _list(config, "sensitive", "suffixes"): + needle = str(candidate).casefold() + if suffix and suffix == needle: + return SensitiveHit("suffix", str(candidate), "suffixes") + # A dot-prefixed suffix such as ".env" also matches "prod.env" when the + # name carries no suffix of its own. + if not suffix and folded.endswith(needle) and folded != needle: + return SensitiveHit("suffix", str(candidate), "suffixes") + + minimum = int(_section(config, "sensitive").get("marker_min_len", 5) or 5) + for candidate in _list(config, "sensitive", "markers"): + marker = str(candidate) + ascii_only = all(ord(char) < 128 for char in marker) + if ascii_only and len(marker) < minimum: + # Refused at load time; skipped here so a hand-edited profile in the + # wild degrades to "missed" rather than "matched everything". + continue + if marker.casefold() in folded: + return SensitiveHit("marker", marker, "markers") + return None + + +def derivative_hint(name: str, config: Any) -> Optional[str]: + """Return the hint substring when a name looks like a scratch copy.""" + + folded = str(name).casefold() + for hint in _profile(config).get("derivative_hints") or (): + needle = str(hint).casefold() + if needle and needle in folded: + return str(hint) + return None + + +def is_regenerable_name(name: str, config: Any) -> bool: + basenames = _list(config, "regenerable", "basenames") + return any(str(name) == str(candidate) for candidate in basenames) + + +def regenerable_restore(name: str, config: Any) -> Dict[str, str]: + restore = _section(config, "regenerable").get("restore") + table = restore if isinstance(restore, Mapping) else {} + entry = table.get(str(name)) or table.get("_default") or {} + if isinstance(entry, Mapping): + return {"zh": str(entry.get("zh", "")), "en": str(entry.get("en", ""))} + return {"zh": str(entry), "en": str(entry)} + + +def match_pinned(name: str, config: Any) -> Optional[str]: + """Exact name or fnmatch glob against the pinned allow list.""" + + text = str(name) + for pattern in _profile(config).get("pinned") or (): + candidate = str(pattern) + if text == candidate or fnmatch.fnmatch(text, candidate): + return candidate + return None + + +def match_name_pattern(name: str, config: Any) -> Optional[NamePattern]: + """User written name rules; substring matching needs at least four characters.""" + + text = str(name) + folded = text.casefold() + _stem, suffix = split_suffix(text) + for raw in _profile(config).get("name_patterns") or (): + if not isinstance(raw, Mapping): + continue + needle = str(raw.get("needle") or "") + if not needle: + continue + suffixes = raw.get("suffixes") + if isinstance(suffixes, (list, tuple)) and suffixes: + allowed = {str(item).casefold() for item in suffixes} + if suffix not in allowed: + continue + how = str(raw.get("match") or "contains") + folded_needle = needle.casefold() + if how == "prefix": + hit = folded.startswith(folded_needle) + elif how == "glob": + hit = fnmatch.fnmatch(text, needle) + else: + hit = len(folded_needle) >= 4 and folded_needle in folded + if not hit: + continue + return NamePattern( + id=str(raw.get("id") or needle), + match=how, + needle=needle, + dest=str(raw["dest"]) if raw.get("dest") else None, + dest_key=str(raw["dest_key"]) if raw.get("dest_key") else None, + ) + return None + + +def is_archive_suffix(suffix: str, config: Any) -> bool: + folded = str(suffix).casefold() + return any( + folded == str(item).casefold() for item in _profile(config).get("archive_suffixes") or () + ) + + +def is_installer_suffix(suffix: str, config: Any) -> bool: + folded = str(suffix).casefold() + return any( + folded == str(item).casefold() for item in _profile(config).get("installer_suffixes") or () + ) + + +def forbidden_markers(config: Any) -> List[str]: + return [str(item) for item in _list(config, "forbidden", "markers")] diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py new file mode 100644 index 0000000..a4b90ed --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/cli.py @@ -0,0 +1,373 @@ +"""Command-line interface: argument parsing and dispatch only. + +Every subcommand resolves to ``.run(args)``. No business logic lives +here, so the subcommands and their flag names are frozen from day zero and each +work package only fills in its own module. + +Three of them are the road most people walk: + + scan -> carl_file_organizer.flow.run one read-only pass, both halves + report -> carl_file_organizer.flow.run one page out of one folder + apply -> carl_file_organizer.flow.run one decisions file, both halves + +The rest are the same work split into steps, and every one of them still writes +the same files it always did: + + plan -> carl_file_organizer.planner.run + build -> carl_file_organizer.notes.run + review -> carl_file_organizer.server.run + undo -> carl_file_organizer.undo.run + status -> carl_file_organizer.status.run + clear-tags -> carl_file_organizer.tags.run + storage-report -> carl_file_organizer.render.run + dispose -> carl_file_organizer.dispose.run + +``apply`` still reads an older approved.json and an older storage decisions +file; ``flow`` recognises them and hands them to the module that owns them. + +A module that has no ``run`` yet produces a one-line message and exit code 2, +never an ImportError traceback. +""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.util +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, List, Optional + +from . import __version__, i18n + +#: subcommand name -> module inside this package that owns it +COMMAND_MODULES = { + "scan": "flow", + "report": "flow", + "plan": "planner", + "build": "notes", + "apply": "flow", + "review": "server", + "undo": "undo", + "status": "status", + "clear-tags": "tags", + "storage-report": "render", + "dispose": "dispose", +} + + +def _iso(text: str) -> datetime: + try: + return datetime.fromisoformat(text) + except ValueError as error: + raise argparse.ArgumentTypeError( + "--now must be an ISO 8601 timestamp, for example 2026-09-02T10:15:00" + ) from error + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="carl-file-organizer", + description="Human-first file organization with explicit approval.", + epilog=( + "Three steps do the job: scan , then report , then " + "apply --dry-run. The other subcommands are those " + "same steps taken apart, for when you want one piece on its own." + ), + ) + parser.add_argument("--version", action="version", version=__version__) + subparsers = parser.add_subparsers(dest="command", required=True) + + # -- scan --------------------------------------------------------------- + scan = subparsers.add_parser( + "scan", help="read-only pass over the folder and the machine (step 1 of 3)" + ) + scan.add_argument("source", nargs="?", type=Path, default=Path.home() / "Downloads") + scan.add_argument("--lang", choices=["zh", "en"], default=None) + scan.add_argument("--profile", choices=["tiered", "simple"], default=None) + scan.add_argument( + "--no-storage", + dest="storage", + action="store_false", + help="skip the whole-machine inventory and only plan the folder", + ) + scan.add_argument("--storage", dest="storage", action="store_true", help=argparse.SUPPRESS) + scan.add_argument( + "--budget-seconds", + type=float, + default=60.0, + help="hard ceiling on the whole-machine scan (default 60)", + ) + scan.add_argument("--home", default=None, help="the home directory to inventory (default: this account)") + scan.add_argument("--hash-duplicates", action="store_true") + scan.add_argument("--large-mb", type=int, default=500) + scan.add_argument("--target-percent", type=float, default=90.0) + scan.add_argument("--no-dir-sizes", dest="dir_sizes", action="store_false") + scan.add_argument("--no-permanent-delete-options", dest="permanent_delete_options", action="store_false") + scan.add_argument("--now", type=_iso, default=None, help="fix the clock (tests and replays)") + scan.add_argument("--allow-outside-home", action="store_true") + scan.add_argument("--allow-referenced", action="store_true") + scan.set_defaults(storage=True, dir_sizes=True, permanent_delete_options=True) + + # -- report ------------------------------------------------------------- + report = subparsers.add_parser( + "report", help="one page out of one managed directory (step 2 of 3)" + ) + report.add_argument("managed_dir", type=Path, help="the folder scan wrote plan.json into") + report.add_argument("--notes", type=Path, default=None, help="notes.json (default: the one in that folder)") + report.add_argument("--analysis", type=Path, default=None, help="analysis.json (default: the one in that folder)") + report.add_argument("-o", "--output", type=Path, default=None, help="where to write report.html") + report.add_argument("--serve", action="store_true", help="serve the page instead of writing it") + report.add_argument("--port", type=int, default=0) + report.add_argument("--no-open", dest="open_browser", action="store_false") + report.add_argument("--allow-permanent-delete", action="store_true") + report.add_argument("--lang", choices=["zh", "en"], default=None) + report.set_defaults(open_browser=True) + + # -- plan --------------------------------------------------------------- + plan = subparsers.add_parser("plan", help="scan read-only and write a review page") + plan.add_argument( + "source", + nargs="?", + type=Path, + default=Path.home() / "Downloads", + help="folder to scan (default: ~/Downloads)", + ) + plan.add_argument("--lang", choices=["zh", "en"], default=None) + plan.add_argument("--profile", choices=["tiered", "simple"], default=None) + plan.add_argument("--hash-duplicates", action="store_true") + plan.add_argument("--large-mb", type=int, default=500) + plan.add_argument("--target-percent", type=float, default=90.0) + plan.add_argument("--no-dir-sizes", dest="dir_sizes", action="store_false") + plan.add_argument( + "--no-permanent-delete-options", + dest="permanent_delete_options", + action="store_false", + ) + plan.add_argument("--now", type=_iso, default=None, help="fix the clock (tests and replays)") + plan.add_argument("--allow-outside-home", action="store_true") + plan.add_argument( + "--allow-referenced", + action="store_true", + help="downgrade referenced-elsewhere holds to a warning", + ) + plan.set_defaults(dir_sizes=True, permanent_delete_options=True) + + # -- build -------------------------------------------------------------- + build = subparsers.add_parser( + "build", help="lay an agent's notes.json over a plan.json" + ) + build.add_argument("plan", type=Path, help="the plan.json to annotate") + build.add_argument( + "--notes", + type=Path, + default=None, + help="notes.json written by the agent; without it the plan is only rewritten", + ) + build.add_argument( + "--output", + type=Path, + default=None, + help="where to write the annotated plan (default: in place)", + ) + build.add_argument( + "--report", + action="store_true", + help="also re-render report.html next to the output plan", + ) + build.add_argument("--lang", choices=["zh", "en"], default=None) + + # -- apply -------------------------------------------------------------- + apply_cmd = subparsers.add_parser( + "apply", help="act on one decisions file, moves first then cleanup (step 3 of 3)" + ) + apply_cmd.add_argument("approval", type=Path, help="carl-file-organizer-decisions.json (or an older approved.json)") + apply_cmd.add_argument("--dry-run", action="store_true") + apply_cmd.add_argument("--allow-permanent-delete", action="store_true") + apply_cmd.add_argument("--audit", type=Path, default=None) + apply_cmd.add_argument("--plan", type=Path, default=None, help="plan.json, when the decisions file carries none") + apply_cmd.add_argument("--analysis", type=Path, default=None, help="analysis.json, when the decisions file carries none") + apply_cmd.add_argument("--managed-dir", type=Path, default=None, help="where to look for those two, and where the paper trail lands") + apply_cmd.add_argument("--lang", choices=["zh", "en"], default=None) + + # -- review ------------------------------------------------------------- + review = subparsers.add_parser("review", help="open the review page") + review.add_argument("plan", type=Path) + review.add_argument("--serve", action="store_true", required=True) + review.add_argument("--port", type=int, default=0) + review.add_argument("--no-open", dest="open_browser", action="store_false") + review.add_argument("--allow-permanent-delete", action="store_true") + review.set_defaults(open_browser=True) + + # -- undo --------------------------------------------------------------- + undo = subparsers.add_parser("undo", help="move things back where they came from") + undo.add_argument("record", type=Path, help="audit.jsonl or a manifest .tsv") + undo.add_argument("--dry-run", action="store_true") + + # -- status ------------------------------------------------------------- + status = subparsers.add_parser("status", help="last tidy-up and current mess") + status.add_argument( + "source", nargs="?", type=Path, default=Path.home() / "Downloads" + ) + status.add_argument("--lang", choices=["zh", "en"], default=None) + status.add_argument("--allow-outside-home", action="store_true") + + # -- storage-report ----------------------------------------------------- + storage_report = subparsers.add_parser( + "storage-report", help="render the whole-machine page from an analysis.json" + ) + storage_report.add_argument("analysis", type=Path, help="analysis.json written by the agent") + storage_report.add_argument( + "--notes", + type=Path, + default=None, + help="notes.json laid over the analysis before rendering", + ) + storage_report.add_argument( + "-o", + "--output", + type=Path, + default=None, + help="where to write report.html (default: next to the analysis)", + ) + storage_report.add_argument( + "--serve", + action="store_true", + help="serve the page on 127.0.0.1 with one-click disposal instead of writing a file", + ) + storage_report.add_argument("--port", type=int, default=0) + storage_report.add_argument("--no-open", dest="open_browser", action="store_false") + storage_report.add_argument("--allow-permanent-delete", action="store_true") + storage_report.add_argument("--lang", choices=["zh", "en"], default=None) + storage_report.set_defaults(open_browser=True) + + # -- dispose ------------------------------------------------------------ + dispose = subparsers.add_parser( + "dispose", help="act on the decisions exported from the whole-machine page" + ) + dispose.add_argument("analysis", type=Path, help="the analysis.json those decisions came from") + dispose.add_argument("decisions", type=Path, help="carl-file-organizer-decisions.json") + dispose.add_argument("--dry-run", action="store_true") + dispose.add_argument("--allow-permanent-delete", action="store_true") + dispose.add_argument("--lang", choices=["zh", "en"], default=None) + + # -- clear-tags --------------------------------------------------------- + clear = subparsers.add_parser("clear-tags", help="remove the temporary Finder tag") + clear.add_argument( + "source", nargs="?", type=Path, default=Path.home() / "Downloads" + ) + clear.add_argument("--from", dest="from_lists", action="append", type=Path, default=None) + clear.add_argument("--all-lists", action="store_true") + clear.add_argument("--tag", default=None) + clear.add_argument("--dry-run", action="store_true") + + return parser + + +def _module_run(module_name: str) -> Optional[Callable[[Any], int]]: + """Return ``module.run`` when it exists, without letting ImportError escape.""" + + full_name = "{0}.{1}".format(__package__, module_name) + try: + if importlib.util.find_spec(full_name) is None: + return None + except (ImportError, ValueError): + return None + module = importlib.import_module(full_name) + run = getattr(module, "run", None) + return run if callable(run) else None + + +def _forgiving_console() -> None: + """Never let a code page turn a finished job into a failure. + + Every line this tool prints can carry Chinese, and on Windows a redirected + stdout is a legacy code page rather than UTF-8: ``print`` then raises + UnicodeEncodeError, the CLI boundary below catches it, and a build that + wrote its plan correctly reports exit code 2. The text is the report, not + the deliverable, so unrepresentable characters get escaped and the run goes + on. A console that can spell them is untouched. + """ + + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: # a stream somebody replaced, in a test or a host + continue + try: + reconfigure(errors="backslashreplace") + except (ValueError, OSError): + continue + + +def _not_implemented(command: str, module_name: str, lang: str) -> int: + print( + "error: " + + i18n.t( + "cli_not_implemented", + lang, + command=command, + module="{0}.{1}".format(__package__, module_name), + ), + file=sys.stderr, + ) + return 2 + + +def _prepare_config(args: argparse.Namespace) -> Any: + """Shared front half of plan/status: refuse bad roots, resolve settings.""" + + from . import config as config_module + from . import paths + + root = paths.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + cfg = config_module.resolve_config(root, lang=args.lang, profile=getattr(args, "profile", None)) + for message in cfg.warnings: + print(i18n.t("warning_prefix", cfg.lang) + message, file=sys.stderr) + return cfg + + +def main(argv: Optional[List[str]] = None) -> int: + _forgiving_console() + parser = build_parser() + args = parser.parse_args(argv) + command = args.command + module_name = COMMAND_MODULES[command] + lang = getattr(args, "lang", None) or "en" + + try: + if command in ("plan", "scan"): + from . import config as config_module + + cfg = _prepare_config(args) + lang = cfg.lang + written = config_module.save_dotfile(cfg, only_if_missing=True) + if written is not None: + print(i18n.t("cli_dotfile_written", cfg.lang, path=written)) + print(i18n.t("cli_lang_source", cfg.lang, lang=cfg.lang, source=cfg.lang_source)) + args.config = cfg + run = _module_run(module_name) + if run is None: + return _not_implemented(command, module_name, cfg.lang) + return int(run(args)) + + if command == "status": + cfg = _prepare_config(args) + lang = cfg.lang + args.config = cfg + + run = _module_run(module_name) + if run is None: + return _not_implemented(command, module_name, lang) + return int(run(args)) + except KeyboardInterrupt: + return 130 + except Exception as error: # noqa: BLE001 - CLI boundary: one clean line, never a traceback + print("error: {0}".format(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py new file mode 100644 index 0000000..caaceb3 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/config.py @@ -0,0 +1,544 @@ +"""Language detection, profile loading and merging, per-folder settings file. + +Everything downstream reads a single :class:`Config`. The contract frozen here: + + resolve_config(root, *, lang=None, profile=None) -> Config + dir_name(config, key) -> "20_知识库/文档资料/PDF" + dest_dir(config, key) -> Path + managed_dir(config) -> Path + partition_prefixes(config) -> ("00_", "10_", ...) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from . import paths + +DOTFILE_NAME = ".carl-file-organizer.json" +DEFAULT_PROFILE = "tiered" +LANG_ENV = "CARL_FILE_ORGANIZER_LANG" +LANGS = ("zh", "en") +MAX_NAME_DEPTH = 3 + +#: Keys whose list values are unioned when a dotfile overrides a built-in profile. +UNION_LIST_KEYS = ( + ("pinned",), + ("derivative_hints",), + ("archive_suffixes",), + ("installer_suffixes",), + ("sensitive", "exact_names"), + ("sensitive", "exact_stems"), + ("sensitive", "suffixes"), + ("sensitive", "markers"), + ("forbidden", "basenames"), + ("forbidden", "suffixes"), + ("forbidden", "markers"), + ("forbidden", "glob"), + ("regenerable", "basenames"), + ("guard", "reference_files"), + ("guard", "reference_scan_roots"), + ("dedupe", "copy_patterns"), +) + +#: Sections merged key by key, with the dotfile value winning per key. +SCALAR_SECTIONS = ("aging", "tags", "dedupe", "guard", "regenerable", "sensitive") + +#: Top-level keys the dotfile simply overwrites. +OVERRIDE_KEYS = ("lang", "profile", "large_bytes", "managed_dir", "transient_tag", "review_log") + +#: Settings that record a choice rather than a rule, and so stay out of the digest. +HASH_IGNORED_KEYS = ("lang",) + + +class LanguageAmbiguous(ValueError): + """Both language layouts exist in the same folder and no language was given.""" + + +@dataclass +class Config: + root: Path + lang: str + lang_source: str + profile_name: str + profile: Dict[str, Any] + profile_source: str + dotfile_path: Optional[Path] + profile_hash: str + warnings: List[str] = field(default_factory=list) + + +# -------------------------------------------------------------------------- +# language detection +# -------------------------------------------------------------------------- + + +def normalize_lang(text: Optional[str]) -> Optional[str]: + if not text: + return None + folded = str(text).strip().casefold().replace("_", "-") + if folded.startswith("zh") or "hans" in folded or "hant" in folded: + return "zh" + if folded.startswith("en"): + return "en" + return None + + +def _apple_languages(timeout_s: float = 1.0) -> Optional[str]: + """First entry of ``defaults read -g AppleLanguages``; ``None`` when unavailable.""" + + if sys.platform != "darwin": + return None + try: + result = subprocess.run( + ["defaults", "read", "-g", "AppleLanguages"], + capture_output=True, + text=True, + timeout=timeout_s, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + match = re.search(r'"?([A-Za-z]{2}[A-Za-z0-9\-_]*)"?', result.stdout) + return match.group(1) if match else None + + +def _posix_locale(env: Mapping[str, str]) -> Optional[str]: + for key in ("LC_ALL", "LC_MESSAGES", "LANG"): + value = env.get(key) + if value: + return value.split(".")[0] + return None + + +def _sniff_dir_names(profile: Mapping[str, Any]) -> Dict[str, List[str]]: + """Directory names that betray which language layout a folder already uses.""" + + probes: Dict[str, List[str]] = {"zh": [], "en": []} + names = profile.get("names") or {} + candidates = [names.get("inbox"), profile.get("managed_dir")] + for entry in candidates: + if not isinstance(entry, Mapping): + continue + zh_name = entry.get("zh") + en_name = entry.get("en") + if not zh_name or not en_name or zh_name == en_name: + continue + probes["zh"].append(str(zh_name)) + probes["en"].append(str(en_name)) + return probes + + +def _sniff_existing_dirs(root: Path, profile: Mapping[str, Any]) -> List[str]: + found: List[str] = [] + probes = _sniff_dir_names(profile) + for lang in LANGS: + for name in probes[lang]: + try: + if (Path(root) / name).is_dir(): + found.append(lang) + break + except OSError: + continue + return found + + +def detect_lang( + root: Path, + override: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + dotfile: Optional[Mapping[str, Any]] = None, + profile: Optional[Mapping[str, Any]] = None, +) -> Tuple[str, str]: + """Return ``(lang, source)``. + + Chain: flag > dotfile > ``CARL_FILE_ORGANIZER_LANG`` > existing directories > + AppleLanguages > POSIX locale > ``en``. + """ + + environ = os.environ if env is None else env + + if override: + chosen = normalize_lang(override) + if chosen is None: + raise ValueError("unsupported language: {0}".format(override)) + return chosen, "flag" + + settings = dotfile if dotfile is not None else (load_dotfile(root) or {}) + chosen = normalize_lang(settings.get("lang") if isinstance(settings, Mapping) else None) + if chosen: + return chosen, "dotfile" + + chosen = normalize_lang(environ.get(LANG_ENV)) + if chosen: + return chosen, "env" + + if profile: + found = _sniff_existing_dirs(root, profile) + if len(found) > 1: + raise LanguageAmbiguous( + "both language layouts exist in {0}; pass --lang zh or --lang en".format(root) + ) + if len(found) == 1: + return found[0], "existing-dirs" + + chosen = normalize_lang(_apple_languages()) + if chosen: + return chosen, "apple-languages" + + chosen = normalize_lang(_posix_locale(environ)) + if chosen: + return chosen, "posix-locale" + + return "en", "default" + + +# -------------------------------------------------------------------------- +# profile loading and merging +# -------------------------------------------------------------------------- + + +def load_builtin_profile(name: str) -> Dict[str, Any]: + """Read ``carl_file_organizer/profiles/.json`` from the installed package.""" + + from importlib.resources import files # 3.9+ + + if not re.fullmatch(r"[a-z0-9_\-]+", name or ""): + raise ValueError("unknown profile: {0!r}".format(name)) + resource = files("carl_file_organizer.profiles").joinpath("{0}.json".format(name)) + try: + text = resource.read_text(encoding="utf-8") + except (FileNotFoundError, OSError) as error: + raise ValueError("unknown profile: {0}".format(name)) from error + profile = json.loads(text) + validate_profile(profile) + return profile + + +def load_dotfile(root: Path) -> Optional[Dict[str, Any]]: + """Read ``/.carl-file-organizer.json``; ``None`` when it does not exist.""" + + path = Path(root) / DOTFILE_NAME + if not path.is_file(): + return None + raw = path.read_text(encoding="utf-8") + try: + data = json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError( + "{0} is not valid JSON (line {1}, column {2}): {3}".format( + path, error.lineno, error.colno, error.msg + ) + ) from error + if not isinstance(data, dict): + raise ValueError("{0} must contain a JSON object".format(path)) + return data + + +def _union(base: Sequence[Any], extra: Sequence[Any]) -> List[Any]: + merged: List[Any] = [] + seen = set() + for item in list(base) + list(extra): + marker = json.dumps(item, sort_keys=True, ensure_ascii=False) if isinstance(item, (dict, list)) else item + if marker in seen: + continue + seen.add(marker) + merged.append(item) + return merged + + +def _union_name_patterns(base: Sequence[Any], extra: Sequence[Any]) -> List[Any]: + merged: List[Any] = [] + index: Dict[str, int] = {} + for item in list(base) + list(extra): + if isinstance(item, dict) and item.get("id"): + key = str(item["id"]) + if key in index: + merged[index[key]] = item + continue + index[key] = len(merged) + merged.append(item) + return merged + + +def _get_in(data: Mapping[str, Any], path: Sequence[str]) -> Any: + current: Any = data + for step in path: + if not isinstance(current, Mapping) or step not in current: + return None + current = current[step] + return current + + +def _set_in(data: Dict[str, Any], path: Sequence[str], value: Any) -> None: + current = data + for step in path[:-1]: + current = current.setdefault(step, {}) + current[path[-1]] = value + + +def validate_profile(profile: Mapping[str, Any]) -> None: + """Refuse footguns before anything reads the profile.""" + + sensitive = profile.get("sensitive") or {} + minimum = int(sensitive.get("marker_min_len", 5) or 5) + for marker in sensitive.get("markers", []) or []: + text = str(marker) + if all(ord(char) < 128 for char in text) and len(text) < minimum: + raise ValueError( + "sensitive.markers entry {0!r} is too short: ASCII markers are substring " + "matched and must be at least {1} characters. Put short words in " + "sensitive.exact_stems instead.".format(text, minimum) + ) + names = profile.get("names") or {} + for key in names: + if len(str(key).split(".")) > MAX_NAME_DEPTH: + raise ValueError( + "names key {0!r} is deeper than {1} levels".format(key, MAX_NAME_DEPTH) + ) + + +def merge_profile(base: Mapping[str, Any], override: Optional[Mapping[str, Any]]) -> Dict[str, Any]: + """Sparse overlay of a dotfile on top of a built-in profile. + + ``names`` and ``rules`` merge key by key; list-shaped settings are unioned; + ``lang`` / ``aging`` / ``large_bytes`` and friends are overwritten. + """ + + merged: Dict[str, Any] = json.loads(json.dumps(base, ensure_ascii=False)) + if not override: + validate_profile(merged) + return merged + + for section in ("names", "rules"): + incoming = override.get(section) + if not isinstance(incoming, Mapping): + continue + target = merged.setdefault(section, {}) + for key, value in incoming.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), Mapping): + combined = dict(target[key]) + combined.update(value) + target[key] = combined + else: + target[key] = value + + for section in SCALAR_SECTIONS: + incoming = override.get(section) + if not isinstance(incoming, Mapping): + continue + target = merged.setdefault(section, {}) + for key, value in incoming.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), Mapping): + combined = dict(target[key]) + combined.update(value) + target[key] = combined + else: + target[key] = value + + # Unions read from ``base`` on purpose: the scalar pass above may have + # replaced a list wholesale, and a union must never lose built-in entries. + for path in UNION_LIST_KEYS: + incoming = _get_in(override, path) + if not isinstance(incoming, list): + continue + current = _get_in(base, path) + _set_in(merged, path, _union(current if isinstance(current, list) else [], incoming)) + + if isinstance(override.get("name_patterns"), list): + current = base.get("name_patterns") + merged["name_patterns"] = _union_name_patterns( + current if isinstance(current, list) else [], override["name_patterns"] + ) + + for key in OVERRIDE_KEYS: + if key in override: + value = override[key] + if key in ("managed_dir", "transient_tag", "review_log") and isinstance(value, Mapping): + combined = dict(merged.get(key) or {}) + combined.update(value) + merged[key] = combined + else: + merged[key] = value + + validate_profile(merged) + return merged + + +def profile_hash(profile: Mapping[str, Any]) -> str: + """Short digest of the rules in a merged profile; stored in plan.json to catch drift. + + The chosen language is left out on purpose. The first ``plan`` in a folder + hashes the settings and then writes the settings file, which records the + language; hashing that back in would make every second run announce that + the settings had changed. Language decides what the folders are called, not + what goes into them, so it is not drift. An edited rule or a new sensitive + name still moves the digest. + """ + + material = { + key: value for key, value in profile.items() if key not in HASH_IGNORED_KEYS + } + encoded = json.dumps(material, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:12] + + +# -------------------------------------------------------------------------- +# resolution and persistence +# -------------------------------------------------------------------------- + + +def resolve_config( + root: Path, + *, + lang: Optional[str] = None, + profile: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, +) -> Config: + """Load the dotfile, pick the profile and the language, return a Config.""" + + root_path = Path(root).expanduser() + settings = load_dotfile(root_path) + profile_name = profile or (settings or {}).get("profile") or DEFAULT_PROFILE + base = load_builtin_profile(str(profile_name)) + merged = merge_profile(base, settings) + + warnings: List[str] = [] + lang_value, lang_source = detect_lang( + root_path, lang, env=env, dotfile=settings or {}, profile=merged + ) + if lang_source == "default": + warnings.append( + "no language signal found; falling back to en (pass --lang zh to change it)" + ) + + return Config( + root=root_path, + lang=lang_value, + lang_source=lang_source, + profile_name=str(profile_name), + profile=merged, + profile_source="builtin+dotfile" if settings else "builtin", + dotfile_path=(root_path / DOTFILE_NAME) if settings is not None else None, + profile_hash=profile_hash(merged), + warnings=warnings, + ) + + +def save_dotfile(config: Config, *, only_if_missing: bool = True) -> Optional[Path]: + """Write the smallest useful settings file; never write the merged profile back.""" + + path = Path(config.root) / DOTFILE_NAME + if only_if_missing and path.exists(): + return None + payload = {"profile": config.profile_name, "lang": config.lang} + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + config.dotfile_path = path + return path + + +# -------------------------------------------------------------------------- +# name resolution +# -------------------------------------------------------------------------- + + +def _pick_lang(entry: Any, lang: str) -> str: + if isinstance(entry, Mapping): + value = entry.get(lang) + if value: + return str(value) + value = entry.get("en") + if value: + return str(value) + for item in entry.values(): + if item: + return str(item) + return "" + return str(entry) + + +def dir_name(config: Config, key: str) -> str: + """``"library.docs.pdf"`` -> ``"20_知识库/文档资料/PDF"``. + + Parent keys are derived from the dotted prefix chain; three levels maximum. + """ + + parts = str(key).split(".") + if len(parts) > MAX_NAME_DEPTH: + raise ValueError( + "destination key {0!r} is deeper than {1} levels".format(key, MAX_NAME_DEPTH) + ) + names = config.profile.get("names") or {} + segments: List[str] = [] + for index in range(1, len(parts) + 1): + prefix = ".".join(parts[:index]) + if prefix not in names: + raise KeyError("profile {0} has no name for {1!r}".format(config.profile_name, prefix)) + segments.append(paths.safe_dir_name(_pick_lang(names[prefix], config.lang))) + return "/".join(segments) + + +def dest_dir(config: Config, key: str) -> Path: + return Path(config.root) / dir_name(config, key) + + +def managed_dir(config: Config) -> Path: + name = _pick_lang(config.profile.get("managed_dir"), config.lang) or "00_File_Organizer" + return Path(config.root) / paths.safe_dir_name(name) + + +def transient_tag(config: Config) -> str: + return _pick_lang(config.profile.get("transient_tag"), config.lang) or "Moved by File Organizer" + + +def transient_tags(config: Config) -> Tuple[str, ...]: + """Every language spelling of the temporary tag, for stripping user tags.""" + + entry = config.profile.get("transient_tag") or {} + if isinstance(entry, Mapping): + return tuple(str(value) for value in entry.values() if value) + return (str(entry),) + + +def review_log_name(config: Config) -> str: + return _pick_lang(config.profile.get("review_log"), config.lang) or "review-log.md" + + +def partition_dir_names(config: Config) -> Tuple[str, ...]: + """Top-level partition directory names in every language, plus the managed folder.""" + + names = config.profile.get("names") or {} + found: List[str] = [] + for key, entry in names.items(): + if "." in str(key): + continue + if isinstance(entry, Mapping): + found.extend(str(value) for value in entry.values() if value) + elif entry: + found.append(str(entry)) + managed = config.profile.get("managed_dir") or {} + if isinstance(managed, Mapping): + found.extend(str(value) for value in managed.values() if value) + return tuple(sorted(set(found))) + + +def partition_prefixes(config: Config) -> Tuple[str, ...]: + """Numeric partition prefixes such as ``("00_", "10_", "20_", "60_", "90_")``.""" + + prefixes = set() + for name in partition_dir_names(config): + match = re.match(r"^(\d+_)", name) + if match: + prefixes.add(match.group(1)) + return tuple(sorted(prefixes)) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/dispose.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/dispose.py new file mode 100644 index 0000000..fd94a1c --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/dispose.py @@ -0,0 +1,908 @@ +"""The gate in front of the whole-machine inventory's disposals. + +``storage_scan.py`` measures, the Agent explains in ``analysis.json``, the report +page collects clicks, and this module is the only thing that acts. It exists +because the page is untrusted input: whatever arrives from the browser, the only +paths this module will ever touch are the ones the Agent already whitelisted in +``analysis.json``, and only for the colour that allows the action asked for. + +Four doors, in this order: + +1. :func:`validate_decisions` matches every submitted id back to an item, refuses + anything red, refuses a permanent delete on anything but green with the flag, + and walks each ``trash_paths`` entry through the path rules: inside the home + directory, no symlink segment, not a no-go basename, and never the home + directory itself or one of its immediate children. A parent directory + standing in for its contents is exactly what that last rule forbids. +2. :func:`preflight` looks at the real filesystem: the path is still there, and + nobody has it open. ``lsof`` being unavailable is recorded as ``unknown`` + rather than treated as "nobody", because the honest answer is that we do not + know. +3. :func:`execute` writes the manifest before it touches anything, appends one + audit line per path the moment that path is done, and stops the batch on the + first failure. A failed trash never turns into a delete: the two branches + never meet. +4. :func:`apply_decisions` is the three of them in order, and the only thing the + CLI and the review server call. + +Nothing here is specific to macOS beyond the trash backend, which lives in +:mod:`carl_file_organizer.trash` and refuses rather than improvises. +""" + +from __future__ import annotations + +import json +import math +import os +import shutil +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence + +from . import manifest as manifest_module +from . import paths +from . import trash as trash_module + +#: Where the storage line keeps its own paper trail, relative to the home +#: directory. Deliberately not the tidy-up managed folder: a tidy-up ``undo`` +#: reads a folder full of moves, and a folder full of disposals has nothing for +#: it to put back. +MANAGED_SUBDIR = (".carl-file-organizer", "storage") + +SCHEMA = "carl-file-organizer/storage-decisions" + +ACTIONS = ("trash", "delete") +COLORS = ("green", "yellow", "red") + +#: One row per path, in the seven columns the storage line writes. +MANIFEST_COLUMNS = ( + "action_at", + "status", + "path", + "size_kib_before", + "reason", + "restore_method", + "error", +) + +MANIFEST_PREFIX = "storage-removals" + +#: result status -> manifest status. +STATUS_MANIFEST = { + "planned": "PLANNED", + "trashed": "TRASHED", + "deleted": "DELETED", + "dry-run": "DRY-RUN", + "skipped": "SKIPPED", + "refused": "REFUSED", + "failed": "FAILED", +} + +#: Statuses that mean the path is really gone from where it was. +DONE_STATUSES = ("trashed", "deleted") + +DEFAULT_MEASURE_BUDGET_S = 5.0 +DEFAULT_LSOF_TIMEOUT_S = 3.0 + +TEXT: Dict[str, Dict[str, str]] = { + "zh": { + "not_object": "决定清单必须是一个 JSON 对象", + "bad_schema": "决定清单的 schema 不是 {expected}，拒绝执行", + "no_items": "决定清单里一个条目都没有", + "bad_item_ids": "item_ids 必须是字符串数组", + "bad_actions": "actions 必须是 id 到 trash 或 delete 的映射", + "analysis_not_object": "analysis.json 必须是一个 JSON 对象", + "analysis_no_items": "analysis.json 里没有 items", + "unknown_item": "这个 id 不在 analysis.json 里，拒绝", + "bad_action": "不认识的动作 {action}，只接受 trash 或 delete", + "red": "红色条目不提供任何删除接口，拒绝", + "yellow_delete": "黄色条目只能进废纸篓，不能永久删除", + "delete_no_flag": "永久删除要带 --allow-permanent-delete，拒绝", + "not_scanned": "这是系统提示条目，没有实测体积，不参与处置", + "no_paths": "这个条目没有核实过的 trash_paths，没有可动的路径", + "outside_home": "路径不在家目录里，拒绝：{path}", + "too_shallow": "这是家目录本身或它的一级子目录，太粗了，要写到具体那一层：{path}", + "symlink": "路径里有软链，拒绝：{path}", + "forbidden": "路径命中禁刀区（{label}），拒绝：{path}", + "duplicate_path": "这条路径在这一批里已经出现过了，跳过：{path}", + "empty_path": "trash_paths 里有一条空路径，拒绝", + "missing": "这条路径已经不在了，跳过", + "in_use": "还有程序占着它（{who}），先退掉再来", + "open_unknown": "这台机器查不了谁在用它，按未知放行", + "dry_run": "只是试跑，什么都没动", + "batch_stopped": "前面一条失败了，这一批剩下的都不动", + "trashed": "已进废纸篓", + "deleted": "已永久删除", + "no_backend": "这台机器没有可用的废纸篓后端，拒绝处置", + "recheck_symlink": "动手前复查发现路径里有软链，拒绝", + "recheck_forbidden": "动手前复查发现路径进了禁刀区，拒绝", + "recheck_outside": "动手前复查发现路径不在家目录里，拒绝", + "error": "出错了：{error}", + "restore_trash": "在废纸篓里选放回原处", + "restore_none": "永久删除，没法还原", + "cli_done": "处置完成：{done} 条动了，{other} 条没动；manifest {manifest}", + "cli_dry": "试跑完成：{total} 条会走一遍，什么都没动", + "cli_line": "{status} {path} {detail}", + }, + "en": { + "not_object": "the decisions file must be a JSON object", + "bad_schema": "the decisions file is not schema {expected}; refusing", + "no_items": "the decisions file names no items", + "bad_item_ids": "item_ids must be a list of strings", + "bad_actions": "actions must map an id to trash or delete", + "analysis_not_object": "analysis.json must be a JSON object", + "analysis_no_items": "analysis.json has no items", + "unknown_item": "this id is not in analysis.json; refused", + "bad_action": "unknown action {action}; only trash or delete", + "red": "a red item has no delete interface at all; refused", + "yellow_delete": "a yellow item can only go to the trash, never a permanent delete", + "delete_no_flag": "a permanent delete needs --allow-permanent-delete; refused", + "not_scanned": "this is a system note with no measured size; it takes no action", + "no_paths": "this item has no verified trash_paths, so there is nothing to act on", + "outside_home": "that path is not inside the home directory; refused: {path}", + "too_shallow": "that is the home directory or one of its immediate children; name the exact folder: {path}", + "symlink": "a symlink sits in that path; refused: {path}", + "forbidden": "that path is in a no-go zone ({label}); refused: {path}", + "duplicate_path": "that path is already in this batch; skipped: {path}", + "empty_path": "trash_paths holds an empty path; refused", + "missing": "that path is not there any more; skipped", + "in_use": "something still has it open ({who}); quit it first", + "open_unknown": "this machine cannot tell who has it open; treated as unknown", + "dry_run": "dry run; nothing was touched", + "batch_stopped": "an earlier item failed; the rest of this batch was left alone", + "trashed": "moved to the trash", + "deleted": "permanently deleted", + "no_backend": "no trash backend on this machine; refusing to dispose of anything", + "recheck_symlink": "the recheck found a symlink in the path; refused", + "recheck_forbidden": "the recheck found the path in a no-go zone; refused", + "recheck_outside": "the recheck found the path outside the home directory; refused", + "error": "error: {error}", + "restore_trash": "put it back from the Trash", + "restore_none": "permanently deleted; nothing to put back", + "cli_done": "done: {done} acted on, {other} left alone; manifest {manifest}", + "cli_dry": "dry run finished: {total} paths would be walked, nothing was touched", + "cli_line": "{status} {path} {detail}", + }, +} + + +class DisposeError(ValueError): + """The decisions file, or the analysis behind it, cannot be acted on.""" + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + table = TEXT.get(lang) or TEXT["en"] + template = table.get(key) or TEXT["en"].get(key) or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +# -------------------------------------------------------------------------- +# shapes +# -------------------------------------------------------------------------- + + +@dataclass +class DisposeRow: + """One path, with the decision that put it here and how far it got.""" + + item_id: str + action: str + color: str + path: Optional[Path] + path_portable: str + status: str + detail: str = "" + size_bytes: int = 0 + open_check: str = "" + reason: str = "" + restore_method: str = "" + + @property + def planned(self) -> bool: + return self.status == "planned" + + +@dataclass +class ValidatedDecisions: + analysis: Dict[str, Any] + decisions: Dict[str, Any] + home: Path + lang: str + allow_permanent_delete: bool + trash_backend: str + rows: List[DisposeRow] = field(default_factory=list) + profile: Dict[str, Any] = field(default_factory=dict) + + @property + def targets(self) -> List[DisposeRow]: + return [row for row in self.rows if row.planned] + + def status_of(self, item_id: str) -> List[str]: + return [row.status for row in self.rows if row.item_id == item_id] + + +@dataclass +class DisposeReport: + run_id: str + dry_run: bool + results: List[Dict[str, Any]] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + manifest: Optional[Path] = None + audit: Optional[Path] = None + trash_backend: str = trash_module.NONE + stopped: bool = False + + @property + def executed_ids(self) -> List[str]: + seen: List[str] = [] + for record in self.results: + if record.get("status") in DONE_STATUSES: + item_id = str(record.get("item_id") or "") + if item_id and item_id not in seen: + seen.append(item_id) + return seen + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + + +def managed_dir(home: Optional[Path] = None) -> Path: + """``$HOME/.carl-file-organizer/storage``: scan, analysis, report, paper trail.""" + + base = Path(home) if home is not None else Path.home() + return base.joinpath(*MANAGED_SUBDIR) + + +def manifest_path(managed: Path, ts: datetime) -> Path: + return Path(managed) / "{0}-{1}.tsv".format( + MANIFEST_PREFIX, manifest_module.timestamp_text(ts) + ) + + +def _pick(value: Any, lang: str) -> str: + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or value.get("zh") or "") + return "" if value is None else str(value) + + +def measure_bytes(path: Path, *, budget_s: float = DEFAULT_MEASURE_BUDGET_S) -> int: + """Logical size of ``path`` before it goes away, with a time budget. + + The number is only ever written into the manifest, so a directory that runs + out of budget returns what it counted rather than holding up the disposal. + """ + + target = Path(path) + try: + if target.is_symlink() or not target.is_dir(): + return int(target.lstat().st_size) + except OSError: + return 0 + total = 0 + deadline = time.monotonic() + max(budget_s, 0.0) + for dirpath, _dirnames, filenames in os.walk(str(target), onerror=lambda _e: None): + for name in filenames: + try: + total += int(os.lstat(os.path.join(dirpath, name)).st_size) + except OSError: + continue + if time.monotonic() > deadline: + break + return total + + +def _kib(size_bytes: int) -> int: + return int(math.ceil(max(int(size_bytes), 0) / 1024.0)) + + +def _default_profile() -> Dict[str, Any]: + from . import config as config_module + + return config_module.load_builtin_profile("tiered") + + +def _who(handles: Sequence[Any]) -> str: + names = [] + for handle in handles: + if isinstance(handle, Mapping): + names.append("{0} (pid {1})".format(handle.get("command"), handle.get("pid"))) + else: + names.append(str(handle)) + return ", ".join(names[:4]) + + +# -------------------------------------------------------------------------- +# stage one: validate +# -------------------------------------------------------------------------- + + +def _refusal(item_id: str, action: str, color: str, portable: str, detail: str) -> DisposeRow: + return DisposeRow( + item_id=item_id, + action=action, + color=color, + path=None, + path_portable=portable, + status="refused", + detail=detail, + ) + + +def _check_path( + portable_text: str, + *, + home: Path, + profile: Mapping[str, Any], + lang: str, +) -> Optional[str]: + """``None`` when the path may be acted on, otherwise the refusal sentence.""" + + if not portable_text or not isinstance(portable_text, str): + return text("empty_path", lang) + expanded = paths.expand_portable(portable_text, home) + if not paths.inside_root(expanded, home): + return text("outside_home", lang, path=portable_text) + try: + relative = paths.realpath(expanded).relative_to(paths.realpath(home)) + except ValueError: + return text("outside_home", lang, path=portable_text) + if len(relative.parts) < 2: + return text("too_shallow", lang, path=portable_text) + if paths.has_symlink_component(expanded, home): + return text("symlink", lang, path=portable_text) + label = paths.is_forbidden(expanded, profile) + if label: + return text("forbidden", lang, label=label, path=portable_text) + return None + + +def validate_decisions( + analysis: Mapping[str, Any], + decisions: Mapping[str, Any], + *, + allow_permanent_delete: bool = False, + home: Optional[Path] = None, + profile: Optional[Mapping[str, Any]] = None, + lang: Optional[str] = None, + trash_backend: Optional[str] = None, +) -> ValidatedDecisions: + """Turn a page's clicks into the paths this tool is willing to touch. + + Every rejection becomes a ``refused`` row rather than an exception, so one + bad id does not cost the person the rest of their selection. Only the two + documents themselves being unusable raises :class:`DisposeError`. + """ + + if not isinstance(analysis, Mapping): + raise DisposeError(text("analysis_not_object", lang or "en")) + speech = lang or str(analysis.get("lang") or "") or "en" + if speech not in TEXT: + speech = "en" + if not isinstance(decisions, Mapping): + raise DisposeError(text("not_object", speech)) + + schema = str(decisions.get("schema") or SCHEMA) + if schema != SCHEMA: + raise DisposeError(text("bad_schema", speech, expected=SCHEMA)) + + raw_items = analysis.get("items") + if not isinstance(raw_items, list): + raise DisposeError(text("analysis_no_items", speech)) + index: Dict[str, Dict[str, Any]] = {} + for entry in raw_items: + if isinstance(entry, Mapping) and entry.get("id"): + index[str(entry["id"])] = dict(entry) + + item_ids = decisions.get("item_ids") + if not isinstance(item_ids, list) or not all(isinstance(i, str) for i in item_ids): + raise DisposeError(text("bad_item_ids", speech)) + if not item_ids: + raise DisposeError(text("no_items", speech)) + actions = decisions.get("actions") or {} + if not isinstance(actions, Mapping) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in actions.items() + ): + raise DisposeError(text("bad_actions", speech)) + + home_dir = Path(home) if home is not None else Path.home() + rules = dict(profile) if profile is not None else _default_profile() + backend = trash_backend if trash_backend is not None else trash_module.detect_backend() + + rows: List[DisposeRow] = [] + seen_paths: Dict[str, str] = {} + for item_id in item_ids: + action = str(actions.get(item_id) or "trash") + item = index.get(item_id) + if item is None: + rows.append(_refusal(item_id, action, "", "", text("unknown_item", speech))) + continue + color = str(item.get("color") or "") + portable = str(item.get("path_portable") or "") + if action not in ACTIONS: + rows.append(_refusal(item_id, action, color, portable, text("bad_action", speech, action=action))) + continue + if color not in COLORS or color == "red": + rows.append(_refusal(item_id, action, color or "red", portable, text("red", speech))) + continue + if item.get("scanned") is False: + rows.append(_refusal(item_id, action, color, portable, text("not_scanned", speech))) + continue + if action == "delete" and color != "green": + rows.append(_refusal(item_id, action, color, portable, text("yellow_delete", speech))) + continue + if action == "delete" and not allow_permanent_delete: + rows.append(_refusal(item_id, action, color, portable, text("delete_no_flag", speech))) + continue + + trash_paths = [p for p in (item.get("trash_paths") or []) if isinstance(p, str)] + if not trash_paths: + rows.append(_refusal(item_id, action, color, portable, text("no_paths", speech))) + continue + + refusal = None + for candidate in trash_paths: + refusal = _check_path(candidate, home=home_dir, profile=rules, lang=speech) + if refusal is not None: + break + if refusal is not None: + # One bad path condemns the whole item: a whitelist that is partly + # wrong is a whitelist nobody checked. + rows.append(_refusal(item_id, action, color, portable, refusal)) + continue + + name = _pick(item.get("name"), speech) or item_id + reason = "{0} · {1}".format(item_id, name) + restore = ( + text("restore_trash", speech) + if action == "trash" + else (_pick(item.get("restore_hint"), speech) or text("restore_none", speech)) + ) + for candidate in trash_paths: + expanded = paths.expand_portable(candidate, home_dir) + key = str(paths.realpath(expanded)) + if key in seen_paths: + rows.append( + DisposeRow( + item_id=item_id, + action=action, + color=color, + path=None, + path_portable=candidate, + status="skipped", + detail=text("duplicate_path", speech, path=candidate), + reason=reason, + restore_method=restore, + ) + ) + continue + seen_paths[key] = item_id + rows.append( + DisposeRow( + item_id=item_id, + action=action, + color=color, + path=expanded, + path_portable=candidate, + status="planned", + size_bytes=int(item.get("size_bytes") or 0) if len(trash_paths) == 1 else 0, + reason=reason, + restore_method=restore, + ) + ) + + return ValidatedDecisions( + analysis=dict(analysis), + decisions=dict(decisions), + home=home_dir, + lang=speech, + allow_permanent_delete=bool(allow_permanent_delete), + trash_backend=backend, + rows=rows, + profile=rules, + ) + + +# -------------------------------------------------------------------------- +# stage two: preflight +# -------------------------------------------------------------------------- + + +def preflight( + validated: ValidatedDecisions, + *, + open_handles_fn: Optional[Callable[[Path], Any]] = None, + measure: bool = True, +) -> ValidatedDecisions: + """Ask the filesystem, right now, whether each planned path can be touched. + + Mutates and returns ``validated``: a row that fails here stops being planned, + so :func:`execute` only ever walks rows the machine agreed with a moment ago. + """ + + from . import guard as guard_module + + lang = validated.lang + for row in validated.rows: + if not row.planned or row.path is None: + continue + target = row.path + if not os.path.lexists(str(target)): + row.status = "skipped" + row.detail = text("missing", lang) + continue + if measure: + measured = measure_bytes(target) + if measured or not row.size_bytes: + row.size_bytes = measured + if open_handles_fn is not None: + handles = open_handles_fn(target) + status = getattr(handles, "status", "ok") + found = list(getattr(handles, "handles", handles) or []) + else: + result = guard_module.open_handles(target, DEFAULT_LSOF_TIMEOUT_S) + status = result.status + found = list(result.handles) + row.open_check = status + if found: + row.status = "refused" + row.detail = text("in_use", lang, who=_who(found)) + continue + if status == "unknown": + # Not a reason to stop: it is a reason to write it down. + row.detail = text("open_unknown", lang) + return validated + + +# -------------------------------------------------------------------------- +# stage three: act +# -------------------------------------------------------------------------- + + +def _recheck(row: DisposeRow, validated: ValidatedDecisions) -> Optional[str]: + """The path rules once more, against the filesystem as it is this instant.""" + + target = row.path + if target is None: + return text("missing", validated.lang) + if not paths.inside_root(target, validated.home): + return text("recheck_outside", validated.lang) + if paths.has_symlink_component(target, validated.home): + return text("recheck_symlink", validated.lang) + if paths.is_forbidden(target, validated.profile): + return text("recheck_forbidden", validated.lang) + return None + + +def execute_row( + row: DisposeRow, + validated: ValidatedDecisions, + *, + dry_run: bool = False, + trash_fn: Optional[Callable[..., None]] = None, +) -> None: + """Act on one path. Sets ``row.status`` and ``row.detail``; never raises.""" + + lang = validated.lang + if dry_run: + row.status = "dry-run" + row.detail = text("dry_run", lang) + return + + refusal = _recheck(row, validated) + if refusal is not None: + row.status = "refused" + row.detail = refusal + return + + target = row.path + if target is None: # _recheck already refused this, belt and braces + row.status = "refused" + row.detail = text("missing", lang) + return + try: + if row.action == "trash": + sender = trash_fn if trash_fn is not None else trash_module.move_to_trash + sender(target, backend=validated.trash_backend, lang=lang) + row.status = "trashed" + row.detail = text("trashed", lang) + return + # A permanent delete only ever reaches here through the green door with + # the flag on; a failed trash above returns rather than falling through. + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(str(target)) + else: + os.unlink(str(target)) + row.status = "deleted" + row.detail = text("deleted", lang) + except trash_module.TrashError as error: + row.status = "failed" + row.detail = str(error) + except Exception as error: # noqa: BLE001 - one bad path must not kill the batch + row.status = "failed" + row.detail = text("error", lang, error=error) + + +def _manifest_row(row: DisposeRow, *, action_at: str) -> Dict[str, Any]: + error = row.detail if row.status in ("failed", "refused") else "" + return { + "action_at": action_at, + "status": STATUS_MANIFEST.get(row.status, row.status.upper()), + "path": row.path_portable, + "size_kib_before": _kib(row.size_bytes), + "reason": row.reason or row.item_id, + "restore_method": row.restore_method, + "error": error, + } + + +def write_manifest(path: Path, rows: Sequence[Mapping[str, Any]]) -> Path: + lines = ["\t".join(MANIFEST_COLUMNS)] + for row in rows: + lines.append( + "\t".join(manifest_module.safe_field(row.get(column, "")) for column in MANIFEST_COLUMNS) + ) + return manifest_module.atomic_write(Path(path), "\n".join(lines) + "\n") + + +def read_manifest(path: Path) -> List[Dict[str, str]]: + target = Path(path) + lines = target.read_text(encoding="utf-8").splitlines() + if not lines: + return [] + header = lines[0].split("\t") + out: List[Dict[str, str]] = [] + for line in lines[1:]: + if not line.strip(): + continue + cells = line.split("\t") + cells += [""] * (len(header) - len(cells)) + out.append(dict(zip(header, cells))) + return out + + +def _audit_record( + row: DisposeRow, + *, + run_id: str, + manifest: str, + trash_backend: str, + now: datetime, +) -> Dict[str, Any]: + return { + "timestamp": now.isoformat(timespec="seconds"), + "run_id": run_id, + "line": "storage", + "item_id": row.item_id, + # ``kind`` is spelled the way the tidy-up audit spells it, so ``undo`` + # reading this file by mistake says "put it back from the Trash" + # instead of tripping over a word it has never seen. + "kind": row.action, + "action": row.action, + "color": row.color, + "source": row.path_portable, + "destination": None, + "status": row.status, + "detail": row.detail, + "size_bytes": row.size_bytes, + "open_check": row.open_check, + "manifest": manifest, + "trash_backend": trash_backend, + } + + +def _result(row: DisposeRow) -> Dict[str, Any]: + return { + "item_id": row.item_id, + "path": row.path_portable, + "action": row.action, + "color": row.color, + "status": row.status, + "detail": row.detail, + "size_bytes": row.size_bytes, + "open_check": row.open_check, + } + + +def execute( + validated: ValidatedDecisions, + *, + dry_run: bool = False, + managed: Optional[Path] = None, + now: Optional[datetime] = None, + trash_fn: Optional[Callable[..., None]] = None, + audit: Optional[Path] = None, +) -> DisposeReport: + """Walk the validated rows once, writing the paper trail as it goes.""" + + lang = validated.lang + moment = now or datetime.now().astimezone() + run_id = manifest_module.timestamp_text(moment) + action_at = moment.isoformat(timespec="seconds") + managed_path = Path(managed) if managed is not None else managed_dir(validated.home) + + report = DisposeReport(run_id=run_id, dry_run=dry_run, trash_backend=validated.trash_backend) + + if not dry_run and validated.targets and validated.trash_backend == trash_module.NONE: + if any(row.action == "trash" for row in validated.targets): + for row in validated.targets: + if row.action == "trash": + row.status = "refused" + row.detail = text("no_backend", lang) + + manifest_file = manifest_path(managed_path, moment) + manifest_text = paths.portable(manifest_file, validated.home) + audit_file = Path(audit) if audit is not None else manifest_module.audit_path(managed_path) + + if not dry_run: + # The manifest lands before anything moves: an interrupted run still + # leaves a page naming every path it was about to touch. + managed_path.mkdir(parents=True, exist_ok=True) + write_manifest(manifest_file, [_manifest_row(r, action_at=action_at) for r in validated.rows]) + report.manifest = manifest_file + + stopped = False + for row in validated.rows: + if row.planned: + if stopped: + row.status = "skipped" + row.detail = text("batch_stopped", lang) + else: + execute_row(row, validated, dry_run=dry_run, trash_fn=trash_fn) + if row.status == "failed": + stopped = True + record = _audit_record( + row, + run_id=run_id, + manifest=manifest_text if not dry_run else "", + trash_backend=validated.trash_backend, + now=moment, + ) + report.results.append(_result(row)) + if not dry_run: + manifest_module.append_audit([record], audit_file) + report.audit = audit_file + + report.stopped = stopped + if not dry_run: + write_manifest(manifest_file, [_manifest_row(r, action_at=action_at) for r in validated.rows]) + + counts: Dict[str, int] = {"total": len(report.results)} + for record in report.results: + key = str(record.get("status")) + counts[key] = counts.get(key, 0) + 1 + report.counts = counts + return report + + +def apply_decisions( + analysis: Mapping[str, Any], + decisions: Mapping[str, Any], + *, + dry_run: bool = False, + allow_permanent_delete: bool = False, + home: Optional[Path] = None, + managed_dir: Optional[Path] = None, # noqa: A002 - the name callers already use + now: Optional[datetime] = None, + profile: Optional[Mapping[str, Any]] = None, + trash_backend: Optional[str] = None, + trash_fn: Optional[Callable[..., None]] = None, + open_handles_fn: Optional[Callable[[Path], Any]] = None, + audit: Optional[Path] = None, +) -> DisposeReport: + """Validate, preflight and execute: the only entry the CLI and server use.""" + + validated = validate_decisions( + analysis, + decisions, + allow_permanent_delete=allow_permanent_delete, + home=home, + profile=profile, + trash_backend=trash_backend, + ) + preflight(validated, open_handles_fn=open_handles_fn) + return execute( + validated, + dry_run=dry_run, + managed=managed_dir, + now=now, + trash_fn=trash_fn, + audit=audit, + ) + + +# -------------------------------------------------------------------------- +# dispose subcommand +# -------------------------------------------------------------------------- + + +def _read_json(path: Path, what: str) -> Dict[str, Any]: + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except OSError as error: + raise DisposeError("cannot read {0} {1}: {2}".format(what, path, error)) from error + except ValueError as error: + raise DisposeError("{0} {1} is not valid JSON: {2}".format(what, path, error)) from error + if not isinstance(data, dict): + raise DisposeError("{0} {1} must hold a JSON object".format(what, path)) + return data + + +def run(args: Any) -> int: + """``dispose [--dry-run] [--allow-permanent-delete]``.""" + + analysis = _read_json(Path(getattr(args, "analysis")), "analysis.json") + decisions = _read_json(Path(getattr(args, "decisions")), "decisions.json") + return run_document( + analysis, + decisions, + dry_run=bool(getattr(args, "dry_run", False)), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + lang=getattr(args, "lang", None), + ) + + +def run_document( + analysis: Mapping[str, Any], + decisions: Mapping[str, Any], + *, + dry_run: bool = False, + allow_permanent_delete: bool = False, + managed: Optional[Path] = None, + lang: Optional[str] = None, +) -> int: + """Act on one decisions document and print what the CLI has always printed. + + Split out of :func:`run` so the combined decisions file can hand over the + analysis it carries without writing it back to disk first, and so the paper + trail can land in the same managed folder as the tidy-up half. + """ + + speech = str(analysis.get("lang") or lang or "en") + if speech not in TEXT: + speech = "en" + lang = speech + + report = apply_decisions( + analysis, + decisions, + dry_run=dry_run, + allow_permanent_delete=allow_permanent_delete, + managed_dir=managed, + ) + for record in report.results: + print( + text( + "cli_line", + lang, + status=record["status"], + path=record["path"], + detail=record["detail"], + ) + ) + done = sum(report.counts.get(key, 0) for key in DONE_STATUSES) + if dry_run: + print(text("cli_dry", lang, total=report.counts.get("total", 0))) + else: + print( + text( + "cli_done", + lang, + done=done, + other=report.counts.get("total", 0) - done, + manifest=report.manifest, + ) + ) + failed = report.counts.get("failed", 0) + return 1 if failed else 0 + + +if __name__ == "__main__": # pragma: no cover - the CLI owns the entry point + sys.exit(2) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py new file mode 100644 index 0000000..60302b6 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/executor.py @@ -0,0 +1,1364 @@ +"""Execution in three stages: validate the plan, preflight each action, then act. + +Nothing here decides anything. The plan already said what could happen and a +person already said which of it should happen; this module's whole job is to +refuse everything else, and to leave a trail that lets the person put it all +back. + +The stages, in order: + + validate_plan the approval file as a whole: schema, root, the id list + preflight_action one item, against the filesystem as it is right now + execute_action the actual move, trash or delete, plus verification + apply_approved_plan wires the three together and writes the paper trail + +Two rules shape everything below. A permanent delete needs an explicit flag +and refuses the entire batch when it is missing, because a silent skip would +let someone believe their disk got emptier. And a move that does not verify is +put straight back where it came from, and the rest of the batch stops. +""" + +from __future__ import annotations + +import copy +import json +import os +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +from . import i18n, ids +from . import manifest as manifest_module +from . import paths +from . import tags as tags_module +from . import trash as trash_module +from .guard import is_observer + +#: Literal on purpose: the executor must not depend on the planner to know +#: which plan format it can run. +SCHEMA_VERSION = 2 + +SUPPORTED_KINDS = ("move", "trash", "delete") + +#: Only these rules produce an action a person may reroute from the page. +REROUTABLE_RULES = ("unknown-ext", "dir:pending", "dir:derivative") + +#: Statuses are lowercase in audit.jsonl and uppercase in the TSV manifests. +STATUS_MANIFEST = { + "planned": "PLANNED", + "moved": "MOVED", + "trashed": "TRASHED", + "deleted": "DELETED", + "dry-run": "DRY-RUN", + "skipped": "SKIPPED", + "conflict": "CONFLICT", + "refused": "REFUSED", + "failed": "FAILED", +} + +DEFAULT_LSOF_TIMEOUT_S = 3.0 +DIR_ENTRY_CAP = 20000 +DIR_TIME_BUDGET_S = 1.5 + +TEXT = { + "source_missing": { + "zh": "源已经不在原处，跳过", + "en": "The source is no longer there; skipped", + }, + "source_symlink": { + "zh": "路径里有软链，拒绝执行", + "en": "A symlink sits in this path; refused", + }, + "outside_root": { + "zh": "源或目标不在整理目录里，拒绝执行", + "en": "Source or destination falls outside the folder; refused", + }, + "forbidden": { + "zh": "禁刀区，拒绝执行（{label}）", + "en": "No-go zone; refused ({label})", + }, + "changed": { + "zh": "计划生成之后这一项被改动过，跳过；重新 plan 就好", + "en": "This item changed after the plan was made; skipped, just run plan again", + }, + "kind_changed": { + "zh": "文件和目录的身份变了，跳过", + "en": "It is no longer the same kind of item; skipped", + }, + "destination_exists": { + "zh": "目标已经有同名的东西，不覆盖，源没有动", + "en": "The destination already exists; nothing was overwritten and the source stayed put", + }, + "destination_missing": { + "zh": "这条动作没有目标路径", + "en": "This action has no destination", + }, + "in_use": { + "zh": "现在有进程开着它（{who}），跳过", + "en": "A process still has it open ({who}); skipped", + }, + "ready": {"zh": "可以执行", "en": "ready"}, + "delete_no_flag": { + "zh": "批准名单里有永久删除，但没有加 --allow-permanent-delete，整批拒绝执行", + "en": ( + "The approval list contains a permanent delete but --allow-permanent-delete " + "was not given; the whole batch is refused" + ), + }, + "delete_not_regenerable": { + "zh": "只允许永久删除可再生的构建产物，{name} 不在名单里", + "en": "Permanent delete is only for regenerable build output, and {name} is not on that list", + }, + "delete_no_survivor": { + "zh": "重复组里没有确定保留的副本，拒绝永久删除", + "en": "No surviving copy is confirmed in this duplicate group; refusing the permanent delete", + }, + "delete_bad_tier": { + "zh": "这一项不属于可永久删除的类别（{tier}）", + "en": "This item is not in a permanently deletable class ({tier})", + }, + "delete_cold_dir": { + "zh": "冷存候选只允许删文件，不允许删目录", + "en": "Cold candidates may only be deleted as files, never as folders", + }, + "moved": {"zh": "已按批准移动", "en": "moved as approved"}, + "trashed": {"zh": "已移入废纸篓", "en": "moved to the trash"}, + "deleted": {"zh": "已永久删除", "en": "permanently deleted"}, + "dry_run": {"zh": "预演通过，没有真的动", "en": "validated; nothing was touched"}, + "verify_failed": { + "zh": "移动之后校验不通过（{detail}），已原路移回", + "en": "The move did not verify ({detail}); it was put back", + }, + "verify_failed_stuck": { + "zh": "移动之后校验不通过（{detail}），而且移回也失败了：{error}", + "en": "The move did not verify ({detail}) and moving it back also failed: {error}", + }, + "batch_stopped": { + "zh": "本批在前一条失败之后停下了，这一条没有执行", + "en": "The batch stopped after an earlier failure; this one was not attempted", + }, + "error": {"zh": "执行出错：{error}", "en": "execution error: {error}"}, + "verify_dst_missing": {"zh": "目标不存在", "en": "destination missing"}, + "verify_src_left": {"zh": "源还在原处", "en": "source still there"}, + "verify_size": { + "zh": "体积对不上（{before} -> {after}）", + "en": "size mismatch ({before} -> {after})", + }, + "verify_entries": { + "zh": "条目数对不上（{before} -> {after}）", + "en": "entry count mismatch ({before} -> {after})", + }, +} + +REVIEW_TEXT = { + "log_moved": { + "zh": "本次整理共移动 {count} 项；执行清单：`{manifest}`。", + "en": "This run moved {count} item(s); manifest: `{manifest}`.", + }, + "log_tagged": { + "zh": "本次移动目标已打 macOS Finder 标签 `{tag}`；标签清单：`{list}`。", + "en": "The moved items carry the macOS Finder tag `{tag}`; tag list: `{list}`.", + }, + "log_sensitive": { + "zh": "已将 {count} 项疑似敏感命名的东西移入 `{dest}`，未读取内容：{names}。", + "en": ( + "{count} item(s) with credential-looking names went to `{dest}`; " + "their contents were never read: {names}." + ), + }, + "log_conflict": { + "zh": "{names} 的目标位置已经有同名的东西，未覆盖，也未删除任一项，本次跳过。", + "en": ( + "{names} already had something at the destination; nothing was overwritten, " + "neither copy was deleted, and it was skipped." + ), + }, + "log_trashed": { + "zh": "已将 {count} 项移入废纸篓（后端 {backend}），在废纸篓里可以放回原处。", + "en": "{count} item(s) went to the trash (backend {backend}) and can be put back from there.", + }, + "log_deleted": { + "zh": "已永久删除 {count} 项可再生的构建产物：{names}。", + "en": "Permanently deleted {count} regenerable build output(s): {names}.", + }, + "log_skipped": { + "zh": "{count} 项在计划生成之后被动过或已经不在，本次跳过。", + "en": "{count} item(s) changed or vanished after the plan was made and were skipped.", + }, + "log_failed": { + "zh": "{count} 项执行失败并已原路移回，本批余下动作已停止。", + "en": "{count} item(s) failed, were put back, and the rest of the batch stopped.", + }, + "log_recheck": { + "zh": "复查结果：dry-run actions={count}。", + "en": "Recheck: dry-run actions={count}.", + }, +} + +CLI_TEXT = { + "cli_audit": {"zh": "审计：{path}", "en": "Audit: {path}"}, + "cli_manifest": {"zh": "清单：{path}", "en": "Manifest: {path}"}, + "cli_tag_list": {"zh": "标签清单：{path}", "en": "Tag list: {path}"}, + "cli_review_log": {"zh": "人工日志：{path}", "en": "Log: {path}"}, + "cli_approved_copy": {"zh": "批准文件已留档：{path}", "en": "Approval kept at: {path}"}, + "cli_dry_run": { + "zh": "预演模式：一个字节都没有写，清单和日志也不生成。", + "en": "Dry run: nothing was written, and no manifest or log was produced.", + }, + "cli_summary": { + "zh": "共 {total} 条：移动 {moved}，废纸篓 {trashed}，永久删除 {deleted}，跳过 {skipped}，失败 {failed}", + "en": ( + "{total} action(s): moved {moved}, trashed {trashed}, deleted {deleted}, " + "skipped {skipped}, failed {failed}" + ), + }, + "cli_profile_drift": { + "zh": "提醒：计划生成之后设置改过了（profile 摘要对不上），只提醒不中断。", + "en": "warning: settings changed after the plan was made (profile hash differs); continuing anyway.", + }, +} + + +class PlanError(ValueError): + """The approval file violates the execution contract.""" + + +class ExecutionRefused(PlanError): + """The batch is refused as a whole, on purpose, before anything is touched.""" + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key) or REVIEW_TEXT.get(key) or CLI_TEXT.get(key) or {} + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class ValidatedPlan: + plan: Dict[str, Any] + root: Path + home: Path + lang: str + actions: List[Dict[str, Any]] + by_id: Dict[str, Dict[str, Any]] + overrides: Dict[str, str] + allow_permanent_delete: bool + trash_backend: str + + +@dataclass +class Preflight: + action: Dict[str, Any] + ok: bool + status: str + detail: str + + +@dataclass +class ApplyReport: + run_id: str + dry_run: bool + results: List[Dict[str, Any]] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + manifest: Optional[Path] = None + tag_list: Optional[Path] = None + audit: Optional[Path] = None + review_log: Optional[Path] = None + approved_copy: Optional[Path] = None + recheck_actions: Optional[int] = None + trash_backend: str = trash_module.NONE + stopped: bool = False + + def status_of(self, action_id: str) -> Optional[str]: + for record in self.results: + if record.get("action_id") == action_id: + return str(record.get("status")) + return None + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def plan_home(plan: Dict[str, Any], home: Optional[Path] = None) -> Path: + """Recover the home directory the plan was written against. + + The arithmetic lives in :func:`carl_file_organizer.paths.plan_home`, which the + review page needs too; this name stays because the executor's callers know it. + """ + + return paths.plan_home(plan, home) + + +def _localized(value: Any, lang: str) -> str: + if isinstance(value, dict): + return str(value.get(lang) or value.get("en") or next(iter(value.values()), "")) + return "" if value is None else str(value) + + +def open_handles( + path: Path, *, timeout_s: float = DEFAULT_LSOF_TIMEOUT_S +) -> List[Dict[str, str]]: + """Processes holding ``path`` open, via ``lsof``. No lsof means no answer. + + This is the single-path spelling used just before a file is touched; the + plan-time batch lives in :mod:`carl_file_organizer.guard`. Both skip the same + indexing and preview daemons, via ``guard.is_observer``. + """ + + executable = shutil.which("lsof") + if not executable: + return [] + try: + result = subprocess.run( + [executable, "-F", "pcn", "--", str(path)], + capture_output=True, + text=True, + timeout=timeout_s, + ) + except (OSError, subprocess.SubprocessError): + return [] + if result.returncode not in (0, 1): + return [] + handles: List[Dict[str, str]] = [] + pid = "" + command = "" + for line in result.stdout.splitlines(): + tag, value = line[:1], line[1:].strip() + if tag == "p": + pid = value + command = "" + elif tag == "c": + command = value + elif tag == "n" and pid and not is_observer(command): + handles.append({"pid": pid, "command": command, "name": value}) + return handles + + +def _count_entries( + path: Path, *, cap: int = DIR_ENTRY_CAP, budget_s: float = DIR_TIME_BUDGET_S +) -> Tuple[int, bool]: + """Bounded recursive entry count; the flag says the count hit a limit.""" + + deadline = time.monotonic() + budget_s + total = 0 + stack = [Path(path)] + while stack: + current = stack.pop() + try: + entries = list(os.scandir(str(current))) + except OSError: + continue + for entry in entries: + total += 1 + if total >= cap or time.monotonic() > deadline: + return total, True + try: + if entry.is_dir(follow_symlinks=False): + stack.append(Path(entry.path)) + except OSError: + continue + return total, False + + +def _shape(path: Path) -> Dict[str, Any]: + """The facts a move has to preserve, captured just before it happens.""" + + target = Path(path) + if target.is_dir() and not target.is_symlink(): + count, truncated = _count_entries(target) + return {"kind": "dir", "entries": count, "truncated": truncated} + try: + size = target.stat().st_size + except OSError: + size = -1 + return {"kind": "file", "size": size} + + +def verify_move(src: Path, dst: Path, before: Dict[str, Any]) -> Tuple[bool, str]: + """Did the move actually land? Returns ``(ok, detail)`` and never raises.""" + + source = Path(src) + destination = Path(dst) + if not os.path.lexists(str(destination)): + return False, text("verify_dst_missing", "en") + if os.path.lexists(str(source)): + return False, text("verify_src_left", "en") + + if before.get("kind") == "dir": + after, after_truncated = _count_entries(destination) + if before.get("truncated") and after_truncated: + return True, "" + if after != before.get("entries"): + return ( + False, + text("verify_entries", "en", before=before.get("entries"), after=after), + ) + return True, "" + + try: + after_size = destination.stat().st_size + except OSError as error: + return False, str(error) + if int(before.get("size", -1)) >= 0 and after_size != before["size"]: + return False, text("verify_size", "en", before=before["size"], after=after_size) + return True, "" + + +# -------------------------------------------------------------------------- +# stage one: the approval file as a whole +# -------------------------------------------------------------------------- + + +def validate_plan( + plan: Dict[str, Any], + *, + allow_permanent_delete: bool, + home: Optional[Path] = None, +) -> ValidatedPlan: + """Check the approval file before a single byte moves. + + The order is part of the contract: schema, root, unknown ids, one action per + subject, unapprovable ids, tampering, reroutes, and finally the permanent + delete gate, which refuses the whole batch instead of skipping quietly. + """ + + if not isinstance(plan, dict): + raise PlanError("the approval file must contain a JSON object") + if plan.get("schema_version") != SCHEMA_VERSION: + raise PlanError( + "unsupported schema_version {0!r}: this build only runs version {1}".format( + plan.get("schema_version"), SCHEMA_VERSION + ) + ) + + # An approval exported from the review page carries only ``$HOME`` forms, + # because the page is shareable and absolute paths name the user. Expanding + # them here means every check below, and the whole preflight, keeps working + # on real paths without knowing where the file came from. + home_dir = plan_home(plan, home) + plan = paths.hydrate_absolute(plan, home_dir) + + root_text = str(plan.get("source_root") or "") + if not root_text: + raise PlanError("the plan has no source_root") + root = Path(root_text).expanduser() + if root.is_symlink(): + raise PlanError("the planned folder is a symlink: {0}".format(root)) + if not root.is_dir(): + raise PlanError("the planned folder is unavailable: {0}".format(root)) + + actions = plan.get("actions") + if not isinstance(actions, list): + raise PlanError("actions must be a list") + approved = plan.get("approved_action_ids") + if not isinstance(approved, list): + raise PlanError("approved_action_ids must be a list") + + by_id: Dict[str, Dict[str, Any]] = {} + for action in actions: + if not isinstance(action, dict) or not action.get("id"): + raise PlanError("every action needs an id") + by_id[str(action["id"])] = action + + approved_ids: List[str] = [] + seen: Set[str] = set() + for entry in approved: + key = str(entry) + if key in seen: + continue + seen.add(key) + approved_ids.append(key) + + unknown = [key for key in approved_ids if key not in by_id] + if unknown: + raise PlanError("approval contains unknown action ids: {0}".format(sorted(unknown))) + + lang = str(plan.get("lang") or "en") + + chosen: List[Dict[str, Any]] = [] + subjects: Dict[str, str] = {} + for key in approved_ids: + action = by_id[key] + subject = str(action.get("subject_id") or "") + if subject and subject in subjects: + raise PlanError( + "two approved actions target the same item: {0} and {1}".format( + subjects[subject], key + ) + ) + subjects[subject] = key + chosen.append(action) + + for action in chosen: + if not action.get("approvable", False): + raise PlanError( + "action {0} is not approvable ({1})".format( + action.get("id"), action.get("rule") + ) + ) + kind = str(action.get("kind") or "") + if kind not in SUPPORTED_KINDS: + raise PlanError( + "unsupported action kind {0!r} for {1}".format(kind, action.get("id")) + ) + _check_not_tampered(action, root, home_dir) + + working = [copy.deepcopy(action) for action in chosen] + working_by_id = {str(action["id"]): action for action in working} + overrides = _apply_overrides(plan, by_id, working_by_id, root, home_dir) + + if not allow_permanent_delete: + for action in working: + if action.get("kind") == "delete": + raise ExecutionRefused(text("delete_no_flag", lang)) + + capabilities = plan.get("capabilities") or {} + backend = str(capabilities.get("trash_backend") or trash_module.NONE) + + return ValidatedPlan( + plan=plan, + root=paths.realpath(root), + home=home_dir, + lang=lang, + actions=working, + by_id=by_id, + overrides=overrides, + allow_permanent_delete=allow_permanent_delete, + trash_backend=backend, + ) + + +def _check_not_tampered(action: Dict[str, Any], root: Path, home: Path) -> None: + """Recompute the action id from the very fields it was derived from.""" + + destination = action.get("destination") + portable = paths.portable(Path(str(destination)), home) if destination else None + expected = ids.action_id( + str(action.get("subject_id") or ""), str(action.get("kind") or ""), portable + ) + if expected != str(action.get("id")): + raise PlanError( + "action {0} was altered after the plan was written".format(action.get("id")) + ) + if destination is not None and not paths.inside_root(Path(str(destination)).parent, root): + raise PlanError( + "action {0} points outside the folder being organized".format(action.get("id")) + ) + + +def _apply_overrides( + plan: Dict[str, Any], + by_id: Dict[str, Dict[str, Any]], + working_by_id: Dict[str, Dict[str, Any]], + root: Path, + home: Path, +) -> Dict[str, str]: + """Reroute pending items, and only through the plan's own names table.""" + + raw = plan.get("overrides") or [] + if not isinstance(raw, list): + raise PlanError("overrides must be a list") + names = plan.get("names") or {} + if not isinstance(names, dict): + raise PlanError("names must be an object") + + resolved: Dict[str, str] = {} + for entry in raw: + if not isinstance(entry, dict): + raise PlanError("every override must be an object") + action_key = str(entry.get("action_id") or "") + key = str(entry.get("destination_key") or "") + if action_key not in by_id: + raise PlanError("override names an unknown action: {0}".format(action_key)) + original = by_id[action_key] + if original.get("kind") != "move" or not original.get("reroutable", False): + raise PlanError("action {0} cannot be rerouted".format(action_key)) + if str(original.get("rule") or "") not in REROUTABLE_RULES: + raise PlanError( + "action {0} has rule {1!r}, which is not reroutable".format( + action_key, original.get("rule") + ) + ) + if key not in names: + raise PlanError( + "override destination_key {0!r} is not in the plan's names table".format(key) + ) + filename = str(original.get("filename") or Path(str(original.get("source"))).name) + destination = Path(root) / str(names[key]) / filename + if not paths.inside_root(destination.parent, root): + raise PlanError("override for {0} escapes the folder".format(action_key)) + resolved[action_key] = key + target = working_by_id.get(action_key) + if target is not None: + target["destination"] = str(destination) + target["destination_portable"] = paths.portable(destination, home) + target["destination_key"] = key + target["rerouted"] = True + return resolved + + +# -------------------------------------------------------------------------- +# stage two: one item, against the filesystem as it is right now +# -------------------------------------------------------------------------- + + +def preflight_action( + action: Dict[str, Any], + root: Path, + config: Any, + *, + lang: str = "en", + open_handles_fn: Optional[Callable[[Path], Sequence[Any]]] = None, + duplicate_survivors: Optional[Iterable[str]] = None, + allow_permanent_delete: bool = True, +) -> Preflight: + """Everything that must still be true a moment before acting.""" + + source = Path(str(action.get("source") or "")) + kind = str(action.get("kind") or "") + + if not os.path.lexists(str(source)): + return Preflight(action, False, "skipped", text("source_missing", lang)) + if source.is_symlink() or paths.has_symlink_component(source, Path(root)): + return Preflight(action, False, "refused", text("source_symlink", lang)) + if not paths.inside_root(source, Path(root)): + return Preflight(action, False, "refused", text("outside_root", lang)) + + label = paths.is_forbidden(source, config) + if label: + return Preflight(action, False, "refused", text("forbidden", lang, label=label)) + + changed = _subject_changed(action, source, lang) + if changed is not None: + return Preflight(action, False, "skipped", changed) + + if kind == "move": + destination_text = action.get("destination") + if not destination_text: + return Preflight(action, False, "refused", text("destination_missing", lang)) + destination = Path(str(destination_text)) + if not paths.inside_root(destination.parent, Path(root)): + return Preflight(action, False, "refused", text("outside_root", lang)) + if paths.has_symlink_component(destination.parent, Path(root)): + return Preflight(action, False, "refused", text("source_symlink", lang)) + if os.path.lexists(str(destination)): + return Preflight(action, False, "conflict", text("destination_exists", lang)) + + if kind == "delete": + refusal = _delete_gate( + action, + source, + config, + lang=lang, + duplicate_survivors=duplicate_survivors, + allow_permanent_delete=allow_permanent_delete, + ) + if refusal is not None: + return Preflight(action, False, "refused", refusal) + + handler = open_handles_fn if open_handles_fn is not None else open_handles + try: + handles = list(handler(source) or []) + except Exception: # noqa: BLE001 - a guard that crashes must not block the batch + handles = [] + if handles: + who = ", ".join( + "{0}({1})".format(item.get("command") or item.get("name", "?"), item.get("pid", "?")) + if isinstance(item, dict) + else str(item) + for item in handles[:3] + ) + return Preflight(action, False, "skipped", text("in_use", lang, who=who)) + + return Preflight(action, True, "ready", text("ready", lang)) + + +def _subject_changed(action: Dict[str, Any], source: Path, lang: str = "en") -> Optional[str]: + """``None`` when the item is still exactly what the plan looked at.""" + + try: + stat = source.stat() + except OSError: + return text("source_missing", lang) + + live_kind = "dir" if source.is_dir() else "file" + recorded_kind = str(action.get("subject_kind") or live_kind) + if live_kind != recorded_kind: + return text("kind_changed", lang) + + recorded = str(action.get("subject_id") or "") + if not recorded: + return None + + live = ids.subject_id(source, live_kind, stat.st_size, stat.st_mtime_ns) + if live == recorded: + return None + + # Directory sizes are computed with a cap, so a planner may well have + # recorded a walked size rather than the inode size. Modification time is + # the real guard for a directory, so that second spelling is allowed, and + # only that one. + if live_kind == "dir": + try: + recorded_size = int(action.get("size_bytes") or 0) + except (TypeError, ValueError): + recorded_size = 0 + alternate = ids.subject_id(source, live_kind, recorded_size, stat.st_mtime_ns) + if alternate == recorded: + return None + + return text("changed", lang) + + +def _delete_gate( + action: Dict[str, Any], + source: Path, + config: Any, + *, + lang: str, + duplicate_survivors: Optional[Iterable[str]], + allow_permanent_delete: bool, +) -> Optional[str]: + """The second door in front of a permanent delete. ``None`` means open.""" + + if not allow_permanent_delete: + return text("delete_no_flag", lang) + + tier = str(action.get("tier") or "") + profile = getattr(config, "profile", {}) or {} + regenerable = (profile.get("regenerable") or {}).get("basenames") or [] + + if tier == "regenerable": + if source.name not in regenerable: + return text("delete_not_regenerable", lang, name=source.name) + return None + + if tier == "duplicate": + survivors = set(str(item) for item in (duplicate_survivors or [])) + subject = str(action.get("subject_id") or "") + if not (survivors - {subject}): + return text("delete_no_survivor", lang) + return None + + if tier == "cold": + if source.is_dir(): + return text("delete_cold_dir", lang) + return None + + return text("delete_bad_tier", lang, tier=tier or "unknown") + + +# -------------------------------------------------------------------------- +# stage three: act +# -------------------------------------------------------------------------- + + +def execute_action( + action: Dict[str, Any], + *, + dry_run: bool = False, + trash_backend: str = trash_module.NONE, + home: Optional[Path] = None, + root: Optional[Path] = None, + config: Any = None, + lang: str = "en", + trash_fn: Optional[Callable[..., None]] = None, +) -> Tuple[str, str]: + """Perform one approved action. Returns ``(status, detail)`` and never raises.""" + + kind = str(action.get("kind") or "") + source = Path(str(action.get("source") or "")) + + if dry_run: + return "dry-run", text("dry_run", lang) + + try: + if kind == "move": + destination = Path(str(action.get("destination"))) + if os.path.lexists(str(destination)): + return "conflict", text("destination_exists", lang) + destination.parent.mkdir(parents=True, exist_ok=True) + before = _shape(source) + shutil.move(str(source), str(destination)) + ok, detail = verify_move(source, destination, before) + if ok: + return "moved", text("moved", lang) + try: + if os.path.lexists(str(destination)) and not os.path.lexists(str(source)): + shutil.move(str(destination), str(source)) + except (OSError, shutil.Error) as error: + return "failed", text( + "verify_failed_stuck", lang, detail=detail, error=error + ) + return "failed", text("verify_failed", lang, detail=detail) + + if kind == "trash": + sender = trash_fn if trash_fn is not None else trash_module.move_to_trash + sender(source, backend=trash_backend, lang=lang) + return "trashed", text("trashed", lang) + + if kind == "delete": + if source.is_dir() and not source.is_symlink(): + if root is not None and paths.has_symlink_component(source, Path(root)): + return "refused", text("source_symlink", lang) + if config is not None and paths.is_forbidden(source, config): + return "refused", text("forbidden", lang, label="recheck") + shutil.rmtree(str(source)) + else: + os.unlink(str(source)) + return "deleted", text("deleted", lang) + + except trash_module.TrashError as error: + return "failed", str(error) + except Exception as error: # noqa: BLE001 - one bad item must not kill the batch + return "failed", text("error", lang, error=error) + + return "refused", text("delete_bad_tier", lang, tier=kind) + + +# -------------------------------------------------------------------------- +# the whole run +# -------------------------------------------------------------------------- + + +def _audit_record( + action: Dict[str, Any], + *, + status: str, + detail: str, + run_id: str, + home: Path, + manifest: str, + trash_backend: str, + tagged: bool = False, + now: Optional[datetime] = None, +) -> Dict[str, Any]: + destination = action.get("destination") + return { + "timestamp": (now or datetime.now().astimezone()).isoformat(timespec="seconds"), + "run_id": run_id, + "action_id": action.get("id"), + "subject_id": action.get("subject_id"), + "kind": action.get("kind"), + "source": paths.portable(Path(str(action.get("source"))), home), + "destination": paths.portable(Path(str(destination)), home) if destination else None, + "status": status, + "detail": detail, + "size_bytes": action.get("size_bytes", 0), + "manifest": manifest, + "tagged": tagged, + "trash_backend": trash_backend, + } + + +def _restore_method(action: Dict[str, Any], lang: str, home: Path) -> str: + kind = str(action.get("kind") or "") + if kind == "move" and action.get("destination"): + destination = paths.portable(Path(str(action["destination"])), home) + parent = paths.portable(Path(str(action["source"])).parent, home) + return 'mv "{0}" "{1}/"'.format(destination, parent) + if kind == "trash": + return i18n.reason("restore_trash", lang) + if kind == "delete": + return i18n.reason("restore_delete", lang) + return "" + + +def _manifest_row( + action: Dict[str, Any], status: str, *, moved_at: str, home: Path, lang: str +) -> Dict[str, Any]: + destination = action.get("destination") + return { + "moved_at": moved_at, + "status": STATUS_MANIFEST.get(status, status.upper()), + "original_path": paths.portable(Path(str(action.get("source"))), home), + "target_path": paths.portable(Path(str(destination)), home) if destination else "", + "size_bytes": action.get("size_bytes", 0), + "reason": _localized(action.get("reason"), lang), + "restore_method": _restore_method(action, lang, home), + "kind": action.get("kind"), + "action_id": action.get("id"), + } + + +def _tag_moved(destination: Path, transient: str) -> bool: + """Put the temporary tag on a moved item; a failure is a warning, not a stop.""" + + try: + if tags_module.add_tag(destination, transient): + return True + # Already carrying the tag from an earlier run counts as tagged, so the + # path still reaches the tag list that clear-tags reads. + return transient in tags_module.user_tags(destination, ()) + except Exception as error: # noqa: BLE001 - tagging never stops a run + print("tag_warning: {0}\t{1}".format(destination, error), file=sys.stderr) + return False + + +def _series_for(actions: Sequence[Dict[str, Any]]) -> str: + kinds = {str(action.get("kind")) for action in actions} + if kinds and kinds <= {"trash", "delete"}: + return "removals" + if any(str(action.get("rule")) == "conflict" for action in actions): + return "conflict-moves" + return "moves" + + +def _survivors(validated: ValidatedPlan) -> Dict[str, Set[str]]: + """Per group, the subjects that are kept rather than trashed or deleted.""" + + members: Dict[str, Set[str]] = {} + disposed: Dict[str, Set[str]] = {} + for group in validated.plan.get("groups") or []: + if not isinstance(group, dict): + continue + group_id = str(group.get("group_id") or "") + members[group_id] = {str(item) for item in (group.get("members") or [])} + disposed.setdefault(group_id, set()) + for action in validated.actions: + group_id = str(action.get("group_id") or "") + if not group_id: + continue + if action.get("kind") in ("trash", "delete"): + disposed.setdefault(group_id, set()).add(str(action.get("subject_id"))) + return { + group_id: (members.get(group_id, set()) - gone) + for group_id, gone in disposed.items() + } + + +def _review_bullets( + report: ApplyReport, + validated: ValidatedPlan, + *, + lang: str, + transient: str, + home: Path, +) -> List[str]: + """The sentences a person reads a week later to remember what happened.""" + + def names(records: Sequence[Dict[str, Any]], limit: int = 6) -> str: + shown = [Path(str(item.get("source"))).name for item in records[:limit]] + joined = "、".join(shown) if lang == "zh" else ", ".join(shown) + if len(records) > len(shown): + joined += "…" if lang == "zh" else ", ..." + return joined + + by_status: Dict[str, List[Dict[str, Any]]] = {} + for record in report.results: + by_status.setdefault(str(record.get("status")), []).append(record) + + bullets: List[str] = [] + moved = by_status.get("moved", []) + if moved: + bullets.append( + text( + "log_moved", + lang, + count=len(moved), + manifest=paths.portable(report.manifest, home) if report.manifest else "", + ) + ) + if report.tag_list is not None: + bullets.append( + text( + "log_tagged", + lang, + tag=transient, + list=paths.portable(report.tag_list, home), + ) + ) + + sensitive = [ + record + for record in moved + if str(validated.by_id.get(str(record.get("action_id")), {}).get("tier")) == "sensitive" + ] + if sensitive: + destination = Path(str(sensitive[0].get("destination") or "")).parent + bullets.append( + text( + "log_sensitive", + lang, + count=len(sensitive), + dest=str(destination), + names=names(sensitive), + ) + ) + + conflicts = by_status.get("conflict", []) + if conflicts: + bullets.append(text("log_conflict", lang, names=names(conflicts))) + + trashed = by_status.get("trashed", []) + if trashed: + bullets.append( + text("log_trashed", lang, count=len(trashed), backend=report.trash_backend) + ) + + deleted = by_status.get("deleted", []) + if deleted: + bullets.append(text("log_deleted", lang, count=len(deleted), names=names(deleted))) + + skipped = by_status.get("skipped", []) + by_status.get("refused", []) + if skipped: + bullets.append(text("log_skipped", lang, count=len(skipped))) + + failed = by_status.get("failed", []) + if failed: + bullets.append(text("log_failed", lang, count=len(failed))) + + if report.recheck_actions is not None: + bullets.append(text("log_recheck", lang, count=report.recheck_actions)) + + return bullets + + +def apply_approved_plan( + plan: Dict[str, Any], + *, + dry_run: bool = False, + allow_permanent_delete: bool = False, + now: Optional[datetime] = None, + recheck_fn: Optional[Callable[[], int]] = None, + continue_on_error: bool = False, + home: Optional[Path] = None, + config: Any = None, + audit: Optional[Path] = None, + open_handles_fn: Optional[Callable[[Path], Sequence[Any]]] = None, + trash_fn: Optional[Callable[..., None]] = None, +) -> ApplyReport: + """Run every approved action, in order, and write the paper trail. + + The order is deliberate. The manifest lands before anything moves, so an + interrupted run still leaves a page saying what was about to happen and how + to put it back. Each action appends its own audit line the moment it + finishes, which is what makes the audit the authority when the manifest is + stale. Tagging, the recheck and the human log all wait until the last file + has settled. + """ + + from . import config as config_module + + validated = validate_plan( + plan, allow_permanent_delete=allow_permanent_delete, home=home + ) + lang = validated.lang + root = validated.root + home_dir = validated.home + + if config is None: + config = config_module.resolve_config(root, lang=lang) + + moment = now or datetime.now().astimezone() + run_id = manifest_module.timestamp_text(moment) + moved_at = moment.isoformat(timespec="seconds") + + managed = config_module.managed_dir(config) + audit_file = Path(audit) if audit is not None else manifest_module.audit_path(managed) + + report = ApplyReport( + run_id=run_id, dry_run=dry_run, trash_backend=validated.trash_backend + ) + if not validated.actions: + report.counts = {"total": 0} + return report + + manifest_file = manifest_module.manifest_path( + managed, root.name, _series_for(validated.actions), moment + ) + manifest_text = paths.portable(manifest_file, home_dir) + + survivors = _survivors(validated) + checks: Dict[str, Preflight] = {} + for action in validated.actions: + group_id = str(action.get("group_id") or "") + checks[str(action["id"])] = preflight_action( + action, + root, + config, + lang=lang, + open_handles_fn=open_handles_fn, + duplicate_survivors=survivors.get(group_id), + allow_permanent_delete=allow_permanent_delete, + ) + + statuses: Dict[str, str] = {} + for action in validated.actions: + check = checks[str(action["id"])] + statuses[str(action["id"])] = "planned" if check.ok else check.status + + def rows() -> List[Dict[str, Any]]: + return [ + _manifest_row( + action, + statuses[str(action["id"])], + moved_at=moved_at, + home=home_dir, + lang=lang, + ) + for action in validated.actions + ] + + if not dry_run: + managed.mkdir(parents=True, exist_ok=True) + manifest_module.write_manifest(manifest_file, rows()) + report.manifest = manifest_file + + transient = config_module.transient_tag(config) + can_tag = (not dry_run) and tags_module.supported() + + stopped = False + tagged_paths: List[str] = [] + for action in validated.actions: + key = str(action["id"]) + check = checks[key] + if not check.ok: + status, detail = check.status, check.detail + elif stopped: + status, detail = "skipped", text("batch_stopped", lang) + else: + status, detail = execute_action( + action, + dry_run=dry_run, + trash_backend=validated.trash_backend, + home=home_dir, + root=root, + config=config, + lang=lang, + trash_fn=trash_fn, + ) + if status == "failed" and not continue_on_error: + stopped = True + statuses[key] = status + + # Tag the moved item before its audit line is written, so the line can + # tell the truth about whether the tag landed. + tagged = False + if status == "moved" and can_tag and action.get("destination"): + destination = Path(str(action["destination"])) + tagged = _tag_moved(destination, transient) + if tagged: + tagged_paths.append(paths.portable(destination, home_dir)) + + record = _audit_record( + action, + status=status, + detail=detail, + run_id=run_id, + home=home_dir, + manifest=manifest_text if not dry_run else "", + trash_backend=validated.trash_backend, + tagged=tagged, + now=moment, + ) + report.results.append(record) + if not dry_run: + manifest_module.append_audit([record], audit_file) + report.audit = audit_file + + report.stopped = stopped + + if not dry_run: + manifest_module.write_manifest(manifest_file, rows()) + + if tagged_paths: + tag_file = manifest_module.tag_list_path(managed, root.name, moment) + manifest_module.write_tag_list(tag_file, tagged_paths) + report.tag_list = tag_file + + if recheck_fn is not None: + try: + report.recheck_actions = int(recheck_fn()) + except Exception as error: # noqa: BLE001 - a recheck must not undo a good run + print("recheck_warning: {0}".format(error), file=sys.stderr) + + counts: Dict[str, int] = {"total": len(report.results)} + for record in report.results: + status = str(record.get("status")) + counts[status] = counts.get(status, 0) + 1 + report.counts = counts + + if not dry_run: + bullets = _review_bullets( + report, validated, lang=lang, transient=transient, home=home_dir + ) + if bullets: + report.review_log = manifest_module.append_review_log( + managed, + lang, + moment.date(), + bullets, + name=config_module.review_log_name(config), + ) + + return report + + +# -------------------------------------------------------------------------- +# apply subcommand +# -------------------------------------------------------------------------- + + +def _recheck_with_planner( + root: Path, config: Any, moment: datetime +) -> Optional[Callable[[], int]]: + """A real recheck when the planner is available, and nothing when it is not.""" + + try: + from . import planner as planner_module + except ImportError: + return None + build = getattr(planner_module, "build_plan", None) + if build is None: + return None + + def recheck() -> int: + import inspect + + options: Dict[str, Any] = {} + try: + parameters = inspect.signature(build).parameters + except (TypeError, ValueError): + parameters = {} + if "config" in parameters: + options["config"] = config + if "now" in parameters: + options["now"] = moment + if "dir_sizes" in parameters: + options["dir_sizes"] = False + fresh = build(root, **options) + actions = fresh.get("actions") or [] + return len([item for item in actions if item.get("approvable", True)]) + + return recheck + + +def run(args: Any) -> int: + """``carl-file-organizer apply [--dry-run] [--allow-permanent-delete]``.""" + + approval_path = Path(args.approval).expanduser() + plan = json.loads(approval_path.read_text(encoding="utf-8")) + if not isinstance(plan, dict): + raise PlanError("the approval file must contain a JSON object") + return run_document( + plan, + dry_run=bool(args.dry_run), + allow_permanent_delete=bool(args.allow_permanent_delete), + audit=Path(args.audit).expanduser() if getattr(args, "audit", None) else None, + approval_path=approval_path, + ) + + +def run_document( + document: Dict[str, Any], + *, + dry_run: bool = False, + allow_permanent_delete: bool = False, + audit: Optional[Path] = None, + approval_path: Optional[Path] = None, +) -> int: + """Apply one approved plan and print the same report the CLI has always printed. + + Split out of :func:`run` so the combined decisions file can hand over the + plan it carries without first writing it back to disk. Everything the + person sees -- the per-action lines, the summary, the manifest and audit + paths -- is produced here, once. + """ + + from . import config as config_module + + plan = paths.hydrate_absolute(document) + if not isinstance(plan, dict): + raise PlanError("the approval file must contain a JSON object") + + root = Path(str(plan.get("source_root") or "")).expanduser() + profile_name = (plan.get("profile") or {}).get("name") + config = config_module.resolve_config(root, lang=plan.get("lang"), profile=profile_name) + lang = config.lang + + recorded_hash = (plan.get("profile") or {}).get("hash") + if recorded_hash and recorded_hash != getattr(config, "profile_hash", ""): + print(text("cli_profile_drift", lang), file=sys.stderr) + + moment = datetime.now().astimezone() + report = apply_approved_plan( + plan, + dry_run=dry_run, + allow_permanent_delete=allow_permanent_delete, + now=moment, + recheck_fn=None if dry_run else _recheck_with_planner(config.root, config, moment), + config=config, + audit=audit, + ) + + for record in report.results: + print( + "{0}\t{1}\t{2}".format( + record.get("status"), record.get("source"), record.get("detail") + ) + ) + + print( + text( + "cli_summary", + lang, + total=report.counts.get("total", 0), + moved=report.counts.get("moved", 0), + trashed=report.counts.get("trashed", 0), + deleted=report.counts.get("deleted", 0), + skipped=report.counts.get("skipped", 0) + report.counts.get("refused", 0), + failed=report.counts.get("failed", 0), + ) + ) + + if report.dry_run: + print(text("cli_dry_run", lang)) + else: + if report.manifest: + print(text("cli_manifest", lang, path=report.manifest)) + if report.tag_list: + print(text("cli_tag_list", lang, path=report.tag_list)) + if report.audit: + print(text("cli_audit", lang, path=report.audit)) + if report.review_log: + print(text("cli_review_log", lang, path=report.review_log)) + if approval_path is not None: + copy_target = _keep_approval(approval_path, config, report.run_id) + if copy_target is not None: + report.approved_copy = copy_target + print(text("cli_approved_copy", lang, path=copy_target)) + + return 2 if (report.counts.get("failed", 0) or report.counts.get("refused", 0)) else 0 + + +def _keep_approval(approval: Path, config: Any, run_id: str) -> Optional[Path]: + """Tuck the approval file into the managed folder so it stops being clutter.""" + + from . import config as config_module + + try: + managed = config_module.managed_dir(config) + managed.mkdir(parents=True, exist_ok=True) + target = managed / "approved-{0}.json".format(run_id) + if paths.realpath(approval) == paths.realpath(target): + return target + shutil.copy2(str(approval), str(target)) + return target + except OSError as error: + print("keep_approval_warning: {0}".format(error), file=sys.stderr) + return None + + +#: The audit writer lives in manifest.py now; this keeps the old import working. +append_audit = manifest_module.append_audit diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/flow.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/flow.py new file mode 100644 index 0000000..cd16eec --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/flow.py @@ -0,0 +1,433 @@ +"""scan, report, apply: the three steps a person actually runs. + +The engine grew two entrances, and for a while the person running it got two of +everything: two scans, two reports, two approval files. This module puts the +one road back: + + scan one read-only pass over the folder and, unless told otherwise, over + the machine; writes plan.json and storage-scan.json side by side in + the folder's own managed directory + report one page from whatever that directory holds -- the plan half, the + analysis half, or both -- with the after picture at the end + apply one decisions file, the moves first and the cleanup second, each + half going through the very same engine the old subcommands used + +Nothing here decides what is safe. ``executor.run_document`` and +``dispose.run_document`` own every check, every refusal and every paper trail; +this module only reads files, works out which half is which, and prints. + +The step-by-step subcommands (plan, build, review, dispose, storage-report, +undo, status, clear-tags) all still work and all still write the same files. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +from . import i18n + +#: The decisions file the combined page exports. +DECISIONS_SCHEMA = "carl-file-organizer/decisions" + +#: The older, storage-only decisions file. ``dispose`` still takes it. +STORAGE_DECISIONS_SCHEMA = "carl-file-organizer/storage-decisions" + +#: Everything ``scan`` writes and ``report`` looks for, all in one directory. +PLAN_NAME = "plan.json" +STORAGE_SCAN_NAME = "storage-scan.json" +ANALYSIS_NAME = "analysis.json" +NOTES_NAME = "notes.json" +REPORT_NAME = "report.html" + +#: ``carl_file_organizer/flow.py`` -> ``scripts/storage_scan.py`` +STORAGE_SCAN_PATH = Path(__file__).resolve().parent.parent / "storage_scan.py" + +_SCANNER: Optional[Any] = None + + +class FlowError(RuntimeError): + """One sentence about a file that is missing, unreadable or the wrong shape.""" + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + + +def read_json(path: Path, what: str) -> Dict[str, Any]: + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except OSError as error: + raise FlowError("cannot read {0} {1}: {2}".format(what, path, error)) from error + except ValueError as error: + raise FlowError("{0} {1} is not valid JSON: {2}".format(what, path, error)) from error + if not isinstance(data, dict): + raise FlowError("{0} {1} must hold a JSON object".format(what, path)) + return data + + +def _optional_json(path: Optional[Path], what: str) -> Dict[str, Any]: + if path is None or not Path(path).is_file(): + return {} + return read_json(Path(path), what) + + +def storage_scanner() -> Any: + """The ``storage_scan`` script, loaded once by path. + + It sits next to the package rather than inside it because a person runs it + by hand, so it is loaded the same way ``render`` loads ``build_report``. + """ + + global _SCANNER + if _SCANNER is not None: + return _SCANNER + if not STORAGE_SCAN_PATH.is_file(): + raise FlowError("the whole-machine scanner is missing: {0}".format(STORAGE_SCAN_PATH)) + spec = importlib.util.spec_from_file_location("carl_file_organizer._storage_scan", str(STORAGE_SCAN_PATH)) + if spec is None or spec.loader is None: + raise FlowError("cannot load the whole-machine scanner from {0}".format(STORAGE_SCAN_PATH)) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except Exception as error: # noqa: BLE001 - a broken scanner is one sentence + sys.modules.pop(spec.name, None) + raise FlowError("the whole-machine scanner failed to load: {0}".format(error)) from error + _SCANNER = module + return module + + +def _now() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +# -------------------------------------------------------------------------- +# scan +# -------------------------------------------------------------------------- + + +def run_scan(args: Any) -> int: + """``scan [--lang] [--no-storage] [--budget-seconds N]``: read only, twice over.""" + + from . import config as config_module + from . import planner + + cfg = getattr(args, "config", None) + if cfg is None: + cfg = config_module.resolve_config( + Path(args.source), lang=getattr(args, "lang", None), profile=getattr(args, "profile", None) + ) + args.config = cfg + lang = cfg.lang + + # The combined report is rendered a step later; a tidy-up-only page here + # would only be a second thing to open and a second thing to keep in step. + args.write_report = False + code = int(planner.run(args)) + if code: + return code + + managed = config_module.managed_dir(cfg) + managed.mkdir(parents=True, exist_ok=True) + + if not bool(getattr(args, "storage", True)): + print(i18n.t("cli_storage_skipped", lang)) + else: + try: + scanner = storage_scanner() + data = scanner.run_scan( + home=getattr(args, "home", None), + budget_seconds=float(getattr(args, "budget_seconds", None) or 60.0), + ) + except Exception as error: # noqa: BLE001 - the tidy-up half is still worth having + print(i18n.t("cli_storage_failed", lang, error=error), file=sys.stderr) + else: + target = managed / STORAGE_SCAN_NAME + target.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(i18n.t("cli_storage_written", lang, path=target)) + + print( + i18n.t( + "cli_next_steps", + lang, + notes=managed / NOTES_NAME, + analysis=managed / ANALYSIS_NAME, + managed=managed, + ) + ) + return 0 + + +# -------------------------------------------------------------------------- +# report +# -------------------------------------------------------------------------- + + +def collect(managed: Path, analysis_path: Optional[Path] = None) -> Dict[str, Any]: + """The combined document for one managed directory: the halves that exist. + + ``plan.json`` is the tidy-up half. The whole-machine half is the Agent's + ``analysis.json``, never the raw ``storage-scan.json``: the scan has no + colours and no plain-language lines, so putting it on a page would show + buttons nobody vouched for. + """ + + managed = Path(managed) + document: Dict[str, Any] = {} + plan_path = managed / PLAN_NAME + if plan_path.is_file(): + document["plan"] = read_json(plan_path, PLAN_NAME) + chosen = Path(analysis_path) if analysis_path else managed / ANALYSIS_NAME + if chosen.is_file(): + document["analysis"] = read_json(chosen, ANALYSIS_NAME) + elif analysis_path is not None: + raise FlowError("cannot read {0} {1}: no such file".format(ANALYSIS_NAME, chosen)) + return document + + +def run_report(args: Any) -> int: + """``report [--notes n.json] [--analysis a.json] [-o page.html] [--serve]``.""" + + managed = Path(getattr(args, "managed_dir")).expanduser() + analysis_arg = getattr(args, "analysis", None) + document = collect(managed, Path(analysis_arg).expanduser() if analysis_arg else None) + lang = getattr(args, "lang", None) or (document.get("plan") or {}).get("lang") or ( + document.get("analysis") or {} + ).get("lang") or "en" + if not document: + raise FlowError(i18n.t("cli_nothing_to_report", lang, managed=managed)) + + notes_arg = getattr(args, "notes", None) + notes_path = Path(notes_arg).expanduser() if notes_arg else managed / NOTES_NAME + notes = _optional_json(notes_path, NOTES_NAME) or None + + if bool(getattr(args, "serve", False)): + from . import server as server_module + + return server_module.serve_review( + document, + port=int(getattr(args, "port", 0) or 0), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + open_browser=bool(getattr(args, "open_browser", True)), + managed_dir=managed, + notes=notes, + lang=getattr(args, "lang", None), + ) + + from . import render + + html = render.render(document, mode="static", lang=getattr(args, "lang", None), notes=notes) + output = Path(getattr(args, "output", None) or (managed / REPORT_NAME)).expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(html, encoding="utf-8") + print(i18n.t("cli_report_combined", lang, path=output)) + return 0 + + +# -------------------------------------------------------------------------- +# apply +# -------------------------------------------------------------------------- + + +def _halves(decisions: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + plan = decisions.get("plan") + storage = decisions.get("storage") + return (plan if isinstance(plan, dict) else {}), (storage if isinstance(storage, dict) else {}) + + +def _managed_of(plan: Dict[str, Any]) -> Optional[Path]: + """The tidy-up half's managed directory, so the cleanup's paper trail joins it. + + One scan, one report, one approval file, one folder holding the record of + what happened. The page exports a plan with ``managed_dir`` stripped and + only its ``$HOME``-relative twin left, which is exactly what this expands. + """ + + from . import paths + + absolute = plan.get("managed_dir") + if isinstance(absolute, str) and absolute: + return Path(absolute).expanduser() + portable = plan.get("managed_dir_portable") + if not isinstance(portable, str) or not portable: + return None + try: + return Path(paths.expand_portable(portable, paths.plan_home(plan))) + except Exception: # noqa: BLE001 - a guess that fails just means the old location + return None + + +def _source_document( + section: Dict[str, Any], override: Any, managed: Optional[Path], name: str +) -> Dict[str, Any]: + """The plan or analysis a half was decided against, in order of trust. + + A file named on the command line wins, then the copy the page put inside + the decisions file, then the one sitting in the managed directory. + """ + + if override: + return read_json(Path(override).expanduser(), name) + document = section.get("document") + if isinstance(document, dict): + return document + if managed and (managed / name).is_file(): + return read_json(managed / name, name) + return {} + + +def _approved_plan(section: Dict[str, Any], document: Dict[str, Any], decisions: Dict[str, Any]) -> Dict[str, Any]: + """The plan the executor validates: the source document plus the ticked ids.""" + + plan = dict(document) + plan["approved_action_ids"] = [str(i) for i in (section.get("approved_action_ids") or [])] + plan["overrides"] = list(section.get("overrides") or []) + plan["approved_at"] = decisions.get("decided_at") or _now() + plan["approved_by"] = decisions.get("decided_by") or "decisions" + return plan + + +def run_apply(args: Any) -> int: + """``apply ``: the moves first, the cleanup second, one summary. + + An older ``approved.json`` and an older storage decisions file both still + work; they are recognised by what they carry and handed to the subcommand + that has always owned them. + """ + + from . import dispose as dispose_module + from . import executor as executor_module + + path = Path(args.approval).expanduser() + decisions = read_json(path, "decisions file") + schema = str(decisions.get("schema") or "") + + if schema != DECISIONS_SCHEMA: + if schema == STORAGE_DECISIONS_SCHEMA: + return _apply_storage_only(args, decisions) + # An approved.json is the plan itself; the executor has always read it. + return int(executor_module.run(args)) + + lang = getattr(args, "lang", None) or "en" + dry_run = bool(getattr(args, "dry_run", False)) + allow_permanent = bool(getattr(args, "allow_permanent_delete", False)) + plan_section, storage_section = _halves(decisions) + managed = Path(getattr(args, "managed_dir", None)).expanduser() if getattr(args, "managed_dir", None) else None + + plan_ids = [str(i) for i in (plan_section.get("approved_action_ids") or [])] + item_ids = [str(i) for i in (storage_section.get("item_ids") or [])] + if not plan_ids and not item_ids: + print(i18n.t("cli_decisions_empty", lang)) + return 0 + + # Both documents are found before either half runs, so a missing analysis + # is a refusal up front rather than a half-applied batch. + plan_document = ( + _source_document(plan_section, getattr(args, "plan", None), managed, PLAN_NAME) if plan_ids else {} + ) + if plan_ids and not plan_document: + raise FlowError(i18n.t("cli_decisions_no_plan", lang)) + analysis_document = ( + _source_document(storage_section, getattr(args, "analysis", None), managed, ANALYSIS_NAME) if item_ids else {} + ) + if item_ids and not analysis_document: + raise FlowError(i18n.t("cli_decisions_no_analysis", lang)) + lang = ( + getattr(args, "lang", None) + or plan_document.get("lang") + or analysis_document.get("lang") + or "en" + ) + if managed is None: + managed = _managed_of(plan_document) + + codes = {"plan": 0, "storage": 0} + + if plan_ids: + print("== " + i18n.t("cli_decisions_plan", lang)) + codes["plan"] = int( + executor_module.run_document( + _approved_plan(plan_section, plan_document, decisions), + dry_run=dry_run, + allow_permanent_delete=allow_permanent, + audit=Path(args.audit).expanduser() if getattr(args, "audit", None) else None, + approval_path=path, + ) + ) + + if item_ids: + print("== " + i18n.t("cli_decisions_storage", lang)) + codes["storage"] = int( + dispose_module.run_document( + analysis_document, + _storage_decisions(storage_section, decisions), + dry_run=dry_run, + allow_permanent_delete=allow_permanent, + managed=managed, + lang=getattr(args, "lang", None), + ) + ) + + print(i18n.t("cli_decisions_summary", lang, plan=codes["plan"], storage=codes["storage"])) + return codes["plan"] or codes["storage"] + + +def _storage_decisions(section: Dict[str, Any], decisions: Dict[str, Any]) -> Dict[str, Any]: + item_ids = [str(i) for i in (section.get("item_ids") or [])] + actions = section.get("actions") if isinstance(section.get("actions"), dict) else {} + return { + "schema": STORAGE_DECISIONS_SCHEMA, + "schema_version": 1, + "item_ids": item_ids, + "actions": {i: str(actions.get(i) or "trash") for i in item_ids}, + "decided_at": decisions.get("decided_at") or _now(), + "decided_by": decisions.get("decided_by") or "decisions", + } + + +def _apply_storage_only(args: Any, decisions: Dict[str, Any]) -> int: + """A storage-only decisions file still applies, as long as an analysis is at hand.""" + + from . import dispose as dispose_module + + lang = getattr(args, "lang", None) or "en" + override = getattr(args, "analysis", None) + managed = Path(getattr(args, "managed_dir", None)).expanduser() if getattr(args, "managed_dir", None) else None + if override: + document = read_json(Path(override).expanduser(), ANALYSIS_NAME) + elif managed and (managed / ANALYSIS_NAME).is_file(): + document = read_json(managed / ANALYSIS_NAME, ANALYSIS_NAME) + else: + raise FlowError(i18n.t("cli_decisions_no_analysis", lang)) + return int( + dispose_module.run_document( + document, + decisions, + dry_run=bool(getattr(args, "dry_run", False)), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + managed=managed, + lang=getattr(args, "lang", None), + ) + ) + + +# -------------------------------------------------------------------------- +# dispatch +# -------------------------------------------------------------------------- + + +def run(args: Any) -> int: + command = getattr(args, "command", "") + if command == "scan": + return run_scan(args) + if command == "report": + return run_report(args) + if command == "apply": + return run_apply(args) + raise FlowError("flow.run does not own the subcommand {0}".format(command)) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py new file mode 100644 index 0000000..483fb6b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/grouping.py @@ -0,0 +1,327 @@ +"""Pairing, duplicate detection and regenerable build output. + +Three questions get answered here, all of them read-only: + + pair_archives is this .zip the same thing as the folder next to it + find_duplicates are two files the same file wearing different names + find_regenerable is this subfolder something a build can recreate + +Nothing here decides anything. Each function returns raw groups, and the +planner turns them into actions and options. +""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import classifier +from .ids import subject_id as _subject_id +from .scanner import DirStats, Entry, dir_stats_bounded, split_suffix + +#: How a name gains a copy marker: "foo (1).pdf", "foo_1.pdf", "Archive 2". +COPY_PATTERNS = ( + re.compile(r"^(?P.+?) \((?P\d+)\)(?P\.[^.]+)?$"), + re.compile(r"^(?P.+?)_(?P\d+)(?P\.[^.]+)?$"), + re.compile(r"^(?P.+?) (?P\d+)$"), +) + +DEFAULT_HASH_MAX_BYTES = 2 * 1024 * 1024 * 1024 + + +@dataclass +class RawGroup: + kind: str # "pair" | "duplicate" + stem: str + evidence: str # "name" | "sha256" | "name-pattern" + members: List[Entry] + sha256: Optional[str] = None + display: str = "" + group_id: str = "" + + +@dataclass +class RegenerableHit: + path: Path + name: str + owner: Entry # the top level directory it was found inside + relative: str # "web-prototype/node_modules" + dir_stats: DirStats + modified_ns: int = 0 + subject_id: str = "" + age_hours: int = 0 + + +# --------------------------------------------------------------------------- +# name arithmetic +# --------------------------------------------------------------------------- + + +def copy_base(name: str) -> Tuple[str, bool]: + """``"foo (1).pdf"`` -> ``("foo.pdf", True)``; unchanged names return False.""" + + text = str(name) + for pattern in COPY_PATTERNS: + match = pattern.match(text) + if match: + base = match.group("base") + ext = match.groupdict().get("ext") or "" + return base + ext, True + return text, False + + +def _strip_copy_marker(name: str) -> str: + base, _found = copy_base(name) + return base + + +def archive_stem(name: str, config: Any = None) -> Optional[str]: + """Folded stem of an archive file name, or ``None`` when it is not an archive.""" + + text = str(name) + stem, suffix = split_suffix(text) + if not suffix: + return None + if config is not None: + if not classifier.is_archive_suffix(suffix, config): + return None + elif suffix not in {".zip", ".tar.gz", ".tgz", ".tar", ".7z", ".rar", ".gz", ".xz", ".bz2"}: + return None + return _strip_copy_marker(stem).casefold() + + +def dir_stem(name: str) -> str: + """``"Archive (1)"`` and ``"Archive 2"`` both fold down to ``"archive"``.""" + + return _strip_copy_marker(str(name)).casefold() + + +def file_sha256(path: Path, *, max_bytes: int = DEFAULT_HASH_MAX_BYTES) -> Optional[str]: + """Digest of a regular file; ``None`` when it is too big or unreadable.""" + + try: + size = os.path.getsize(str(path)) + except OSError: + return None + if size > max_bytes: + return None + digest = hashlib.sha256() + try: + with open(str(path), "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +# --------------------------------------------------------------------------- +# pairing +# --------------------------------------------------------------------------- + + +def pair_archives(entries: Sequence[Entry], config: Any = None) -> List[RawGroup]: + """An archive and an extracted folder that share a stem belong together.""" + + buckets: Dict[str, Dict[str, List[Entry]]] = {} + for entry in entries: + if entry.kind == "dir": + stem = dir_stem(entry.name) + side = "dirs" + else: + stem = archive_stem(entry.name, config) + side = "files" + if stem is None: + continue + if not stem: + continue + buckets.setdefault(stem, {"dirs": [], "files": []})[side].append(entry) + + groups: List[RawGroup] = [] + for stem in sorted(buckets): + bucket = buckets[stem] + if not bucket["dirs"] or not bucket["files"]: + continue + members = sorted(bucket["dirs"], key=lambda item: item.name.casefold()) + members += sorted(bucket["files"], key=lambda item: item.name.casefold()) + groups.append( + RawGroup( + kind="pair", + stem=stem, + evidence="name", + members=members, + display=members[0].name, + ) + ) + return groups + + +# --------------------------------------------------------------------------- +# duplicates +# --------------------------------------------------------------------------- + + +def find_duplicates( + entries: Sequence[Entry], + *, + hash_duplicates: bool = False, + hash_max_bytes: int = DEFAULT_HASH_MAX_BYTES, + exclude: Optional[Sequence[str]] = None, +) -> List[RawGroup]: + """Byte-identical files first, then names that merely look like copies.""" + + skip = set(exclude or ()) + files = [ + entry + for entry in entries + if entry.kind == "file" and entry.subject_id not in skip and entry.size_bytes > 0 + ] + + groups: List[RawGroup] = [] + hashed: Dict[str, List[Entry]] = {} + claimed: set = set() + + if hash_duplicates: + by_size: Dict[int, List[Entry]] = {} + for entry in files: + by_size.setdefault(entry.size_bytes, []).append(entry) + for size in sorted(by_size): + candidates = by_size[size] + if len(candidates) < 2: + continue + for entry in candidates: + digest = file_sha256(entry.path, max_bytes=hash_max_bytes) + if digest is None: + continue + hashed.setdefault(digest, []).append(entry) + for digest in sorted(hashed): + members = hashed[digest] + if len(members) < 2: + continue + ordered = sorted(members, key=lambda item: item.name.casefold()) + for entry in ordered: + claimed.add(entry.subject_id) + groups.append( + RawGroup( + kind="duplicate", + stem=ordered[0].name, + evidence="sha256", + members=ordered, + sha256=digest, + display=ordered[0].name, + ) + ) + + by_base: Dict[str, List[Entry]] = {} + for entry in files: + if entry.subject_id in claimed: + continue + base, _marked = copy_base(entry.name) + by_base.setdefault(base.casefold(), []).append(entry) + for base in sorted(by_base): + members = by_base[base] + if len(members) < 2: + continue + ordered = sorted(members, key=lambda item: item.name.casefold()) + groups.append( + RawGroup( + kind="duplicate", + stem=ordered[0].name, + evidence="name-pattern", + members=ordered, + display=ordered[0].name, + ) + ) + return groups + + +# --------------------------------------------------------------------------- +# regenerable build output +# --------------------------------------------------------------------------- + + +def find_regenerable( + entries: Sequence[Entry], + config: Any, + *, + max_depth: int = 3, + now: Any = None, + dir_limits: Optional[Dict[str, Any]] = None, +) -> List[RegenerableHit]: + """Walk top level folders looking for ``node_modules`` and friends. + + Depth is counted from the top level folder, so ``proj/node_modules`` is one + and ``proj/a/b/c/node_modules`` is four. ``.git`` and anything in a no-go + zone are never entered, and a match is never descended into. + """ + + from . import paths as paths_module + + limits = dir_limits or {} + hits: List[RegenerableHit] = [] + for entry in entries: + if entry.kind != "dir": + continue + if paths_module.is_forbidden(entry.path, config): + continue + stack: List[Tuple[Path, int]] = [(entry.path, 0)] + while stack: + current, depth = stack.pop() + if depth >= max_depth: + continue + try: + children = sorted(os.scandir(str(current)), key=lambda item: item.name) + except OSError: + continue + for child in children: + name = child.name + try: + if child.is_symlink() or not child.is_dir(follow_symlinks=False): + continue + except OSError: + continue + if name == ".git": + continue + child_path = Path(child.path) + if classifier.is_regenerable_name(name, config): + stats = dir_stats_bounded( + child_path, + int(limits.get("max_entries", 50000)), + float(limits.get("time_budget_s", 1.5)), + ) + try: + child_stat = child.stat(follow_symlinks=False) + modified_ns = child_stat.st_mtime_ns + stat_size = child_stat.st_size + except OSError: + modified_ns = entry.modified_ns + stat_size = 0 + relative = "{0}/{1}".format( + entry.name, child_path.relative_to(entry.path).as_posix() + ) + age_hours = entry.age_hours + if now is not None: + seconds = now.timestamp() - (modified_ns / 1_000_000_000) + age_hours = int(seconds // 3600) if seconds > 0 else 0 + hits.append( + RegenerableHit( + path=child_path, + name=name, + owner=entry, + relative=relative, + dir_stats=stats, + modified_ns=modified_ns, + subject_id=_subject_id(child_path, "dir", stat_size, modified_ns), + age_hours=age_hours, + ) + ) + continue + if paths_module.is_forbidden(child_path, config): + continue + stack.append((child_path, depth + 1)) + hits.sort(key=lambda hit: hit.relative) + return hits diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py new file mode 100644 index 0000000..5897b82 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/guard.py @@ -0,0 +1,773 @@ +"""The safety net: who else is still using this path. + +Moving a file is reversible, but a launch agent that hard-codes its path is not +going to notice. So before anything is proposed, every candidate is asked four +read-only questions, each with a timeout, each degrading to "unknown" rather +than raising: + + G1 open_handles is a process holding it open right now + G2 config_references does launchd, cron or a shell rc file name it + G3 incoming_symlinks does a symlink somewhere else point into it + G4 dir_shape is it a worktree, a repo, a virtualenv, a project + G5 recently_modified was it touched in the last ten minutes + +Every entry point takes an injection hook (``lsof_runner``, ``reference_files``, +``scan_roots``, ``git_status_runner``) so tests never touch the real machine. +""" + +from __future__ import annotations + +import os +import shutil +import stat as stat_module +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +from . import paths as paths_module + +#: Files that routinely hard-code a path, relative to the home directory. +SHELL_RC_FILES = ( + ".zshrc", + ".zprofile", + ".zshenv", + ".bashrc", + ".bash_profile", + ".profile", +) + +LAUNCH_DIRS_HOME = ("Library/LaunchAgents",) +LAUNCH_DIRS_ABSOLUTE = ("/Library/LaunchAgents", "/Library/LaunchDaemons") + +#: Daemons that merely look at a file. Spotlight opens everything that lands in +#: a download folder within seconds, so without this list a fresh download is +#: always "in use" and the plan proposes nothing at all. These processes read, +#: index or preview; none of them breaks when a path changes. +#: +#: Finder is on the list for the same reason: selecting a file in a Finder window +#: hands it to the QuickLook thumbnail agent, and a folder the user is looking at +#: while the plan is built would otherwise come back held open by the very act of +#: looking. Names are matched as prefixes because lsof truncates its COMMAND +#: column somewhere between nine and fifteen characters, so the thumbnail agent +#: arrives as ``com.apple.quicklook.ThumbnailsA`` more often than under its full +#: name. +OBSERVER_COMMANDS = ( + "mdworker", + "mds", + "mds_stores", + "mdimport", + "mdsync", + "mdbulkimport", + "quicklookd", + "qlmanage", + "QuickLookUIService", + "com.apple.quicklook.ThumbnailsAgent", + "com.apple.quicklook.ThumbnailsA", + "Finder", + "Spotlight", + "suggestd", + "photoanalysisd", + "cloudd", + "bird", + "fseventsd", + "backupd", + "XProtect", + "diskimages-helper", + "revisiond", + "distnoted", +) + +#: The same list, folded once, so the per-handle check stays a plain lookup. +_OBSERVER_PREFIXES = tuple(sorted({name.casefold() for name in OBSERVER_COMMANDS})) + + +def is_observer(command: str) -> bool: + """True for a daemon that only reads the file and never owns its path. + + The match is on the prefix: lsof gives its COMMAND column a fixed width and + cuts long names off, so a truncated ``com.apple.quicklook.ThumbnailsA`` has + to read the same as the whole name. + """ + + folded = str(command or "").casefold() + if not folded: + return False + return folded.startswith(_OBSERVER_PREFIXES) + + +MAX_REFERENCE_FILES = 400 +MAX_REFERENCE_BYTES = 2 * 1024 * 1024 +SYMLINK_SCAN_BUDGET_S = 2.0 +SYMLINK_SCAN_MAX_ENTRIES = 40000 + + +@dataclass +class OpenHandles: + handles: List[Dict[str, Any]] = field(default_factory=list) + status: str = "ok" # "ok" | "unknown" + + def __bool__(self) -> bool: + return bool(self.handles) + + +@dataclass +class DirShape: + labels: List[str] = field(default_factory=list) + uncommitted: Optional[int] = None + + def __contains__(self, item: object) -> bool: + return item in self.labels + + +# --------------------------------------------------------------------------- +# G1: open file handles +# --------------------------------------------------------------------------- + + +def _default_lsof_runner(argv: Sequence[str], timeout_s: float): + return subprocess.run(list(argv), capture_output=True, text=True, timeout=timeout_s) + + +def _parse_lsof(text: str) -> List[Dict[str, Any]]: + found: List[Dict[str, Any]] = [] + seen = set() + for _name, handle in _parse_lsof_pairs(text): + marker = (handle["command"], handle["pid"]) + if marker in seen: + continue + seen.add(marker) + found.append(handle) + return found + + +def _parse_lsof_pairs(text: str) -> List[Tuple[str, Dict[str, Any]]]: + """``[(reported path, {command, pid})]`` from ``lsof -F pcn`` output.""" + + pairs: List[Tuple[str, Dict[str, Any]]] = [] + pid: Optional[int] = None + command: Optional[str] = None + for line in text.splitlines(): + if not line: + continue + tag, value = line[0], line[1:] + if tag == "p": + try: + pid = int(value) + except ValueError: + pid = None + command = None + elif tag == "c": + command = value + elif tag == "n": + if pid is None or is_observer(command or ""): + continue + pairs.append((value, {"command": command or "", "pid": pid})) + return pairs + + +def lsof_possible() -> bool: + """Can this machine answer "who has that file open" at all? + + Windows never can: there is no lsof, and the closest native answers + (handle.exe, Restart Manager) need a download or an elevated process, so the + honest answer is "unknown" rather than a silent "nobody". That distinction + matters because "nobody has it open" is what lets an item be moved, and + guessing it would be the one lie this tool cannot afford. + """ + + if sys.platform == "win32": + return False + return shutil.which("lsof") is not None + + +def open_handles( + path: Path, + timeout_s: float = 5.0, + *, + lsof_runner: Optional[Callable[..., Any]] = None, + is_dir: Optional[bool] = None, +) -> OpenHandles: + """Ask lsof who has ``path`` open. No lsof, or a timeout, means unknown.""" + + runner = lsof_runner + if runner is None: + if not lsof_possible(): + return OpenHandles([], "unknown") + runner = _default_lsof_runner + + directory = os.path.isdir(str(path)) if is_dir is None else bool(is_dir) + if directory: + argv = ["lsof", "-F", "pcn", "+d", str(path)] + else: + argv = ["lsof", "-F", "pcn", "--", str(path)] + + try: + result = runner(argv, timeout_s) + except subprocess.TimeoutExpired: + return OpenHandles([], "unknown") + except (OSError, subprocess.SubprocessError): + return OpenHandles([], "unknown") + except Exception: # noqa: BLE001 - an injected runner must never break a plan + return OpenHandles([], "unknown") + + stdout = getattr(result, "stdout", "") or "" + returncode = getattr(result, "returncode", 0) + if returncode not in (0, 1): + return OpenHandles([], "unknown") + return OpenHandles(_parse_lsof(stdout), "ok") + + +def probe_open_handles( + file_paths: Sequence[Path], + dir_paths: Sequence[Path] = (), + timeout_s: float = 5.0, + *, + lsof_runner: Optional[Callable[..., Any]] = None, +) -> Dict[str, OpenHandles]: + """One lsof call for all the files, one per directory. + + lsof is slow enough that asking it once per entry turns a plan into a + coffee break, so the files go in a single batch and only directories, + which need ``+d``, are asked one at a time. + """ + + results: Dict[str, OpenHandles] = {} + runner = lsof_runner + if runner is None: + if not lsof_possible(): + for item in list(file_paths) + list(dir_paths): + results[str(item)] = OpenHandles([], "unknown") + return results + runner = _default_lsof_runner + + files = [Path(item) for item in file_paths] + for item in files: + results[str(item)] = OpenHandles([], "ok") + + by_real = {} + for item in files: + try: + by_real[os.path.realpath(str(item))] = str(item) + except OSError: + continue + + chunk = 150 + for start in range(0, len(files), chunk): + batch = files[start : start + chunk] + argv = ["lsof", "-F", "pcn", "--"] + [str(item) for item in batch] + try: + result = runner(argv, timeout_s) + except Exception: # noqa: BLE001 - unknown beats crashing + for item in batch: + results[str(item)] = OpenHandles([], "unknown") + continue + if getattr(result, "returncode", 0) not in (0, 1): + for item in batch: + results[str(item)] = OpenHandles([], "unknown") + continue + for reported, handle in _parse_lsof_pairs(getattr(result, "stdout", "") or ""): + key = by_real.get(reported) or by_real.get(os.path.realpath(reported)) + if key is None: + continue + existing = results.setdefault(key, OpenHandles([], "ok")) + marker = (handle["command"], handle["pid"]) + if marker not in {(h["command"], h["pid"]) for h in existing.handles}: + existing.handles.append(handle) + + for item in dir_paths: + results[str(item)] = open_handles( + Path(item), timeout_s, lsof_runner=runner, is_dir=True + ) + return results + + +# --------------------------------------------------------------------------- +# G2: configuration references +# --------------------------------------------------------------------------- + + +def path_forms(path: Path, home: Optional[Path] = None) -> List[str]: + """The spellings a config file might use for the same path. + + Three everywhere: the absolute path, ``~/...`` and ``$HOME/...``. On + Windows there are four, because the absolute path has two spellings there + and a config file carries whichever one its author typed: a ``.gitconfig`` + or a shell profile written under Git Bash or WSL says ``C:/Users/...``, + while a PowerShell profile says ``C:\\Users\\...``. Missing the other one + means missing the reference, and a missed reference is a moved file that + breaks something silently. + """ + + base = Path(home) if home is not None else Path.home() + absolute = str(path) + forms = [absolute] + slashed = Path(absolute).as_posix() + if slashed != absolute: + forms.append(slashed) + try: + relative = Path(absolute).relative_to(base) + except ValueError: + return forms + forms.append("~/" + relative.as_posix()) + forms.append("$HOME/" + relative.as_posix()) + return forms + + +def _read_lines(path: Path) -> Optional[List[str]]: + try: + if not path.is_file(): + return None + if path.stat().st_size > MAX_REFERENCE_BYTES: + return None + return path.read_text(encoding="utf-8", errors="replace").splitlines() + except (OSError, ValueError): + return None + + +def default_reference_files(config: Any, home: Optional[Path] = None) -> List[Path]: + """Launch agents, launch daemons, shell rc files, plus the user's own list.""" + + base = Path(home) if home is not None else Path.home() + found: List[Path] = [] + for relative in LAUNCH_DIRS_HOME: + directory = base / relative + try: + found.extend(sorted(directory.glob("*.plist"))) + except OSError: + continue + for absolute in LAUNCH_DIRS_ABSOLUTE: + try: + found.extend(sorted(Path(absolute).glob("*.plist"))) + except OSError: + continue + for name in SHELL_RC_FILES: + found.append(base / name) + + profile = getattr(config, "profile", config) or {} + guard = profile.get("guard") if hasattr(profile, "get") else None + extra = (guard or {}).get("reference_files") or () + for item in extra: + found.append(paths_module.expand_portable(str(item), base).expanduser()) + return found[:MAX_REFERENCE_FILES] + + +def _crontab_text(crontab_runner: Optional[Callable[[], Any]]) -> Optional[List[str]]: + runner = crontab_runner + if runner is None: + if shutil.which("crontab") is None: + return None + + def runner(): # type: ignore[misc] + return subprocess.run( + ["crontab", "-l"], capture_output=True, text=True, timeout=3 + ) + + try: + result = runner() + except Exception: # noqa: BLE001 - no crontab is a normal machine, not an error + return None + if getattr(result, "returncode", 1) != 0: + return None + return (getattr(result, "stdout", "") or "").splitlines() + + +def load_reference_texts( + config: Any, + home: Optional[Path] = None, + *, + reference_files: Optional[Sequence[Any]] = None, + crontab_runner: Optional[Callable[[], Any]] = None, + include_crontab: bool = True, +) -> List[Tuple[str, List[str]]]: + """Read every reference source once, so per-entry matching is pure string work.""" + + base = Path(home) if home is not None else Path.home() + if reference_files is None: + candidates = default_reference_files(config, base) + else: + candidates = [Path(str(item)).expanduser() for item in reference_files] + + texts: List[Tuple[str, List[str]]] = [] + for candidate in candidates: + lines = _read_lines(Path(candidate)) + if lines is None: + continue + texts.append((paths_module.portable(Path(candidate), base), lines)) + if include_crontab: + cron = _crontab_text(crontab_runner) + if cron: + texts.append(("crontab", cron)) + return texts + + +def config_references( + path: Path, + config: Any, + home: Optional[Path] = None, + *, + reference_files: Optional[Sequence[Any]] = None, + crontab_runner: Optional[Callable[[], Any]] = None, + texts: Optional[Sequence[Tuple[str, List[str]]]] = None, +) -> List[Dict[str, Any]]: + """``[{file, line, form}]`` for every config file that names ``path``.""" + + base = Path(home) if home is not None else Path.home() + sources = ( + list(texts) + if texts is not None + else load_reference_texts( + config, + base, + reference_files=reference_files, + crontab_runner=crontab_runner, + include_crontab=reference_files is None, + ) + ) + forms = path_forms(Path(path), base) + found: List[Dict[str, Any]] = [] + for display, lines in sources: + for number, line in enumerate(lines, start=1): + for form in forms: + if form in line: + found.append({"file": display, "line": number, "form": form}) + break + return found + + +# --------------------------------------------------------------------------- +# G3: symlinks pointing in +# --------------------------------------------------------------------------- + + +def default_scan_roots(config: Any, home: Optional[Path] = None) -> List[Path]: + base = Path(home) if home is not None else Path.home() + profile = getattr(config, "profile", config) or {} + guard = profile.get("guard") if hasattr(profile, "get") else None + roots = (guard or {}).get("reference_scan_roots") or () + found: List[Path] = [] + for item in roots: + found.append(paths_module.expand_portable(str(item), base).expanduser()) + return found + + +def symlink_index( + config: Any, + home: Optional[Path] = None, + *, + scan_roots: Optional[Sequence[Any]] = None, + max_depth: int = 2, +) -> List[Tuple[str, Path]]: + """Every symlink under the scan roots, with the real path it resolves to. + + Built once per plan; the per-entry check is then a containment test. + """ + + base = Path(home) if home is not None else Path.home() + if scan_roots is None: + roots = default_scan_roots(config, base) + else: + roots = [Path(str(item)).expanduser() for item in scan_roots] + + skip_names = {"Library", "Applications", ".Trash", "node_modules", ".git"} + started = time.monotonic() + seen_entries = 0 + index: List[Tuple[str, Path]] = [] + visited = set() + + for root in roots: + stack: List[Tuple[Path, int]] = [(Path(root), 0)] + while stack: + current, depth = stack.pop() + try: + key = os.path.realpath(str(current)) + except OSError: + continue + if key in visited: + continue + visited.add(key) + if (time.monotonic() - started) > SYMLINK_SCAN_BUDGET_S: + return index + try: + iterator = list(os.scandir(str(current))) + except OSError: + continue + for item in iterator: + seen_entries += 1 + if seen_entries > SYMLINK_SCAN_MAX_ENTRIES: + return index + try: + if item.is_symlink(): + target = Path(os.path.realpath(item.path)) + index.append((paths_module.portable(Path(item.path), base), target)) + continue + if depth + 1 < max_depth and item.is_dir(follow_symlinks=False): + if item.name in skip_names or item.name.startswith("."): + continue + stack.append((Path(item.path), depth + 1)) + except OSError: + continue + return index + + +def incoming_symlinks( + path: Path, + config: Any, + home: Optional[Path] = None, + *, + scan_roots: Optional[Sequence[Any]] = None, + index: Optional[Sequence[Tuple[str, Path]]] = None, + max_depth: int = 2, +) -> List[str]: + """Portable paths of symlinks whose target lands inside ``path``.""" + + base = Path(home) if home is not None else Path.home() + entries = ( + list(index) + if index is not None + else symlink_index(config, base, scan_roots=scan_roots, max_depth=max_depth) + ) + target = Path(os.path.realpath(str(path))) + found: List[str] = [] + for display, resolved in entries: + if resolved == target: + found.append(display) + continue + try: + resolved.relative_to(target) + except ValueError: + continue + found.append(display) + return sorted(set(found)) + + +# --------------------------------------------------------------------------- +# G4: shape +# --------------------------------------------------------------------------- + + +def _default_git_status_runner(path: Path, timeout_s: float): + return subprocess.run( + ["git", "-C", str(path), "status", "--porcelain"], + capture_output=True, + text=True, + timeout=timeout_s, + ) + + +def dir_shape( + path: Path, + *, + git_status_runner: Optional[Callable[..., Any]] = None, + timeout_s: float = 2.0, + count_uncommitted: bool = True, +) -> DirShape: + """What kind of folder this is: worktree, repo, virtualenv, project.""" + + target = Path(path) + labels: List[str] = [] + uncommitted: Optional[int] = None + try: + if not target.is_dir(): + return DirShape([], None) + except OSError: + return DirShape([], None) + + git_path = target / ".git" + try: + if git_path.is_file(): + labels.append("git-worktree") + elif git_path.is_dir(): + labels.append("git-dir") + except OSError: + pass + + for name in (".venv", "venv", ".tox"): + try: + if (target / name).is_dir(): + labels.append("venv") + break + except OSError: + continue + + try: + if (target / ".env").exists(): + labels.append("dotenv") + except OSError: + pass + + try: + if (target / "package.json").is_file(): + labels.append("node-project") + except OSError: + pass + + if "git-dir" in labels and count_uncommitted: + runner = git_status_runner + if runner is None and shutil.which("git") is not None: + runner = _default_git_status_runner + if runner is not None: + try: + result = runner(target, timeout_s) + if getattr(result, "returncode", 1) == 0: + text = getattr(result, "stdout", "") or "" + uncommitted = len([line for line in text.splitlines() if line.strip()]) + except Exception: # noqa: BLE001 - a slow repo is not a planning failure + uncommitted = None + return DirShape(labels, uncommitted) + + +def file_shape(path: Path) -> List[str]: + """``executable`` and ``shebang`` markers for a plain file.""" + + labels: List[str] = [] + target = Path(path) + try: + mode = target.stat().st_mode + if mode & (stat_module.S_IXUSR | stat_module.S_IXGRP | stat_module.S_IXOTH): + labels.append("executable") + except OSError: + return labels + try: + with open(str(target), "rb") as handle: + if handle.read(2) == b"#!": + labels.append("shebang") + except OSError: + pass + return labels + + +# --------------------------------------------------------------------------- +# G5: freshly touched +# --------------------------------------------------------------------------- + + +def recently_modified(entry: Any, now: datetime, minutes: int = 10) -> bool: + """True when the entry was written inside the last ``minutes``.""" + + if minutes <= 0: + return False + modified_ns = getattr(entry, "modified_ns", None) + if modified_ns is None and isinstance(entry, Mapping): + modified_ns = entry.get("modified_ns") + if modified_ns is None: + return False + delta = now.timestamp() - (float(modified_ns) / 1_000_000_000) + return 0 <= delta < minutes * 60 + + +# --------------------------------------------------------------------------- +# one context per plan +# --------------------------------------------------------------------------- + + +@dataclass +class GuardContext: + home: Path + enabled: bool = True + reference_texts: List[Tuple[str, List[str]]] = field(default_factory=list) + symlinks: List[Tuple[str, Path]] = field(default_factory=list) + lsof_runner: Optional[Callable[..., Any]] = None + lsof_enabled: bool = True + lsof_timeout_s: float = 5.0 + git_status_runner: Optional[Callable[..., Any]] = None + recent_minutes: int = 10 + open_by: Dict[str, OpenHandles] = field(default_factory=dict) + + def prime(self, file_paths: Sequence[Path], dir_paths: Sequence[Path] = ()) -> None: + """Batch the lsof work for a whole scan before any entry is decided.""" + + if not self.enabled or not self.lsof_enabled: + return + self.open_by = probe_open_handles( + file_paths, dir_paths, self.lsof_timeout_s, lsof_runner=self.lsof_runner + ) + + +def build_context( + config: Any, + *, + home: Optional[Path] = None, + enabled: bool = True, + reference_files: Optional[Sequence[Any]] = None, + scan_roots: Optional[Sequence[Any]] = None, + lsof_runner: Optional[Callable[..., Any]] = None, + lsof_enabled: bool = True, + crontab_runner: Optional[Callable[[], Any]] = None, + git_status_runner: Optional[Callable[..., Any]] = None, +) -> GuardContext: + """Collect everything the per-entry checks need, once.""" + + base = Path(home) if home is not None else Path.home() + profile = getattr(config, "profile", config) or {} + guard = (profile.get("guard") if hasattr(profile, "get") else None) or {} + context = GuardContext( + home=base, + enabled=enabled, + lsof_runner=lsof_runner, + lsof_enabled=lsof_enabled, + lsof_timeout_s=float(guard.get("lsof_timeout_s", 5) or 5), + git_status_runner=git_status_runner, + recent_minutes=int(guard.get("recent_minutes", 10) or 0), + ) + if not enabled: + return context + context.reference_texts = load_reference_texts( + config, + base, + reference_files=reference_files, + crontab_runner=crontab_runner, + include_crontab=reference_files is None, + ) + context.symlinks = symlink_index(config, base, scan_roots=scan_roots) + return context + + +def inspect( + path: Path, + config: Any, + context: GuardContext, + *, + is_dir: bool = False, +) -> Dict[str, Any]: + """Run every check for one entry and return the plan.json ``guard`` block.""" + + empty = {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []} + if not context.enabled: + return empty + + handles = OpenHandles([], "unknown") + if context.lsof_enabled: + cached = context.open_by.get(str(path)) + if cached is not None: + handles = cached + else: + handles = open_handles( + path, + context.lsof_timeout_s, + lsof_runner=context.lsof_runner, + is_dir=is_dir, + ) + references = config_references(path, config, context.home, texts=context.reference_texts) + links = incoming_symlinks(path, config, context.home, index=context.symlinks) + if is_dir: + shape = dir_shape(path, git_status_runner=context.git_status_runner) + labels = list(shape.labels) + uncommitted = shape.uncommitted + else: + labels = file_shape(path) + uncommitted = None + block = { + "open_by": handles.handles, + "referenced_in": references, + "incoming_symlinks": links, + "shape": labels, + } + if handles.status == "unknown": + block["open_by_status"] = "unknown" + if uncommitted is not None: + block["uncommitted"] = uncommitted + return block diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py new file mode 100644 index 0000000..03c15ba --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/i18n.py @@ -0,0 +1,397 @@ +"""Bilingual strings: UI copy and plan reason text. + +Two entry points are frozen for every work package: + + t(key, lang, **kw) UI copy (buttons, headings, CLI lines) + reason(key, lang, **kw) one line of plan reasoning + +``both(key, **kw)`` is a convenience that returns ``{"zh": ..., "en": ...}``, +which is exactly the shape plan.json stores in ``action.reason`` / +``action.detail`` / ``group.reason``. + +Rules for editors: keys are only ever added, never renamed or removed; the zh +column avoids straight double quotes and reads like a person talking. +""" + +from __future__ import annotations + +from typing import Any, Dict, Mapping + +LANGS = ("zh", "en") +DEFAULT_LANG = "en" + +#: Reason text attached to actions and groups. +REASONS: Dict[str, Dict[str, str]] = { + # --- extension rules ------------------------------------------------- + "ext_doc": { + "zh": "文档，按类型放进 {dest}", + "en": "A document, so it goes by type into {dest}", + }, + "ext_image": { + "zh": "图片，按类型放进 {dest}", + "en": "An image, so it goes by type into {dest}", + }, + "ext_video": { + "zh": "视频，按类型放进 {dest}", + "en": "A video, so it goes by type into {dest}", + }, + "ext_audio": { + "zh": "音频，按类型放进 {dest}", + "en": "Audio, so it goes by type into {dest}", + }, + "ext_web": { + "zh": "网页文件，按类型放进 {dest}", + "en": "A saved web page, so it goes by type into {dest}", + }, + "ext_script": { + "zh": "脚本，放进工具与导出", + "en": "A script, so it goes under tools and exports", + }, + "ext_data": { + "zh": "数据文件，放进数据库与查询", + "en": "A data file, so it goes under database and queries", + }, + "ext_archive": { + "zh": "压缩包，放满 {hours} 小时没人动过，可以归档了", + "en": "An archive that has sat untouched for {hours}h, ready to be filed away", + }, + "ext_installer": { + "zh": "安装包，放满 {hours} 小时没人动过，可以归档了", + "en": "An installer that has sat untouched for {hours}h, ready to be filed away", + }, + "ext_other": { + "zh": "扩展名规则命中，放进 {dest}", + "en": "The extension matched a rule, so it goes into {dest}", + }, + # --- sensitive ------------------------------------------------------- + "sensitive_isolated": { + "zh": "名字看着像凭证，先单独隔离起来，未读取内容", + "en": "The name looks like a credential, so it is set aside on its own; the contents were never read", + }, + "sensitive_manual": { + "zh": "你自己标成了敏感，移到敏感信息区，未读取内容", + "en": "You marked it sensitive, so it goes to the sensitive area; the contents were never read", + }, + # --- directories ----------------------------------------------------- + "dir_pending": { + "zh": "整个目录原样搬进待判断，名字不动，里面不拆", + "en": "The whole folder goes to Pending as it is, name and contents untouched", + }, + "dir_derivative": { + "zh": "看着像别的目录的一份副本（{hint}），先放待判断，你来定", + "en": "Looks like a copy of another folder ({hint}); it waits in Pending for you", + }, + # --- grouping -------------------------------------------------------- + "pair": { + "zh": "压缩包和它解压出来的目录是一对，建议留目录", + "en": "The archive and the folder it unpacked into are a pair; keeping the folder is the usual call", + }, + "dup_sha256": { + "zh": "内容一模一样，sha256 对得上，留一份就够", + "en": "Byte for byte the same, the sha256 matches; one copy is enough", + }, + "dup_name": { + "zh": "名字像副本，但内容没有逐字节比过，默认两份都留着", + "en": "The names look like copies, but the contents were never compared, so both stay by default", + }, + "regenerable": { + "zh": "构建产物，删了还能再生成，{restore}", + "en": "Build output, it can be made again ({restore})", + }, + # --- cold storage candidates ---------------------------------------- + "cold_large": { + "zh": "个头不小（{size}），要不要留只有你知道", + "en": "A big one ({size}); only you know whether it is still wanted", + }, + "cold_installer": { + "zh": "安装包或压缩包，装完解完通常就用不上了", + "en": "An installer or archive; once used, it is usually done with", + }, + # --- conflicts and unknowns ----------------------------------------- + "conflict": { + "zh": "目标位置已经有同名的东西，不覆盖，改放重复待确认", + "en": "Something with this name is already at the destination; nothing is overwritten, it goes to Duplicates Review instead", + }, + "unknown_ext": { + "zh": "扩展名不在规则表里，先放待判断，页面上可以改去向", + "en": "The extension is not in the rule table, so it waits in Pending; you can reroute it on the page", + }, + "name_pattern": { + "zh": "命中你自己写的名字规则 {pattern}", + "en": "Matched your own name rule {pattern}", + }, + "tag_group": { + "zh": "按你打的 Finder 标签 {tag} 归到一起", + "en": "Grouped by your Finder tag {tag}", + }, + # --- holds ----------------------------------------------------------- + "hold_aging": { + "zh": "刚下载没多久，才放了 {age} 小时，满 {needed} 小时再动", + "en": "Still fresh, only {age}h in; it moves once it has sat for {needed}h", + }, + "hold_pinned": { + "zh": "你之前把它挪回过顶层，算常驻，不再动它", + "en": "You put it back at the top level once, so it stays put", + }, + "hold_forbidden": { + "zh": "禁刀区，只展示不处理（{label}）", + "en": "No-go zone, shown here but never touched ({label})", + }, + "hold_in_use": { + "zh": "现在有进程开着它，{who}，先不动", + "en": "Something still has it open ({who}), so it stays where it is", + }, + "hold_referenced": { + "zh": "有配置或脚本写死了这个路径（{where}），动它之前先改引用", + "en": "A config or script points at this path ({where}); fix that reference before moving it", + }, + "hold_recent": { + "zh": "十分钟内刚改过，先放一放", + "en": "Changed within the last ten minutes, so it is left alone for now", + }, + "hold_worktree": { + "zh": "这是 git worktree，路径写死在主仓库里，搬了就断", + "en": "This is a git worktree; the main repository records its path, so moving it breaks it", + }, + "hold_no_trash": { + "zh": "这台机器没有可用的废纸篓，处置选项先不给", + "en": "No trash backend on this machine, so disposal is not offered", + }, + # --- hints ----------------------------------------------------------- + "hint_git_repo": { + "zh": "这是 git 仓库，还有 {count} 项没提交，搬之前建议先 push", + "en": "A git repository with {count} uncommitted item(s); consider pushing before moving it", + }, + "hint_venv": { + "zh": "里面有虚拟环境，路径写死在文件里，搬完得重建", + "en": "Contains a virtualenv with absolute paths baked in; rebuild it after the move", + }, + "hint_dotenv": { + "zh": "里面有 .env，搬之前确认没有别的程序在读它", + "en": "Contains a .env; make sure nothing else reads it before moving", + }, + "hint_derivative": { + "zh": "名字里带 {hint}，像是随手复制的一份", + "en": "The name contains {hint}, which looks like a scratch copy", + }, + # --- details --------------------------------------------------------- + "detail_aged": { + "zh": "已经放了 {age} 小时，过了 {needed} 小时的线", + "en": "sat for {age}h, past the {needed}h line", + }, + "detail_sensitive": { + "zh": "命中敏感词 {needle}，不等静置期", + "en": "matched {needle}; no waiting period for this one", + }, + "detail_pair": { + "zh": "和 {other} 同名成对，一起搬进 {dest}", + "en": "pairs with {other} by name; both go to {dest}", + }, + "detail_dup_unverified": { + "zh": "名字像副本，内容没有比过", + "en": "the name looks like a copy; the contents were not compared", + }, + "detail_referenced_warning": { + "zh": "别处引用了它（{where}），你说照常处理，那动完记得改引用", + "en": "It is referenced elsewhere ({where}); you asked to go ahead anyway, so fix that reference afterwards", + }, + "detail_dir_truncated": { + "zh": "目录太大，数到上限就停了，体积只是下限", + "en": "The folder is big; counting stopped at the cap, so the size is a lower bound", + }, + "detail_no_trash": { + "zh": "这台机器没有废纸篓，所以只剩永久删除这一个选项", + "en": "No trash backend on this machine, so permanent delete is the only option offered", + }, + # --- restore methods ------------------------------------------------- + "restore_move": { + "zh": "mv {dest} {src}", + "en": "mv {dest} {src}", + }, + "restore_trash": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash", + }, + "restore_delete": { + "zh": "删了就没了，这一项只对能重新生成的构建产物用", + "en": "Gone for good; only use this on build output that can be regenerated", + }, + # --- folder mess, one line per colour --------------------------------- + "mess_green": { + "zh": "散件 {loose} 件，没有等着你判断的东西，这个目录现在挺清爽", + "en": "{loose} loose items and nothing waiting on you; this folder is in good shape", + }, + "mess_yellow": { + "zh": "散件 {loose} 件，其中 {pending} 件要你看一眼，还没到失控的地步", + "en": "{loose} loose items, {pending} of them want a look from you; not out of hand yet", + }, + "mess_red": { + "zh": "散件 {loose} 件，{pending} 件等你判断，另有 {groups} 组重复或可再生的东西，该花点时间了", + "en": ( + "{loose} loose items, {pending} waiting on you and {groups} duplicate or " + "regenerable groups; this one needs real time" + ), + }, +} + +#: UI copy: CLI lines, report headings, buttons. +UI: Dict[str, Dict[str, str]] = { + "app_name": {"zh": "文件整理", "en": "Carl File Organizer"}, + "tagline": { + "zh": "先站好队，再决定谁走", + "en": "Everything lines up first, then you decide what moves", + }, + "cli_plan_written": {"zh": "计划写在 {path}", "en": "Plan written to {path}"}, + "cli_report_written": {"zh": "审核页在 {path}，双击就能看", "en": "Review page at {path}, open it in a browser"}, + "cli_nothing_moved": { + "zh": "本次只读扫描，没有移动或删除任何文件。", + "en": "Read-only scan. Nothing was moved or deleted.", + }, + "cli_dotfile_written": {"zh": "设置记在 {path}，下次不用再传参数", "en": "Settings remembered in {path}, no flags needed next time"}, + "cli_notes_merged": { + "zh": "说明已并入计划 {path}，{count} 条提醒", + "en": "Notes merged into the plan at {path}, {count} warning(s)", + }, + "cli_lang_source": {"zh": "语言用 {lang}，依据是 {source}", "en": "Language {lang}, decided by {source}"}, + "cli_not_implemented": { + "zh": "子命令 {command} 还没接上，{module} 里没有 run 入口。", + "en": "Subcommand {command} is not wired up yet; {module} has no run() entry point.", + }, + "cli_lang_ambiguous": { + "zh": "这个目录里中英文分区都有，说不准该用哪套，请加 --lang zh 或 --lang en。", + "en": "This folder has both Chinese and English partitions and it is not clear which to follow; pass --lang zh or --lang en.", + }, + "warning_prefix": {"zh": "提醒，", "en": "warning: "}, + "section_pending_moves": {"zh": "等你批准的移动", "en": "Moves awaiting your go-ahead"}, + "section_groups": {"zh": "配对与重复", "en": "Pairs and duplicates"}, + "section_disposal": {"zh": "可再生与冷存候选", "en": "Regenerable and cold candidates"}, + "section_holding": {"zh": "还在静置的与常驻的", "en": "Settling and pinned"}, + "section_forbidden": {"zh": "禁刀区", "en": "No-go zone"}, + "section_undecided": {"zh": "待判断", "en": "Undecided"}, + "button_export": {"zh": "导出批准文件", "en": "Export approved plan"}, + "button_apply": {"zh": "执行已勾选的项", "en": "Apply the checked items"}, + "button_show_delete": {"zh": "显示永久删除选项", "en": "Show permanent delete options"}, + "banner_reversible": { + "zh": "移动只改路径，内容一个字节都不动。哪个软件找不到文件了，用 undo 或者清单里的 mv 命令原路放回。", + "en": "Moving only changes the path, never the contents. If some app loses track of a file, run undo or the mv line in the manifest.", + }, + "banner_do_not_touch": { + "zh": "计划生成之后先别动这些目录，中途改过的条目执行时会自动跳过。", + "en": "Leave these folders alone once the plan exists; anything changed in between is skipped at apply time.", + }, + "option_keep_all": {"zh": "都保留，一起搬进待判断", "en": "Keep both and move them to Pending together"}, + "option_keep_one": {"zh": "留 {keep}，{drop} 进废纸篓", "en": "Keep {keep}, send {drop} to the Trash"}, + "option_keep_one_delete": { + "zh": "留 {keep}，{drop} 永久删除", + "en": "Keep {keep}, permanently delete {drop}", + }, + "option_trash_all": {"zh": "全部进废纸篓", "en": "Send all of them to the Trash"}, + "option_delete_all": {"zh": "全部永久删除", "en": "Permanently delete all of them"}, + "option_keep_all_plain": {"zh": "都保留", "en": "Keep both"}, + "cli_report_skipped": { + "zh": "审核页还没接上（{module}），这次只写了计划文件。", + "en": "The review page is not wired up yet ({module}); only the plan file was written.", + }, + "cli_scan_summary": { + "zh": "扫到 {entries} 项，候选动作 {actions} 条，其中 {approvable} 条等你批准。", + "en": "Scanned {entries} item(s), {actions} candidate action(s), {approvable} of them waiting on you.", + }, + "cli_storage_written": { + "zh": "整机盘点写在 {path}", + "en": "Whole-machine scan written to {path}", + }, + "cli_storage_skipped": { + "zh": "这次跳过整机盘点，报告里就只有整理这一半。", + "en": "The whole-machine scan was skipped, so the report carries the tidy-up half only.", + }, + "cli_storage_failed": { + "zh": "整机盘点没跑成（{error}），整理这一半照常。", + "en": "The whole-machine scan did not finish ({error}); the tidy-up half is unaffected.", + }, + "cli_next_steps": { + "zh": "下一步，把每一项的人话写进 {notes}，把盘点定色写进 {analysis}，然后跑 report {managed}。", + "en": "Next: write the plain-language notes into {notes}, the colour calls into {analysis}, then run report {managed}.", + }, + "cli_report_combined": { + "zh": "一份报告写在 {path}，清理、搬动和整理后预览都在里面。", + "en": "One report at {path}: the cleanup, the moves and the after picture together.", + }, + "cli_nothing_to_report": { + "zh": "{managed} 里既没有 plan.json 也没有 analysis.json，先跑一次 scan。", + "en": "{managed} holds neither plan.json nor analysis.json; run scan first.", + }, + "cli_decisions_plan": {"zh": "先搬动这一半", "en": "The moves first"}, + "cli_decisions_storage": {"zh": "再清理这一半", "en": "The cleanup second"}, + "cli_decisions_empty": { + "zh": "这份决定清单一项都没勾，什么都没做。", + "en": "Nothing is ticked in this decisions file, so nothing was done.", + }, + "cli_decisions_summary": { + "zh": "两段合计，搬动这段返回 {plan}，清理这段返回 {storage}。", + "en": "Both halves done: the moves returned {plan}, the cleanup returned {storage}.", + }, + "cli_decisions_no_analysis": { + "zh": "决定清单里没带 analysis.json，加 --analysis 指给我，或者改用 dispose 子命令。", + "en": "The decisions file carries no analysis.json; pass --analysis, or use the dispose subcommand.", + }, + "cli_decisions_no_plan": { + "zh": "决定清单里没带 plan.json，加 --plan 指给我。", + "en": "The decisions file carries no plan.json; pass --plan.", + }, + "status_last_run": {"zh": "最后整理时间", "en": "Last tidy-up"}, + "status_dirty_now": {"zh": "现在是否又脏了", "en": "Messy again?"}, + "status_never": {"zh": "还没整理过", "en": "not tidied yet"}, +} + + +def _lookup(table: Mapping[str, Mapping[str, str]], key: str, lang: str) -> str: + entry = table.get(key) + if entry is None: + return key + chosen = entry.get(lang) + if chosen is None: + chosen = entry.get(DEFAULT_LANG) + if chosen is None: + chosen = next(iter(entry.values()), key) + return chosen + + +def _format(template: str, kw: Mapping[str, Any]) -> str: + if not kw: + return template + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def t(key: str, lang: str = DEFAULT_LANG, /, **kw: Any) -> str: + """UI copy for ``key`` in ``lang``; unknown keys return the key itself. + + ``key`` and ``lang`` are positional-only so a template placeholder may be + called ``lang`` without colliding with the parameter. + """ + + return _format(_lookup(UI, key, lang), kw) + + +def reason(key: str, lang: str = DEFAULT_LANG, /, **kw: Any) -> str: + """Plan reasoning for ``key`` in ``lang``; unknown keys return the key itself.""" + + return _format(_lookup(REASONS, key, lang), kw) + + +def both(key: str, /, **kw: Any) -> Dict[str, str]: + """``{"zh": ..., "en": ...}`` for a reason key, ready to drop into plan.json.""" + + return {lang: reason(key, lang, **kw) for lang in LANGS} + + +def ui_both(key: str, /, **kw: Any) -> Dict[str, str]: + """``{"zh": ..., "en": ...}`` for a UI key.""" + + return {lang: t(key, lang, **kw) for lang in LANGS} + + +def has_reason(key: str) -> bool: + return key in REASONS diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py new file mode 100644 index 0000000..d95edca --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/ids.py @@ -0,0 +1,24 @@ +"""Stable identifiers shared by planner, executor, server and undo. + +subject_id identifies a file or directory as it was observed (path + kind + +size + mtime). action_id identifies one proposed operation on that subject. +Both are truncated SHA-256 hex digests; any change to the underlying file +invalidates the id, which is how tampering and drift are detected. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + + +def subject_id(path: Path, kind: str, size: int, modified_ns: int) -> str: + real = os.path.realpath(str(path)) + material = f"{real}\0{kind}\0{size}\0{modified_ns}".encode("utf-8") + return hashlib.sha256(material).hexdigest()[:16] + + +def action_id(subject: str, kind: str, destination_portable: str | None) -> str: + material = f"{subject}\0{kind}\0{destination_portable or ''}".encode("utf-8") + return hashlib.sha256(material).hexdigest()[:16] diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py new file mode 100644 index 0000000..e0bc895 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/manifest.py @@ -0,0 +1,381 @@ +"""Paper trail: TSV manifests, tag lists, audit log, the human review log. + +Four kinds of file live in the managed directory, and every one of them is +written for a person to read first and a program second: + + downloads-moves-.tsv what moved, with a restore command per row + downloads-move-tags-.txt the paths that got the temporary Finder tag + audit.jsonl one JSON line per executed action + 人工调整日志.md / review-log.md a dated summary in plain sentences + +The first seven manifest columns are byte-for-byte the ones the reference +script has been writing since August, so old manifests stay readable and old +readers keep working; ``kind`` and ``action_id`` are appended after them. + +Paths inside these files are always written in ``$HOME/...`` form. +""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass, field +from datetime import date, datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence + +MANIFEST_COLUMNS = ( + "moved_at", + "status", + "original_path", + "target_path", + "size_bytes", + "reason", + "restore_method", + "kind", + "action_id", +) + +#: The seven columns the reference script wrote; anything after them is extra. +LEGACY_COLUMNS = MANIFEST_COLUMNS[:7] + +MANIFEST_SERIES = ("conflict-moves", "moves", "removals", "undo") + +#: Series that count as a tidy-up run when answering "when was this last done". +TIDY_SERIES = ("conflict-moves", "moves", "removals") + +AUDIT_NAME = "audit.jsonl" +TAG_LIST_SERIES = "move-tags" +TIMESTAMP_FORMAT = "%Y-%m-%d_%H-%M-%S" + +REVIEW_LOG_NAMES = {"zh": "人工调整日志.md", "en": "review-log.md"} +REVIEW_LOG_TITLES = {"zh": "# 人工调整日志", "en": "# Review log"} + +_MANIFEST_RE = re.compile( + r"^(?P.+?)-(?Pconflict-moves|moves|removals|undo)-" + r"(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.tsv$" +) + +#: Manifest statuses that mean the action really happened. +DONE_STATUSES = ("MOVED", "TRASHED", "DELETED", "RESTORED") + + +@dataclass +class RunInfo: + """One past run, read from a manifest file name and its rows.""" + + path: Path + series: str + timestamp: Optional[datetime] + moved_at: str + rows: int + done: int + bytes: int = 0 + statuses: Dict[str, int] = field(default_factory=dict) + + +# -------------------------------------------------------------------------- +# names and small helpers +# -------------------------------------------------------------------------- + + +def safe_field(value: Any) -> str: + """Flatten a value so it cannot break the tab separated layout.""" + + return str(value).replace("\t", " ").replace("\r", " ").replace("\n", " ") + + +def slug(name: str) -> str: + """``"Downloads"`` -> ``"downloads"``; used as the manifest name prefix.""" + + folded = str(name).strip().casefold() + folded = re.sub(r"[\s/\\]+", "-", folded) + folded = re.sub(r"[^0-9a-z一-鿿._-]+", "", folded) + folded = folded.strip("-._") + return folded or "folder" + + +def timestamp_text(ts: datetime) -> str: + return ts.strftime(TIMESTAMP_FORMAT) + + +def manifest_path(managed: Path, root_name: str, series: str, ts: datetime) -> Path: + """``downloads-moves-2026-09-02_10-20-11.tsv`` inside the managed directory.""" + + if series not in MANIFEST_SERIES: + raise ValueError("unknown manifest series: {0}".format(series)) + return Path(managed) / "{0}-{1}-{2}.tsv".format( + slug(root_name), series, timestamp_text(ts) + ) + + +def tag_list_path(managed: Path, root_name: str, ts: datetime) -> Path: + """``downloads-move-tags-2026-09-02_10-20-11.txt``.""" + + return Path(managed) / "{0}-{1}-{2}.txt".format( + slug(root_name), TAG_LIST_SERIES, timestamp_text(ts) + ) + + +def audit_path(managed: Path) -> Path: + return Path(managed) / AUDIT_NAME + + +def review_log_name(lang: str) -> str: + return REVIEW_LOG_NAMES.get(lang or "en", REVIEW_LOG_NAMES["en"]) + + +def _atomic_write(path: Path, text: str) -> Path: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + with temporary.open("w", encoding="utf-8", newline="") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + os.replace(str(temporary), str(target)) + return target + + +def atomic_write(path: Path, text: str) -> Path: + """Write ``text`` to ``path`` through a temporary file and one ``fsync``. + + The tidy-up manifests are not the only paper trail in this package: the + storage line writes a differently shaped table of its own. Both want the + same guarantee, that a file which exists is a file that is complete, so the + primitive is public and there is only one of it. + """ + + return _atomic_write(path, text) + + +# -------------------------------------------------------------------------- +# manifests +# -------------------------------------------------------------------------- + + +def write_manifest(path: Path, rows: Sequence[Mapping[str, Any]]) -> Path: + """Write every row under :data:`MANIFEST_COLUMNS`, atomically.""" + + lines = ["\t".join(MANIFEST_COLUMNS)] + for row in rows: + lines.append( + "\t".join(safe_field(row.get(column, "")) for column in MANIFEST_COLUMNS) + ) + return _atomic_write(path, "\n".join(lines) + "\n") + + +def read_manifest(path: Path) -> List[Dict[str, str]]: + """Read a manifest by its own header line, so seven column files still parse.""" + + target = Path(path) + if not target.is_file(): + raise FileNotFoundError(str(target)) + text = target.read_text(encoding="utf-8") + lines = [line for line in text.split("\n") if line != ""] + if not lines: + return [] + header = lines[0].split("\t") + rows: List[Dict[str, str]] = [] + for line in lines[1:]: + fields = line.split("\t") + row: Dict[str, str] = {} + for index, column in enumerate(header): + row[column] = fields[index] if index < len(fields) else "" + rows.append(row) + return rows + + +def write_tag_list(path: Path, portable_paths: Iterable[str]) -> Path: + items = [safe_field(item) for item in portable_paths] + return _atomic_write(path, "\n".join(items) + ("\n" if items else "")) + + +def read_tag_list(path: Path) -> List[str]: + target = Path(path) + if not target.is_file(): + return [] + return [ + line.strip() + for line in target.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# -------------------------------------------------------------------------- +# audit log +# -------------------------------------------------------------------------- + + +def append_audit(results: Iterable[Mapping[str, Any]], path: Path) -> Path: + """Append audit records; this is the authority when a run is interrupted.""" + + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("a", encoding="utf-8") as handle: + for record in results: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + handle.flush() + os.fsync(handle.fileno()) + return target + + +def read_audit(path: Path) -> List[Dict[str, Any]]: + target = Path(path) + if not target.is_file(): + raise FileNotFoundError(str(target)) + records: List[Dict[str, Any]] = [] + for line in target.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped: + continue + try: + record = json.loads(stripped) + except ValueError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +# -------------------------------------------------------------------------- +# human review log +# -------------------------------------------------------------------------- + + +def append_review_log( + managed: Path, + lang: str, + day: date, + bullets: Sequence[str], + *, + name: Optional[str] = None, +) -> Path: + """Add ``bullets`` under a ``## YYYY-MM-DD`` heading, creating what is missing.""" + + target = Path(managed) / (name or review_log_name(lang)) + target.parent.mkdir(parents=True, exist_ok=True) + heading = "## {0}".format(day.isoformat()) + items = ["- {0}".format(safe_field(bullet)) for bullet in bullets if str(bullet).strip()] + if not items: + return target + + if target.is_file(): + lines = target.read_text(encoding="utf-8").split("\n") + else: + lines = [REVIEW_LOG_TITLES.get(lang or "en", REVIEW_LOG_TITLES["en"]), ""] + + start = None + for index, line in enumerate(lines): + if line.strip() == heading: + start = index + break + + if start is None: + while lines and lines[-1].strip() == "": + lines.pop() + lines.extend(["", heading, ""]) + lines.extend(items) + lines.append("") + else: + end = len(lines) + for index in range(start + 1, len(lines)): + if lines[index].startswith("## "): + end = index + break + section = lines[start:end] + while section and section[-1].strip() == "": + section.pop() + section.extend(items) + section.append("") + lines[start:end] = section + + target.write_text("\n".join(lines).rstrip("\n") + "\n", encoding="utf-8") + return target + + +# -------------------------------------------------------------------------- +# last run +# -------------------------------------------------------------------------- + + +def _parse_timestamp(text: str) -> Optional[datetime]: + try: + return datetime.strptime(text, TIMESTAMP_FORMAT) + except (ValueError, TypeError): + return None + + +def _row_bytes(row: Mapping[str, str]) -> int: + try: + return int(str(row.get("size_bytes", "") or 0)) + except ValueError: + return 0 + + +def run_info(path: Path) -> Optional[RunInfo]: + """Read one manifest file into a :class:`RunInfo`, or ``None`` if it is not one.""" + + target = Path(path) + match = _MANIFEST_RE.match(target.name) + if match is None: + return None + try: + rows = read_manifest(target) + except (OSError, ValueError): + rows = [] + statuses: Dict[str, int] = {} + moved_at = "" + total = 0 + done = 0 + for row in rows: + status = str(row.get("status", "")).strip().upper() + statuses[status] = statuses.get(status, 0) + 1 + stamp = str(row.get("moved_at", "")).strip() + if stamp > moved_at: + moved_at = stamp + if status in DONE_STATUSES: + done += 1 + total += _row_bytes(row) + return RunInfo( + path=target, + series=match.group("series"), + timestamp=_parse_timestamp(match.group("ts")), + moved_at=moved_at or match.group("ts").replace("_", " ").replace("-", "-"), + rows=len(rows), + done=done, + bytes=total, + statuses=statuses, + ) + + +def list_runs(managed: Path, *, series: Sequence[str] = TIDY_SERIES) -> List[RunInfo]: + """Every manifest in ``managed``, newest first, judged only by file names.""" + + folder = Path(managed) + if not folder.is_dir(): + return [] + found: List[RunInfo] = [] + try: + names = sorted(entry.name for entry in folder.iterdir() if entry.is_file()) + except OSError: + return [] + for name in names: + match = _MANIFEST_RE.match(name) + if match is None or match.group("series") not in series: + continue + info = run_info(folder / name) + if info is not None: + found.append(info) + found.sort( + key=lambda item: (item.timestamp or datetime.min, item.path.name), reverse=True + ) + return found + + +def latest_run(managed: Path, *, series: Sequence[str] = TIDY_SERIES) -> Optional[RunInfo]: + """The most recent run, read from manifest names only; folder mtimes lie.""" + + runs = list_runs(managed, series=series) + return runs[0] if runs else None diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py new file mode 100644 index 0000000..862db2e --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/notes.py @@ -0,0 +1,285 @@ +"""The agent explanation layer: prose laid on top of a plan, changing nothing. + +The rules decide, and they say why in one fixed sentence per rule. That reads +like a form letter, because it is one. An agent that has looked at the folder +can say what a thing actually is, why it is there and what breaks if it goes +away, and that is what this module carries. + +The one rule that makes it safe: notes are text. :func:`merge_notes` copies +nothing but strings, and it copies them into ``plan["notes"]``, never into an +action. No note can add an action, approve one, change a destination, change a +colour on an action, or turn a hold into anything else. The executor still only +knows the ids the planner computed, so a wrong note is a wrong sentence and +nothing worse. + +The single exception is the folder colour: thresholds are a guess about a folder +an agent has actually looked at, so ``notes["mess"]["color"]`` may override +``plan["mess"]["color"]``. The original stays in ``rule_color`` and the plan +records who decided, so nobody has to wonder later. + + plan = merge_notes(plan, json.loads(Path("notes.json").read_text())) + +Unknown ids only warn. A note left over from a previous scan is a stale +sentence, not a reason to refuse to write a report. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from . import i18n +from .planner import COLORS + +#: Version of the notes.json shape. Bumped only when a field is removed. +NOTES_SCHEMA = 1 + +#: The three sentences an agent may write about one action or one group. +ITEM_FIELDS = ("what", "why", "if_removed") + +#: Top level keys notes.json is allowed to carry. +TOP_FIELDS = ("mess", "actions", "groups", "folder_line") + +#: Longest a single sentence may be; longer is truncated, never dropped. +MAX_TEXT = 600 + + +class NotesError(ValueError): + """The notes file is not a JSON object at all. Everything else only warns.""" + + +# --------------------------------------------------------------------------- +# reading and cleaning +# --------------------------------------------------------------------------- + + +def load_notes(path: Path) -> Dict[str, Any]: + """Read notes.json, or raise :class:`NotesError` with a readable line.""" + + try: + raw = Path(path).read_text(encoding="utf-8") + except OSError as error: + raise NotesError("cannot read notes file {0}: {1}".format(path, error)) from error + try: + data = json.loads(raw) + except ValueError as error: + raise NotesError("notes file {0} is not valid JSON: {1}".format(path, error)) from error + if not isinstance(data, dict): + raise NotesError("notes file {0} must hold a JSON object".format(path)) + return data + + +def _text(value: Any) -> Optional[str]: + if not isinstance(value, str): + return None + cleaned = value.strip() + if not cleaned: + return None + return cleaned[:MAX_TEXT] + + +def _clean_item( + raw: Any, where: str, warnings: List[str] +) -> Optional[Dict[str, str]]: + """Keep the three known sentences, drop everything else with a warning.""" + + if not isinstance(raw, dict): + warnings.append("{0}: expected an object, ignored".format(where)) + return None + kept: Dict[str, str] = {} + for key, value in raw.items(): + if key not in ITEM_FIELDS: + warnings.append("{0}: unknown field {1!r}, ignored".format(where, key)) + continue + text = _text(value) + if text is None: + warnings.append("{0}.{1}: not a non-empty string, ignored".format(where, key)) + continue + kept[key] = text + if not kept: + return None + return kept + + +def _clean_map( + raw: Any, + known_ids: Mapping[str, Any], + label: str, + warnings: List[str], +) -> Dict[str, Dict[str, str]]: + if raw is None: + return {} + if not isinstance(raw, dict): + warnings.append("{0}: expected an object keyed by id, ignored".format(label)) + return {} + cleaned: Dict[str, Dict[str, str]] = {} + for key, value in raw.items(): + if not isinstance(key, str): + warnings.append("{0}: non-string id ignored".format(label)) + continue + if key not in known_ids: + warnings.append("{0}: no such id in this plan: {1}".format(label, key)) + continue + item = _clean_item(value, "{0}[{1}]".format(label, key), warnings) + if item is not None: + cleaned[key] = item + return cleaned + + +def _clean_mess(raw: Any, warnings: List[str]) -> Dict[str, Any]: + if raw is None: + return {} + if not isinstance(raw, dict): + warnings.append("mess: expected an object, ignored") + return {} + cleaned: Dict[str, Any] = {} + for key, value in raw.items(): + if key == "color": + if value in COLORS: + cleaned["color"] = value + else: + warnings.append("mess.color: not one of {0}, ignored".format(", ".join(COLORS))) + elif key == "line": + text = _text(value) + if text is None: + warnings.append("mess.line: not a non-empty string, ignored") + else: + cleaned["line"] = text + else: + warnings.append("mess: unknown field {0!r}, ignored".format(key)) + return cleaned + + +# --------------------------------------------------------------------------- +# merging +# --------------------------------------------------------------------------- + + +def validate_notes( + plan: Mapping[str, Any], notes: Mapping[str, Any] +) -> Tuple[Dict[str, Any], List[str]]: + """``(cleaned notes, warnings)``. Nothing here reads the filesystem.""" + + warnings: List[str] = [] + if not isinstance(notes, dict): + raise NotesError("notes must be a JSON object") + + action_ids = { + str(action.get("id")): action for action in (plan.get("actions") or []) if isinstance(action, dict) + } + group_ids = { + str(group.get("group_id")): group for group in (plan.get("groups") or []) if isinstance(group, dict) + } + + for key in notes: + if key not in TOP_FIELDS: + warnings.append("unknown top level field {0!r}, ignored".format(key)) + + cleaned: Dict[str, Any] = { + "schema": NOTES_SCHEMA, + "mess": _clean_mess(notes.get("mess"), warnings), + "actions": _clean_map(notes.get("actions"), action_ids, "actions", warnings), + "groups": _clean_map(notes.get("groups"), group_ids, "groups", warnings), + "folder_line": _text(notes.get("folder_line")), + } + if notes.get("folder_line") is not None and cleaned["folder_line"] is None: + warnings.append("folder_line: not a non-empty string, ignored") + return cleaned, warnings + + +def merge_notes(plan: Mapping[str, Any], notes: Mapping[str, Any]) -> Dict[str, Any]: + """A copy of ``plan`` with ``notes`` merged into ``plan["notes"]``. + + The input plan is not touched. Merging twice is the same as merging once + with the second set of notes: the previous block is replaced, not appended, + so a re-run after fixing a sentence does not leave the old one behind. + """ + + if not isinstance(plan, dict): + raise NotesError("plan must be a JSON object") + cleaned, warnings = validate_notes(plan, notes) + merged = copy.deepcopy(dict(plan)) + cleaned["warnings"] = warnings + merged["notes"] = cleaned + + mess = merged.get("mess") + if isinstance(mess, dict): + # Put the rule's own answer somewhere permanent before anything can + # overwrite it, whether or not the notes carry a colour. + mess.setdefault("rule_color", mess.get("color")) + chosen = cleaned["mess"].get("color") + if chosen: + mess["color"] = chosen + mess["color_source"] = "notes" + else: + mess["color_source"] = mess.get("color_source") or "rule" + line = cleaned["mess"].get("line") + if line: + mess["line"] = line + elif cleaned["mess"]: + warnings.append("mess: this plan has no mess block, the folder colour was not changed") + return merged + + +def merge_notes_file(plan: Mapping[str, Any], path: Path) -> Dict[str, Any]: + """:func:`merge_notes` with the reading done for you.""" + + return merge_notes(plan, load_notes(path)) + + +# --------------------------------------------------------------------------- +# CLI entry point: carl-file-organizer build +# --------------------------------------------------------------------------- + + +def run(args: Any) -> int: + """``build``: lay notes.json over a plan.json and write the plan back.""" + + plan_path = Path(getattr(args, "plan")) + lang = getattr(args, "lang", None) or "en" + try: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + except OSError as error: + raise NotesError("cannot read plan {0}: {1}".format(plan_path, error)) from error + except ValueError as error: + raise NotesError("plan {0} is not valid JSON: {1}".format(plan_path, error)) from error + if not isinstance(plan, dict): + raise NotesError("plan {0} must hold a JSON object".format(plan_path)) + lang = plan.get("lang") or lang + + notes_path = getattr(args, "notes", None) + if notes_path is None: + merged = plan + warnings: List[str] = [] + else: + merged = merge_notes(plan, load_notes(Path(notes_path))) + warnings = list((merged.get("notes") or {}).get("warnings") or ()) + + output = Path(getattr(args, "output", None) or plan_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(merged, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + for line in warnings: + print(i18n.t("warning_prefix", lang) + line) + print(i18n.t("cli_notes_merged", lang, path=output, count=len(warnings))) + + if bool(getattr(args, "report", False)): + try: + from . import render + + report_path = output.parent / "report.html" + report_path.write_text( + render.render(merged, mode="static", lang=lang), encoding="utf-8" + ) + except Exception as error: # noqa: BLE001 - the plan is the deliverable + print( + i18n.t("cli_report_skipped", lang, module="build_report.py") + + " ({0})".format(error) + ) + else: + print(i18n.t("cli_report_written", lang, path=report_path)) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py new file mode 100644 index 0000000..e46de3b --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/opener.py @@ -0,0 +1,82 @@ +"""Show a path to the person sitting in front of the machine. + +This is the only thing offered for the items the tool refuses to touch: it +cannot move them, it cannot dispose of them, but it can put the folder on +screen with the item already selected, so the decision is made by a person +looking at the real thing. + +Nothing here writes, deletes or renames. Every call goes through +:func:`subprocess.Popen` with an argument list and never through a shell, so a +file called ``$(rm -rf ~).pdf`` is a file with a silly name and nothing else. + + reveal(Path("~/Downloads/report.pdf").expanduser()) + +Returns True when the platform's file manager was launched, False when there is +nothing to launch it with; it never raises for a missing file manager, because +"I could not show you this" is not worth ending a review session over. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable, List, Optional + +DARWIN = "darwin" +WINDOWS = "win32" + +#: Handed to Popen so a file manager that lives a long time does not keep the +#: review server alive with it. +_POPEN_KW = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL} + + +def supported(platform: Optional[str] = None) -> bool: + """True when this machine has a file manager we know how to call.""" + + return reveal_command(Path.home(), platform=platform) is not None + + +def reveal_command(path: Path, *, platform: Optional[str] = None) -> Optional[List[str]]: + """The exact argv used to reveal ``path``, or None when there is none. + + Split out from :func:`reveal` so the shape of each platform's command can be + asserted from any other platform. macOS and Windows both select the item + itself; ``xdg-open`` has no such option anywhere, so on Linux the parent + folder is opened and the item is merely visible in it. + """ + + system = platform or sys.platform + absolute = os.path.abspath(str(path)) + + if system == DARWIN: + return ["open", "-R", absolute] + if system == WINDOWS: + # explorer wants exactly this comma with no space after it, and it is + # one argument, not two. + return ["explorer", "/select,{0}".format(absolute)] + opener = shutil.which("xdg-open") + if opener is None: + return None + return [opener, os.path.dirname(absolute) or absolute] + + +def reveal( + path: Path, + *, + platform: Optional[str] = None, + runner: Optional[Callable[..., Any]] = None, +) -> bool: + """Open the file manager with ``path`` shown. Never raises for a missing one.""" + + argv = reveal_command(Path(path), platform=platform) + if argv is None: + return False + launch = runner if runner is not None else subprocess.Popen + try: + launch(argv, **_POPEN_KW) + except OSError: + return False + return True diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py new file mode 100644 index 0000000..322e87d --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/paths.py @@ -0,0 +1,438 @@ +"""Path helpers: portable $HOME form, boundary checks, safe names, no-go zones. + +This module never imports other Carl File Organizer modules, so every work package can +depend on it. ``config`` objects are accepted by duck typing (anything exposing +``.profile`` as a mapping is fine). + +It also owns the two halves of the privacy rule, because both are pure path +arithmetic: :func:`strip_absolute` drops every absolute path out of a plan +before it is embedded in a shareable page, and +:func:`hydrate_absolute` puts them back from the ``$HOME`` twins at apply time. +""" + +from __future__ import annotations + +import copy +import fnmatch +import os +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +#: The one token written to disk, on every platform. plan.json, the manifests, +#: the audit log and report.html all use this and only this, because those files +#: travel: a plan made on a Mac gets opened on a Windows machine, and a report is +#: a page somebody forwards. One stored token means one parser and one privacy +#: check, and that check is a single ``startswith`` over the whole document. +HOME_TOKEN = "$HOME" + +#: The same idea spelled the way Windows spells it. Understood on input, and +#: shown on screen by :func:`display_portable`, but never written into a file. +WINDOWS_HOME_TOKEN = "%USERPROFILE%" + +#: Both spellings, for everything that reads rather than writes. +HOME_TOKENS = (HOME_TOKEN, WINDOWS_HOME_TOKEN) + +#: Path separators a stored portable string may arrive with. +SEPARATORS = ("/", "\\") + + +def home_token(platform: Optional[str] = None) -> str: + """The token a person on this platform recognises at a glance.""" + + system = platform if platform is not None else sys.platform + return WINDOWS_HOME_TOKEN if system == "win32" else HOME_TOKEN + + +def split_home_token(text: str) -> Optional[Tuple[str, str]]: + """``(token, remainder)`` when ``text`` starts with either home token. + + The remainder has no leading separator and always comes back with forward + slashes, so callers can hand it straight to :class:`~pathlib.Path`. + """ + + if not isinstance(text, str): + return None + for token in HOME_TOKENS: + if text == token: + return token, "" + for separator in SEPARATORS: + prefix = token + separator + if text.startswith(prefix): + rest = text[len(prefix) :] + return token, rest.replace("\\", "/") + return None + + +def display_portable(text: str, platform: Optional[str] = None) -> str: + """Rewrite a stored ``$HOME/...`` string for the eyes of this platform. + + Storage never changes; only the rendering does. Everywhere but Windows + this hands back exactly what it was given. + """ + + token = home_token(platform) + if token == HOME_TOKEN: + return text + split = split_home_token(text) + if split is None: + return text + _found, rest = split + if not rest: + return token + return token + "\\" + rest.replace("/", "\\") + +#: Compound suffixes that must survive as one unit, so ``a.tar.gz`` is an archive. +#: :mod:`carl_file_organizer.scanner` re-exports this pair; there is only one spelling. +DOUBLE_SUFFIXES = (".tar.gz", ".tar.bz2", ".tar.xz", ".tar.zst", ".tar.lz") + +#: Envelope fields that hold an absolute path and have a ``_portable`` twin. +PLAN_ABSOLUTE_FIELDS = ("source_root", "managed_dir") + +#: The same, per action. +ACTION_ABSOLUTE_FIELDS = ("source", "destination") + +#: Roots that are refused outright, relative to ``$HOME``. +REFUSED_HOME_SUBDIRS = ("Library", "Applications") + +#: Absolute roots that are refused outright regardless of ``$HOME``. +REFUSED_ABSOLUTE = ("/", "/System", "/Library", "/Applications", "/private", "/usr", "/bin", "/etc", "/var") + + +def _home(home: Optional[Path] = None) -> Path: + return Path(os.path.realpath(str(home))) if home is not None else Path(os.path.realpath(str(Path.home()))) + + +def realpath(path: Path) -> Path: + """``os.path.realpath`` as a :class:`Path`, without touching the filesystem twice.""" + + return Path(os.path.realpath(str(path))) + + +def portable(path: Path, home: Optional[Path] = None) -> str: + """Render ``path`` as ``$HOME/...`` when it lives under the home directory. + + Forward slashes on every platform, because this string is stored, compared + and shipped rather than opened. :func:`display_portable` is what turns it + into something a Windows user reads. + """ + + base = home if home is not None else Path.home() + text = str(path) + try: + return HOME_TOKEN + "/" + Path(text).relative_to(base).as_posix() + except ValueError: + return text + + +def expand_portable(text: str, home: Optional[Path] = None) -> Path: + """Inverse of :func:`portable`, reading either home token. + + A plan written on Windows and applied on a Mac, or a person who typed + ``%USERPROFILE%\\Downloads`` into a settings file by hand, both land here + and both come out as a real path on this machine. + """ + + base = home if home is not None else Path.home() + split = split_home_token(text) + if split is not None: + _token, rest = split + return Path(base) / rest if rest else Path(base) + return Path(text).expanduser() + + +def inside_root(path: Path, root: Path) -> bool: + """True when ``path`` resolves inside ``root`` (both sides realpath'ed).""" + + resolved = realpath(path) + base = realpath(root) + if resolved == base: + return True + try: + resolved.relative_to(base) + return True + except ValueError: + return False + + +def has_symlink_component(path: Path, root: Path) -> bool: + """True when any path segment below ``root`` is a symlink.""" + + base = realpath(root) + try: + relative = Path(path).relative_to(root) + except ValueError: + try: + relative = realpath(path).relative_to(base) + except ValueError: + return True + current = Path(root) + for part in relative.parts: + current = current / part + if current.is_symlink(): + return True + return False + + +def safe_dir_name(name: str) -> str: + """Turn an arbitrary entry name into something safe to use as a directory name.""" + + cleaned = name.replace("\x00", "").replace("/", "-").replace(os.sep, "-") + cleaned = cleaned.lstrip("$").strip() + had_quote = '"' in cleaned or "'" in cleaned + cleaned = cleaned.replace('"', "").replace("'", "").strip() + cleaned = cleaned.strip(".").strip() + if not cleaned: + return "quote-dir" if had_quote else "unnamed" + return cleaned + + +def split_suffix(name: str) -> Tuple[str, str]: + """``("archive", ".tar.gz")``; a leading dot never counts as a suffix. + + The stem keeps its original case, the suffix is folded, so callers comparing + against a rule table do not have to fold again. ``name[len(stem):]`` gives + the suffix back exactly as it was written. + """ + + folded = name.casefold() + for double in DOUBLE_SUFFIXES: + if folded.endswith(double) and len(name) > len(double): + return name[: -len(double)], double + stem, dot, ext = name.rpartition(".") + if not dot or not stem: + return name, "" + return stem, ("." + ext).casefold() + + +def conflict_name(name: str, ts: datetime, *, is_dir: bool = False) -> str: + """Name used when a destination is taken: ``quarterly-report-2026-09-02_10-20-11.pdf``. + + The stamp goes before the suffix so the copy still opens with a double click. + Directories and names without a suffix keep the plain ``-`` + shape, and ``a.tar.gz`` stays one unit because the split is the scanner's. + """ + + safe = safe_dir_name(name) + stamp = ts.strftime("%Y-%m-%d_%H-%M-%S") + if is_dir: + return "{0}-{1}".format(safe, stamp) + stem, folded = split_suffix(safe) + if not folded: + return "{0}-{1}".format(safe, stamp) + return "{0}-{1}{2}".format(stem, stamp, safe[len(stem) :]) + + +def _forbidden_section(config: Any) -> dict: + profile = getattr(config, "profile", config) + section = profile.get("forbidden") if hasattr(profile, "get") else None + return section if isinstance(section, dict) else {} + + +def is_forbidden(path: Path, config: Any) -> Optional[str]: + """Return a rule label when ``path`` sits in a no-go zone, otherwise ``None``.""" + + section = _forbidden_section(config) + name = Path(path).name + folded = name.casefold() + + for basename in section.get("basenames", []) or []: + if name == basename or folded == str(basename).casefold(): + return "forbidden:basename:{0}".format(basename) + + for suffix in section.get("suffixes", []) or []: + if folded.endswith(str(suffix).casefold()): + return "forbidden:suffix:{0}".format(suffix) + + for pattern in section.get("glob", []) or []: + if fnmatch.fnmatch(name, str(pattern)): + return "forbidden:glob:{0}".format(pattern) + + try: + is_dir = Path(path).is_dir() + except OSError: + is_dir = False + if is_dir: + for marker in section.get("markers", []) or []: + try: + if (Path(path) / str(marker)).exists(): + return "forbidden:marker:{0}".format(marker) + except OSError: + continue + return None + + +def refuse_root( + root: Path, + *, + allow_outside_home: bool = False, + home: Optional[Path] = None, +) -> Path: + """Validate a scan root and return its realpath, or raise ``ValueError``.""" + + candidate = Path(root).expanduser() + if candidate.is_symlink(): + raise ValueError("refused root: {0} is a symlink".format(candidate)) + resolved = realpath(candidate) + if not resolved.is_dir(): + raise ValueError("refused root: {0} is not a directory".format(resolved)) + + home_dir = _home(home) + if str(resolved) in REFUSED_ABSOLUTE: + raise ValueError("refused root: {0} is a system directory".format(resolved)) + # The top of a filesystem, spelled whichever way this platform spells it: + # "/" on a Mac, "C:\\" or "D:\\" on Windows. The list above only knows the + # first one, and a drive letter is not something to enumerate. + if str(resolved) == resolved.anchor: + raise ValueError("refused root: {0} is a system directory".format(resolved)) + if resolved == home_dir: + raise ValueError("refused root: the home directory itself is out of scope") + for sub in REFUSED_HOME_SUBDIRS: + blocked = home_dir / sub + if resolved == blocked or inside_root(resolved, blocked): + raise ValueError("refused root: {0} is out of scope".format(blocked)) + if not allow_outside_home and not inside_root(resolved, home_dir): + raise ValueError( + "refused root: {0} is outside {1}; pass --allow-outside-home to override".format( + resolved, home_dir + ) + ) + return resolved + + +# --------------------------------------------------------------------------- +# the privacy rule: absolute paths never leave the managed folder +# --------------------------------------------------------------------------- + + +def plan_home(plan: Any, home: Optional[Path] = None) -> Path: + """Recover the home directory a plan was written against. + + ``source_root`` and ``source_root_portable`` are two spellings of the same + path, so subtracting one from the other gives back ``$HOME`` and nobody has + to store it anywhere. A stripped plan has lost the absolute half, and then + the answer is this machine's home, which is the only one apply can act on. + """ + + if home is not None: + return Path(home) + envelope = plan if isinstance(plan, dict) else {} + root = str(envelope.get("source_root") or "") + portable_text = str(envelope.get("source_root_portable") or "") + if not root: + return Path.home() + split = split_home_token(portable_text) + if split is None: + return Path.home() + _token, rest = split + if not rest: + return Path(root) + relative = Path(rest).parts + whole = Path(root).parts + if len(whole) > len(relative) and whole[-len(relative) :] == relative: + return Path(*whole[: -len(relative)]) + return Path.home() + + +def _base_spellings(base: str) -> Tuple[str, ...]: + """The home directory as a plan may spell it, both separators. + + Plans travel: one written on a Mac is read on Windows and the other way + round, so the strings inside a plan can use the other machine's separator + while the home directory recovered from it uses this one's. Both spellings + name the same directory, and a privacy rule that only rewrites one of them + is a privacy rule with a hole in it. + """ + + if not base: + return () + found = [] + for spelling in (base, base.replace("\\", "/"), base.replace("/", "\\")): + if spelling in ("/", "\\", os.sep) or spelling in found: + continue + found.append(spelling) + return tuple(found) + + +def _portable_text(text: str, base: str) -> str: + """Rewrite every mention of the home directory, in either separator style.""" + + spellings = _base_spellings(base) + if text in spellings: + return HOME_TOKEN + for spelling in spellings: + for separator in ("/", "\\"): + text = text.replace(spelling + separator, HOME_TOKEN + "/") + return text + + +def _scrub(value: Any, base: str) -> Any: + """Every remaining string rendered in ``$HOME`` form, no exemptions.""" + + if isinstance(value, dict): + return {k: _scrub(v, base) for k, v in value.items()} + if isinstance(value, list): + return [_scrub(item, base) for item in value] + if isinstance(value, str): + return _portable_text(value, base) + return value + + +def strip_absolute(plan: Any, home: Optional[Path] = None) -> Any: + """A copy of ``plan`` carrying no absolute path, only the ``_portable`` twins. + + This is what the review page embeds and what ``/api/plan`` serves, so a + report.html handed to somebody else never spells out the account it came + from. The four fields with a twin are dropped outright; anything that names + the home directory is rewritten rather than deleted, so a field added later + cannot quietly reopen the leak. The plan.json in the managed folder is not + touched, and :func:`hydrate_absolute` reverses this at apply time. + """ + + if not isinstance(plan, dict): + return plan + base = str(plan_home(plan, home)) + data = copy.deepcopy(plan) + for field in PLAN_ABSOLUTE_FIELDS: + data.pop(field, None) + actions = data.get("actions") + if isinstance(actions, list): + for action in actions: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + action.pop(field, None) + return _scrub(data, base) + + +def _rehydrate(holder: Dict[str, Any], field: str, home: Optional[Path]) -> None: + if holder.get(field): + return + twin = holder.get(field + "_portable") + if isinstance(twin, str) and twin: + holder[field] = str(expand_portable(twin, home)) + else: + holder[field] = holder.get(field) + + +def hydrate_absolute(plan: Any, home: Optional[Path] = None) -> Any: + """A copy of ``plan`` with the absolute paths rebuilt from their twins. + + Idempotent: a plan that still has them is returned unchanged. A plan + exported from the review page has only ``$HOME`` forms, and they are expanded + against this machine's home unless one is injected. + """ + + if not isinstance(plan, dict): + return plan + base = plan_home(plan, home) + data = copy.deepcopy(plan) + for field in PLAN_ABSOLUTE_FIELDS: + _rehydrate(data, field, base) + actions = data.get("actions") + if isinstance(actions, list): + for action in actions: + if isinstance(action, dict): + for field in ACTION_ABSOLUTE_FIELDS: + _rehydrate(action, field, base) + return data diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py new file mode 100644 index 0000000..a2e758f --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/planner.py @@ -0,0 +1,1444 @@ +"""The decision chain: scan, classify, group, guard, and write plan.json v2. + +Nothing in this module touches the filesystem beyond reading. It produces a +list of candidate actions and a list of grouped choices, and every one of them +starts out unapproved. The order of the chain is the whole design: + + pinned you already put it back once, so it stays + no-go zone shown, never touched + guard a process, a launch agent or a symlink still needs it + sensitive naming isolated without ever opening the file + your Finder tags optional, off by default + your name rules from the settings file + archive pairing a .zip and its extracted folder move together + top level folders whole, keeping their name + extension rules plus the settling period + unknown extension waits in Pending and can be rerouted + cold candidates large items and used-up installers get a disposal option + +Afterwards, duplicate and regenerable groups add their disposal options, and any +move whose destination already exists is redirected to a Duplicates Review +folder rather than overwriting anything. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import sys +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from . import __version__ +from . import classifier +from . import config as config_module +from . import grouping, guard, i18n, paths, trash +from .ids import action_id as _action_id +from .scanner import Entry, scan_top_level, split_suffix, user_tags + +SCHEMA_VERSION = 2 + +DEFAULT_LARGE_BYTES = 500 * 1024 * 1024 +DEFAULT_TARGET_PERCENT = 90.0 + +CONFIDENCE = { + "ext": 0.98, + "sensitive": 0.9, + "name": 0.9, + "tag": 0.85, + "pair": 0.8, + "dir": 0.6, + "unknown": 0.55, + "dup_sha256": 0.95, + "dup_name": 0.7, + "regenerable": 0.99, + "cold": 0.5, + "hold": 1.0, +} + +# --------------------------------------------------------------------------- +# the three colours +# --------------------------------------------------------------------------- + +GREEN = "green" +YELLOW = "yellow" +RED = "red" + +#: Written into ``action.color``, ``group.color`` and ``mess.color``. +COLORS = (GREEN, YELLOW, RED) + +#: Folder mess thresholds. ``references/tiers.md`` is the prose version of this +#: table, and an agent may override the resulting colour through notes.json; +#: it may never override an action's colour, because the buttons follow that. +MESS_THRESHOLDS = { + GREEN: {"loose": 3, "pending": 0}, + YELLOW: {"loose": 15, "pending": 5}, +} + +#: Weights behind ``mess.score``: how much of each thing counts as "full". The +#: score only sets the length of a bar; the colour comes from the thresholds +#: above, and neither is derived from the other. +MESS_WEIGHTS = { + "loose": 15.0, + "pending": 5.0, + "aging": 20.0, + "groups": 12.0, + "top_dirs": 20.0, +} + +#: Top level folders below this count do not add to the score at all. +MESS_TOP_DIRS_FREE = 5 + + +def action_color(action: Mapping[str, Any]) -> str: + """Green go ahead, yellow look first, red leave it alone. + + The buttons on the review page follow this one field, so it is derived from + the decision rather than written by hand: every hold is red; a move is green + unless it is one of the reroutable ones still waiting on a human; a disposal + is green only when the thing disposed of can be rebuilt from source. + """ + + if action.get("kind") == "hold": + return RED + if action.get("kind") == "move": + return YELLOW if action.get("reroutable") else GREEN + return GREEN if action.get("tier") == "regenerable" else YELLOW + + +def mess_counts( + entries: Sequence[Any], actions: Sequence[Mapping[str, Any]], groups: Sequence[Any] +) -> Dict[str, int]: + """The five numbers the folder colour is decided on.""" + + return { + "loose": len(entries), + "aging": sum( + 1 for a in actions if a.get("kind") == "hold" and a.get("tier") == "aging" + ), + "pending": len({a["subject_id"] for a in actions if a.get("reroutable")}), + "groups": len(groups), + "top_dirs": sum(1 for entry in entries if getattr(entry, "is_dir", False)), + } + + +def mess_score(counts: Mapping[str, int]) -> int: + """0 to 100, bigger is messier. Only a bar length, never a verdict. + + Two properties matter and a plain weighted sum has neither. The bar must + never sit at 100 on a folder that is merely untidy, or the one folder that + really is a landslide has nowhere left to go; and it must keep moving, so + that clearing thirty files out of two hundred visibly does something. So + the pressure is fed through ``p / (p + 1)``, which saturates instead of + clipping: it approaches 100 and never arrives. + + ``loose`` and ``pending`` are combined with max rather than added, because + they measure the same folder two ways and adding them counts one pile twice. + """ + + load = max( + counts.get("loose", 0) / MESS_WEIGHTS["loose"], + counts.get("pending", 0) / MESS_WEIGHTS["pending"], + ) + extra = ( + counts.get("aging", 0) / MESS_WEIGHTS["aging"] + + counts.get("groups", 0) / MESS_WEIGHTS["groups"] + + max(0, counts.get("top_dirs", 0) - MESS_TOP_DIRS_FREE) / MESS_WEIGHTS["top_dirs"] + ) + pressure = load + extra + return int(round(100 * pressure / (pressure + 1))) + + +def mess_color(counts: Mapping[str, int]) -> str: + """Green, yellow or red for the folder as a whole.""" + + loose = counts.get("loose", 0) + pending = counts.get("pending", 0) + if loose <= MESS_THRESHOLDS[GREEN]["loose"] and pending <= MESS_THRESHOLDS[GREEN]["pending"]: + return GREEN + if loose <= MESS_THRESHOLDS[YELLOW]["loose"] or pending <= MESS_THRESHOLDS[YELLOW]["pending"]: + return YELLOW + return RED + + +def build_mess( + entries: Sequence[Any], actions: Sequence[Mapping[str, Any]], groups: Sequence[Any] +) -> Dict[str, Any]: + """The ``mess`` block: how bad this folder is, and the one line that says why.""" + + counts = mess_counts(entries, actions, groups) + colour = mess_color(counts) + return { + "score": mess_score(counts), + "color": colour, + "color_source": "rule", + "counts": counts, + "reason": i18n.both("mess_" + colour, **counts), + } + + +def color_totals(actions: Sequence[Mapping[str, Any]]) -> Dict[str, Dict[str, int]]: + """``summary.by_color``: how many actions wear each colour, and how big they are. + + Counted per action, not per subject, exactly like ``by_tier`` and ``by_kind``: + one file can hold a green move and a yellow disposal at the same time, so the + bytes column adds up to more than the folder holds. It is a shape, not a total. + """ + + totals = {colour: {"actions": 0, "bytes": 0} for colour in COLORS} + for action in actions: + slot = totals.get(str(action.get("color"))) + if slot is None: + continue + slot["actions"] += 1 + slot["bytes"] += int(action.get("size_bytes") or 0) + return totals + + +@dataclass +class PlanPaths: + managed: Path + plan: Path + report: Path + + +def plan_paths(config: Any) -> PlanPaths: + managed = config_module.managed_dir(config) + return PlanPaths(managed=managed, plan=managed / "plan.json", report=managed / "report.html") + + +# --------------------------------------------------------------------------- +# small helpers +# --------------------------------------------------------------------------- + + +def action_id(subject: str, kind: str, destination_portable: Optional[str]) -> str: + """Thin re-export so no caller ever hand-rolls an id.""" + + return _action_id(subject, kind, destination_portable) + + +def _disk_status(root: Path, target_percent: float) -> Dict[str, Any]: + usage = shutil.disk_usage(str(root)) + percent = (usage.used / usage.total * 100) if usage.total else 0.0 + target_used = int(usage.total * target_percent / 100) + return { + "total_bytes": usage.total, + "used_bytes": usage.used, + "free_bytes": usage.free, + "used_percent": round(percent, 2), + "target_percent": target_percent, + "bytes_to_remove_for_target": max(0, usage.used - target_used), + "note": "Moves on this volume reclaim 0 bytes.", + } + + +def detect_trash_backend() -> str: + """Which trash backend this machine has, without importing the executor. + + One spelling only: :mod:`carl_file_organizer.trash` owns the answer, and a + platform added there shows up in the plan without a second edit here. + """ + + return trash.detect_backend() + + +def _finder_tags_available() -> bool: + return sys.platform == "darwin" and shutil.which("xattr") is not None + + +def _aware(moment: Optional[datetime]) -> datetime: + if moment is None: + return datetime.now().astimezone() + if moment.tzinfo is None: + return moment.astimezone() + return moment + + +def _format_bytes(value: int) -> str: + size = float(value) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024: + return "{0:.1f} {1}".format(size, unit) + size /= 1024 + return "{0:.1f} TB".format(size) + + +# --------------------------------------------------------------------------- +# planning context +# --------------------------------------------------------------------------- + + +@dataclass +class _PairRef: + group_id: str + display: str + stem: str + + +@dataclass +class PlanContext: + config: Any + root: Path + home: Path + now: datetime + large_bytes: int + offer_delete: bool + trash_backend: str + allow_referenced: bool + guard_context: Any + pairs: Dict[str, _PairRef] = field(default_factory=dict) + duplicates: Dict[str, Any] = field(default_factory=dict) + taken: set = field(default_factory=set) + + @property + def trash_available(self) -> bool: + return self.trash_backend != "none" + + +def _portable(ctx: PlanContext, path: Path) -> str: + return paths.portable(path, ctx.home) + + +def _dest_base(ctx: PlanContext, key: str) -> Path: + return config_module.dest_dir(ctx.config, key) + + +def _claim(ctx: PlanContext, entry: Entry, key: str, target: Path) -> Tuple[Path, str, bool]: + """Hand back a destination that does not exist yet; collisions get redirected. + + ``entry`` rather than its name, because a redirected file keeps its suffix + (``report-.pdf``) while a directory does not. + """ + + occupied = target.exists() or target.is_symlink() or str(target) in ctx.taken + if not occupied: + ctx.taken.add(str(target)) + return target, key, False + dup_key = "sensitive.dup" if str(key).split(".")[0] == "sensitive" else "inbox.dup" + try: + redirected = _dest_base(ctx, dup_key) / paths.conflict_name( + entry.name, ctx.now, is_dir=entry.is_dir + ) + except (KeyError, ValueError): + ctx.taken.add(str(target)) + return target, key, False + ctx.taken.add(str(redirected)) + return redirected, dup_key, True + + +# --------------------------------------------------------------------------- +# action construction +# --------------------------------------------------------------------------- + + +def make_action( + entry: Entry, + kind: str, + *, + ctx: PlanContext, + rule: str, + reason: Mapping[str, str], + tier: str, + confidence: float, + destination: Optional[Path] = None, + destination_key: Optional[str] = None, + detail: Optional[Mapping[str, str]] = None, + group_id: Optional[str] = None, + approvable: bool = True, + reroutable: bool = False, + requires: Optional[Sequence[str]] = None, + hints: Optional[Sequence[str]] = None, + hold_reason: Optional[Mapping[str, Any]] = None, + guard_block: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """One row of ``plan.actions``, with every frozen field present.""" + + source_portable = _portable(ctx, entry.path) + destination_portable = _portable(ctx, destination) if destination is not None else None + + if kind == "move" and destination is not None: + restore = i18n.both( + "restore_move", + dest='"{0}"'.format(destination_portable), + src='"{0}/"'.format(_portable(ctx, entry.path.parent)), + ) + elif kind == "trash": + restore = i18n.both("restore_trash") + elif kind == "delete": + restore = i18n.both("restore_delete") + else: + restore = None + + action = { + "id": _action_id(entry.subject_id, kind, destination_portable), + "subject_id": entry.subject_id, + "kind": kind, + "subject_kind": entry.kind, + "source": str(entry.path), + "source_portable": source_portable, + "filename": entry.name, + "destination": str(destination) if destination is not None else None, + "destination_portable": destination_portable, + "destination_key": destination_key, + "size_bytes": entry.size_bytes, + "modified_ns": entry.modified_ns, + "age_hours": entry.age_hours, + "rule": rule, + "reason": dict(reason), + "detail": dict(detail) if detail else None, + "confidence": confidence, + "tier": tier, + "group_id": group_id, + "approvable": bool(approvable), + "reroutable": bool(reroutable), + "default_selected": False, + "requires": list(requires or ()), + "reclaims_bytes": entry.size_bytes if kind in ("trash", "delete") else 0, + "restore_method": restore, + "tags": user_tags(entry.tags, config_module.transient_tags(ctx.config)), + "hints": list(hints or ()), + "hold_reason": dict(hold_reason) if hold_reason else None, + "guard": dict(guard_block) + if guard_block + else {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []}, + "dir_stats": entry.dir_stats.as_dict() if entry.dir_stats is not None else None, + "color": None, + } + # Derived last so it can never disagree with the fields above; build_plan + # recomputes it once more after the duplicate pass rewrites a few tiers. + action["color"] = action_color(action) + return action + + +def _hold( + entry: Entry, + ctx: PlanContext, + *, + rule: str, + tier: str, + reason: Mapping[str, str], + extra: Optional[Mapping[str, Any]] = None, + hints: Optional[Sequence[str]] = None, + guard_block: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + hold_reason = dict(reason) + if extra: + hold_reason.update(extra) + return make_action( + entry, + "hold", + ctx=ctx, + rule=rule, + reason=reason, + tier=tier, + confidence=CONFIDENCE["hold"], + approvable=False, + hold_reason=hold_reason, + hints=hints, + guard_block=guard_block, + ) + + +def disposal_alternatives( + entry: Entry, + ctx: PlanContext, + *, + rule: str, + reason: Mapping[str, str], + tier: str, + confidence: float, + group_id: Optional[str] = None, + guard_block: Optional[Mapping[str, Any]] = None, + allow_delete: bool = True, + detail: Optional[Mapping[str, str]] = None, +) -> List[Dict[str, Any]]: + """Trash and, when it is offered, permanent delete for the same subject.""" + + found: List[Dict[str, Any]] = [] + if ctx.trash_available: + found.append( + make_action( + entry, + "trash", + ctx=ctx, + rule=rule, + reason=reason, + detail=detail, + tier=tier, + confidence=confidence, + group_id=group_id, + guard_block=guard_block, + ) + ) + if ctx.offer_delete and allow_delete: + found.append( + make_action( + entry, + "delete", + ctx=ctx, + rule=rule, + reason=reason, + detail=detail, + tier=tier, + confidence=confidence, + group_id=group_id, + requires=["allow-permanent-delete"], + guard_block=guard_block, + ) + ) + return found + + +# --------------------------------------------------------------------------- +# the guard verdict +# --------------------------------------------------------------------------- + + +def _who(handles: Sequence[Mapping[str, Any]]) -> str: + return "、".join( + "{0}({1})".format(item.get("command") or "?", item.get("pid")) for item in handles + ) + + +def _where(references: Sequence[Mapping[str, Any]], links: Sequence[str]) -> str: + parts = ["{0}:{1}".format(item.get("file"), item.get("line")) for item in references] + parts.extend(str(link) for link in links) + return "、".join(parts) + + +def guard_verdict( + entry: Entry, block: Mapping[str, Any], ctx: PlanContext +) -> Optional[Tuple[str, str, Dict[str, str]]]: + """``(rule, tier, reason)`` when the safety net says leave this alone.""" + + handles = block.get("open_by") or [] + if handles: + return "hold:in_use", "in_use", i18n.both("hold_in_use", who=_who(handles)) + + if "git-worktree" in (block.get("shape") or []): + return "hold:referenced", "referenced", i18n.both("hold_worktree") + + references = block.get("referenced_in") or [] + links = block.get("incoming_symlinks") or [] + if (references or links) and not ctx.allow_referenced: + return ( + "hold:referenced", + "referenced", + i18n.both("hold_referenced", where=_where(references, links)), + ) + + if guard.recently_modified(entry, ctx.now, ctx.guard_context.recent_minutes): + return "hold:aging", "aging", i18n.both("hold_recent") + return None + + +def _guard_hints( + block: Mapping[str, Any], ctx: PlanContext +) -> Tuple[List[str], Optional[Dict[str, str]]]: + """Observations that never change the verdict, plus the downgrade warning.""" + + hints: List[str] = [] + shape = block.get("shape") or [] + for label, hint in ( + ("git-dir", "git-repo"), + ("venv", "venv"), + ("dotenv", "dotenv"), + ("node-project", "node-project"), + ): + if label in shape: + hints.append(hint) + detail = None + references = block.get("referenced_in") or [] + links = block.get("incoming_symlinks") or [] + if (references or links) and ctx.allow_referenced: + hints.append("referenced-warning") + detail = i18n.both("detail_referenced_warning", where=_where(references, links)) + return hints, detail + + +# --------------------------------------------------------------------------- +# the chain +# --------------------------------------------------------------------------- + + +def decide_entry(entry: Entry, ctx: PlanContext) -> List[Dict[str, Any]]: + """Every candidate action for one scanned item, in priority order.""" + + config = ctx.config + + # 1. pinned ------------------------------------------------------------- + pinned = classifier.match_pinned(entry.name, config) + if pinned: + return [ + _hold( + entry, + ctx, + rule="hold:pinned", + tier="pinned", + reason=i18n.both("hold_pinned"), + extra={"pattern": pinned}, + ) + ] + + # 2. no-go zone --------------------------------------------------------- + label = paths.is_forbidden(entry.path, config) + if label: + return [ + _hold( + entry, + ctx, + rule="hold:forbidden", + tier="forbidden", + reason=i18n.both("hold_forbidden", label=label), + extra={"label": label}, + ) + ] + + # 3. the safety net ----------------------------------------------------- + block = guard.inspect(entry.path, config, ctx.guard_context, is_dir=entry.is_dir) + verdict = guard_verdict(entry, block, ctx) + if verdict is not None: + rule, tier, reason = verdict + return [_hold(entry, ctx, rule=rule, tier=tier, reason=reason, guard_block=block)] + hints, guard_detail = _guard_hints(block, ctx) + + # 4. sensitive naming --------------------------------------------------- + hit = classifier.is_sensitive(entry.name, config) + if hit is not None: + target, key, conflicted = _claim( + ctx, + entry, + "sensitive.pending", + _dest_base(ctx, "sensitive.pending") / entry.name, + ) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "sensitive:{0}".format(hit.layer), + reason=i18n.both("conflict") if conflicted else i18n.both("sensitive_isolated"), + detail=guard_detail or i18n.both("detail_sensitive", needle=hit.needle), + tier="sensitive", + confidence=CONFIDENCE["sensitive"], + hints=hints, + guard_block=block, + ) + ] + + # 5. your own Finder tags ---------------------------------------------- + profile = getattr(config, "profile", config) + group_by_tag = bool((profile.get("tags") or {}).get("group_by_user_tags")) + mine = user_tags(entry.tags, config_module.transient_tags(config)) + if group_by_tag and mine: + tag = sorted(mine)[0] + base = _dest_base(ctx, "inbox.by_tag") / paths.safe_dir_name(tag) + target, key, conflicted = _claim(ctx, entry, "inbox.by_tag", base / entry.name) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "tag:{0}".format(tag), + reason=i18n.both("conflict") if conflicted else i18n.both("tag_group", tag=tag), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["tag"], + hints=hints, + guard_block=block, + ) + ] + + # 6. your own name rules ------------------------------------------------ + pattern = classifier.match_name_pattern(entry.name, config) + if pattern is not None: + if pattern.dest_key: + base = _dest_base(ctx, pattern.dest_key) + key = pattern.dest_key + keyed = True + else: + base = ctx.root / str(pattern.dest) + key = "inbox.pending" + keyed = False + target, key, conflicted = _claim(ctx, entry, key, base / entry.name) + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key if (keyed or conflicted) else None, + rule="conflict" if conflicted else "name:{0}".format(pattern.id), + reason=i18n.both("conflict") + if conflicted + else i18n.both("name_pattern", pattern=pattern.id), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["name"], + hints=hints, + guard_block=block, + ) + ] + + # 7. archive and extracted folder -------------------------------------- + pair = ctx.pairs.get(entry.subject_id) + if pair is not None: + folder = paths.safe_dir_name(split_suffix(pair.display)[0]) + base = _dest_base(ctx, "inbox.pending") / folder + target, key, conflicted = _claim(ctx, entry, "inbox.pending", base / entry.name) + reason = i18n.both("pair") + found = [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "pair", + reason=i18n.both("conflict") if conflicted else reason, + detail=guard_detail, + tier="duplicate", + confidence=CONFIDENCE["pair"], + group_id=pair.group_id, + hints=hints, + guard_block=block, + ) + ] + found.extend( + disposal_alternatives( + entry, + ctx, + rule="pair", + reason=reason, + tier="duplicate", + confidence=CONFIDENCE["pair"], + group_id=pair.group_id, + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + return found + + in_duplicate_group = entry.subject_id in ctx.duplicates + + # 8. top level folders -------------------------------------------------- + if entry.is_dir: + hint = classifier.derivative_hint(entry.name, config) + if hint: + hints = list(hints) + ["derivative-copy:{0}".format(hint)] + target, key, conflicted = _claim( + ctx, entry, "inbox.pending", _dest_base(ctx, "inbox.pending") / entry.name + ) + rule = "dir:derivative" if hint else "dir:pending" + return [ + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else rule, + reason=i18n.both("conflict") + if conflicted + else (i18n.both("dir_derivative", hint=hint) if hint else i18n.both("dir_pending")), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["dir"], + reroutable=not conflicted, + hints=hints, + guard_block=block, + ) + ] + + # 9. extension rules and the settling period --------------------------- + rule_hit = classifier.match_rule(entry.suffix, config) + found = [] + if rule_hit is not None and rule_hit.dest_key: + if entry.age_hours < rule_hit.min_age_hours: + ready = datetime.fromtimestamp( + entry.modified_ns / 1_000_000_000, tz=ctx.now.tzinfo + ) + timedelta(hours=rule_hit.min_age_hours) + return [ + _hold( + entry, + ctx, + rule="hold:aging", + tier="aging", + reason=i18n.both( + "hold_aging", age=entry.age_hours, needed=rule_hit.min_age_hours + ), + extra={"ready_at": ready.isoformat()}, + hints=hints, + guard_block=block, + ) + ] + dest_name = config_module.dir_name(config, rule_hit.dest_key) + target, key, conflicted = _claim( + ctx, entry, rule_hit.dest_key, _dest_base(ctx, rule_hit.dest_key) / entry.name + ) + detail = guard_detail + if detail is None and rule_hit.min_age_hours > 0: + detail = i18n.both("detail_aged", age=entry.age_hours, needed=rule_hit.min_age_hours) + found.append( + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "ext:{0}".format(entry.suffix), + reason=i18n.both("conflict") + if conflicted + else i18n.both(rule_hit.reason_key, dest=dest_name, hours=rule_hit.min_age_hours), + detail=detail, + tier="routine", + confidence=CONFIDENCE["ext"], + hints=hints, + guard_block=block, + ) + ) + else: + # 10. unknown extension --------------------------------------------- + target, key, conflicted = _claim( + ctx, entry, "inbox.pending", _dest_base(ctx, "inbox.pending") / entry.name + ) + found.append( + make_action( + entry, + "move", + ctx=ctx, + destination=target, + destination_key=key, + rule="conflict" if conflicted else "unknown-ext", + reason=i18n.both("conflict") if conflicted else i18n.both("unknown_ext"), + detail=guard_detail, + tier="routine", + confidence=CONFIDENCE["unknown"], + reroutable=not conflicted, + hints=hints, + guard_block=block, + ) + ) + + # 11. cold storage candidates ------------------------------------------- + if not in_duplicate_group: + if entry.size_bytes >= ctx.large_bytes: + found.extend( + disposal_alternatives( + entry, + ctx, + rule="cold:large", + reason=i18n.both("cold_large", size=_format_bytes(entry.size_bytes)), + tier="cold", + confidence=CONFIDENCE["cold"], + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + elif rule_hit is not None and ( + classifier.is_installer_suffix(entry.suffix, config) + or classifier.is_archive_suffix(entry.suffix, config) + ): + found.extend( + disposal_alternatives( + entry, + ctx, + rule="cold:installer", + reason=i18n.both("cold_installer"), + tier="cold", + confidence=CONFIDENCE["cold"], + guard_block=block, + allow_delete=not entry.is_dir, + ) + ) + return found + + +# --------------------------------------------------------------------------- +# groups +# --------------------------------------------------------------------------- + + +def _group_id(prefix: str, material: str) -> str: + return "{0}-{1}".format(prefix, hashlib.sha256(material.encode("utf-8")).hexdigest()[:8]) + + +def _member_label(entry: Entry) -> str: + return entry.name + "/" if entry.is_dir else entry.name + + +def _keep_options( + members: Sequence[Entry], + index: Mapping[Tuple[str, str], str], + *, + label_of, +) -> List[Dict[str, Any]]: + options: List[Dict[str, Any]] = [] + for keep in members: + ids = [index.get((keep.subject_id, "move"))] + dropped = [m for m in members if m.subject_id != keep.subject_id] + ids += [index.get((m.subject_id, "trash")) for m in dropped] + if not all(ids): + continue + options.append( + { + "key": "keep:{0}".format(keep.subject_id), + "label": i18n.ui_both( + "option_keep_one", + keep=label_of(keep), + drop="、".join(label_of(m) for m in dropped), + ), + "action_ids": ids, + "requires": [], + } + ) + return options + + +def _build_pair_group( + raw: Any, group_id: str, index: Mapping[Tuple[str, str], str], ctx: PlanContext +) -> Optional[Dict[str, Any]]: + members = list(raw.members) + options: List[Dict[str, Any]] = [] + + move_ids = [index.get((m.subject_id, "move")) for m in members] + if all(move_ids): + options.append( + { + "key": "keep_all", + "label": i18n.ui_both("option_keep_all"), + "action_ids": list(move_ids), + "requires": [], + } + ) + options.extend(_keep_options(members, index, label_of=_member_label)) + + default_member = next((m for m in members if m.is_dir), members[0]) + dropped = [m for m in members if m.subject_id != default_member.subject_id] + delete_ids = [index.get((default_member.subject_id, "move"))] + [ + index.get((m.subject_id, "delete")) for m in dropped + ] + if ctx.offer_delete and dropped and all(delete_ids): + options.append( + { + "key": "keep:{0}:delete".format(default_member.subject_id), + "label": i18n.ui_both( + "option_keep_one_delete", + keep=_member_label(default_member), + drop="、".join(_member_label(m) for m in dropped), + ), + "action_ids": delete_ids, + "requires": ["allow-permanent-delete"], + } + ) + + if not options: + return None + keys = [option["key"] for option in options] + preferred = "keep:{0}".format(default_member.subject_id) + default_choice = preferred if preferred in keys else keys[0] + return { + "group_id": group_id, + "kind": "pair", + "tier": "duplicate", + "stem": raw.stem, + "evidence": raw.evidence, + "sha256": raw.sha256, + "members": [m.subject_id for m in members], + "member_labels": {m.subject_id: _member_label(m) for m in members}, + "reason": i18n.both("pair"), + "options": options, + "default_choice": default_choice, + "potential_bytes": sum(m.size_bytes for m in dropped) + if default_choice == preferred + else 0, + } + + +def _build_duplicate_group( + raw: Any, group_id: str, index: Mapping[Tuple[str, str], str], ctx: PlanContext +) -> Optional[Dict[str, Any]]: + members = list(raw.members) + verified = raw.evidence == "sha256" + reason = i18n.both("dup_sha256") if verified else i18n.both("dup_name") + options: List[Dict[str, Any]] = [] + + move_ids = [index.get((m.subject_id, "move")) for m in members] + if all(move_ids): + options.append( + { + "key": "keep_all", + "label": i18n.ui_both("option_keep_all_plain"), + "action_ids": list(move_ids), + "requires": [], + } + ) + options.extend(_keep_options(members, index, label_of=lambda item: item.name)) + + if verified and ctx.offer_delete: + keep = members[0] + dropped = [m for m in members if m.subject_id != keep.subject_id] + delete_ids = [index.get((keep.subject_id, "move"))] + [ + index.get((m.subject_id, "delete")) for m in dropped + ] + if dropped and all(delete_ids): + options.append( + { + "key": "keep:{0}:delete".format(keep.subject_id), + "label": i18n.ui_both( + "option_keep_one_delete", + keep=keep.name, + drop="、".join(m.name for m in dropped), + ), + "action_ids": delete_ids, + "requires": ["allow-permanent-delete"], + } + ) + + if not options: + return None + keys = [option["key"] for option in options] + if verified: + preferred = "keep:{0}".format(members[0].subject_id) + default_choice = preferred if preferred in keys else keys[0] + potential = sum(m.size_bytes for m in members[1:]) if default_choice == preferred else 0 + else: + default_choice = "keep_all" if "keep_all" in keys else keys[0] + potential = 0 + return { + "group_id": group_id, + "kind": "duplicate", + "tier": "duplicate", + "stem": raw.stem, + "evidence": raw.evidence, + "sha256": raw.sha256, + "members": [m.subject_id for m in members], + "member_labels": {m.subject_id: m.name for m in members}, + "reason": reason, + "options": options, + "default_choice": default_choice, + "potential_bytes": potential, + } + + +# --------------------------------------------------------------------------- +# build_plan +# --------------------------------------------------------------------------- + + +def build_plan( + source_root: Path, + *, + config: Any = None, + now: Optional[datetime] = None, + hash_duplicates: bool = False, + large_bytes: int = DEFAULT_LARGE_BYTES, + target_percent: float = DEFAULT_TARGET_PERCENT, + dir_sizes: bool = True, + offer_permanent_delete: bool = True, + allow_referenced: bool = False, + home: Optional[Path] = None, + guard_options: Optional[Mapping[str, Any]] = None, + trash_backend: Optional[str] = None, +) -> Dict[str, Any]: + """Scan ``source_root`` read-only and return a schema v2 plan.""" + + requested = Path(source_root).expanduser() + cfg = config if config is not None else config_module.resolve_config(requested) + # The config owns the root: destinations are built from it, so anything else + # here would make every destination fall outside the root it is compared to. + root = Path(cfg.root) + if not root.is_dir(): + raise ValueError("Source is not a directory: {0}".format(root)) + moment = _aware(now) + home_dir = Path(home) if home is not None else Path.home() + backend = trash_backend if trash_backend is not None else detect_trash_backend() + + options = dict(guard_options or {}) + guard_context = guard.build_context( + cfg, + home=home_dir, + enabled=bool(options.get("enabled", True)), + reference_files=options.get("reference_files"), + scan_roots=options.get("scan_roots"), + lsof_runner=options.get("lsof_runner"), + lsof_enabled=bool(options.get("lsof_enabled", True)), + crontab_runner=options.get("crontab_runner"), + git_status_runner=options.get("git_status_runner"), + ) + + entries, notes = scan_top_level(root, cfg, now=moment, dir_sizes=dir_sizes) + guard_context.prime( + [entry.path for entry in entries if not entry.is_dir], + [entry.path for entry in entries if entry.is_dir], + ) + + profile = cfg.profile + dedupe = profile.get("dedupe") or {} + hash_max_bytes = int(dedupe.get("hash_max_bytes", grouping.DEFAULT_HASH_MAX_BYTES)) + regen_depth = int((profile.get("regenerable") or {}).get("scan_depth", 3)) + + ctx = PlanContext( + config=cfg, + root=root, + home=home_dir, + now=moment, + large_bytes=int(large_bytes), + offer_delete=bool(offer_permanent_delete), + trash_backend=backend, + allow_referenced=bool(allow_referenced), + guard_context=guard_context, + ) + + raw_pairs = grouping.pair_archives(entries, cfg) + paired_subjects = set() + for raw in raw_pairs: + raw.group_id = _group_id("pair", raw.stem) + for member in raw.members: + paired_subjects.add(member.subject_id) + ctx.pairs[member.subject_id] = _PairRef(raw.group_id, raw.display, raw.stem) + + raw_dups = grouping.find_duplicates( + entries, + hash_duplicates=hash_duplicates, + hash_max_bytes=hash_max_bytes, + exclude=paired_subjects, + ) + for raw in raw_dups: + raw.group_id = _group_id("dup", (raw.sha256 or raw.stem) + raw.evidence) + for member in raw.members: + ctx.duplicates[member.subject_id] = raw + + actions: List[Dict[str, Any]] = [] + for entry in entries: + actions.extend(decide_entry(entry, ctx)) + + by_subject: Dict[str, List[Dict[str, Any]]] = {} + for action in actions: + by_subject.setdefault(action["subject_id"], []).append(action) + + # duplicate groups relabel the primary move and add disposal options + for raw in raw_dups: + verified = raw.evidence == "sha256" + reason = i18n.both("dup_sha256") if verified else i18n.both("dup_name") + rule = "dup:sha256" if verified else "dup:name" + confidence = CONFIDENCE["dup_sha256"] if verified else CONFIDENCE["dup_name"] + for member in raw.members: + primary = [ + item for item in by_subject.get(member.subject_id, []) if item["kind"] == "move" + ] + if not primary: + continue + head = primary[0] + head["group_id"] = raw.group_id + if head["rule"] != "conflict": + head["rule"] = rule + head["reason"] = dict(reason) + head["confidence"] = confidence + head["tier"] = "duplicate" + head["reroutable"] = False + actions.extend( + disposal_alternatives( + member, + ctx, + rule=rule, + reason=reason, + tier="duplicate", + confidence=confidence, + group_id=raw.group_id, + guard_block=head["guard"], + allow_delete=verified and not member.is_dir, + detail=None if verified else i18n.both("detail_dup_unverified"), + ) + ) + + # regenerable build output inside top level folders + regen_groups: List[Dict[str, Any]] = [] + for hit in grouping.find_regenerable(entries, cfg, max_depth=regen_depth, now=moment): + subject = Entry( + path=hit.path, + name=hit.name, + kind="dir", + size_bytes=hit.dir_stats.bytes, + modified_ns=hit.modified_ns, + age_hours=hit.age_hours, + suffix="", + tags=[], + dir_stats=hit.dir_stats, + subject_id=hit.subject_id, + ) + group_id = _group_id("regen", hit.relative) + restore = classifier.regenerable_restore(hit.name, cfg) + reason = { + "zh": i18n.reason("regenerable", "zh", restore=restore.get("zh", "")), + "en": i18n.reason("regenerable", "en", restore=restore.get("en", "")), + } + made = disposal_alternatives( + subject, + ctx, + rule="regenerable:{0}".format(hit.name), + reason=reason, + tier="regenerable", + confidence=CONFIDENCE["regenerable"], + group_id=group_id, + ) + if not made: + continue + actions.extend(made) + by_kind = {item["kind"]: item["id"] for item in made} + options = [] + if "trash" in by_kind: + options.append( + { + "key": "trash_all", + "label": i18n.ui_both("option_trash_all"), + "action_ids": [by_kind["trash"]], + "requires": [], + } + ) + if "delete" in by_kind: + options.append( + { + "key": "delete_all", + "label": i18n.ui_both("option_delete_all"), + "action_ids": [by_kind["delete"]], + "requires": ["allow-permanent-delete"], + } + ) + regen_groups.append( + { + "group_id": group_id, + "kind": "regenerable", + "tier": "regenerable", + "stem": hit.name, + "evidence": "basename", + "sha256": None, + "members": [hit.subject_id], + "member_labels": {hit.subject_id: hit.relative}, + "reason": reason, + "options": options, + "default_choice": options[0]["key"], + "potential_bytes": hit.dir_stats.bytes, + } + ) + + index: Dict[Tuple[str, str], str] = {} + for action in actions: + index[(action["subject_id"], action["kind"])] = action["id"] + + groups: List[Dict[str, Any]] = [] + for raw in raw_pairs: + built = _build_pair_group(raw, raw.group_id, index, ctx) + if built is not None: + groups.append(built) + for raw in raw_dups: + built = _build_duplicate_group(raw, raw.group_id, index, ctx) + if built is not None: + groups.append(built) + groups.extend(regen_groups) + + known_groups = {group["group_id"] for group in groups} + for action in actions: + if action["group_id"] is not None and action["group_id"] not in known_groups: + action["group_id"] = None + + # The duplicate pass rewrites a few tiers and clears a reroutable flag, so + # every colour is settled here, once, on the finished list. + for action in actions: + action["color"] = action_color(action) + # A group is a question, and most questions are yellow: whichever option you + # pick, something gets disposed of that a rule alone would not touch. The + # regenerable group is the exception, and it follows its own members: build + # output rebuilds itself from source, so both of its options are green and + # colouring the question yellow would only ask for a look nobody needs. + for group in groups: + group["color"] = GREEN if group.get("kind") == "regenerable" else YELLOW + + names: Dict[str, str] = {} + for key in profile.get("names") or {}: + try: + names[key] = config_module.dir_name(cfg, key) + except (KeyError, ValueError): + continue + + by_tier: Dict[str, int] = {} + by_kind_count: Dict[str, int] = {} + by_destination: Dict[str, int] = {} + reclaimable: Dict[str, int] = {} + for action in actions: + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + by_kind_count[action["kind"]] = by_kind_count.get(action["kind"], 0) + 1 + if action["kind"] == "move" and action["destination"]: + try: + relative = Path(action["destination"]).parent.relative_to(root).as_posix() + except ValueError: + relative = str(Path(action["destination"]).parent) + by_destination[relative] = by_destination.get(relative, 0) + 1 + if action["kind"] in ("trash", "delete"): + reclaimable[action["subject_id"]] = action["size_bytes"] + + managed = config_module.managed_dir(cfg) + return { + "schema_version": SCHEMA_VERSION, + "generator": {"name": "carl-file-organizer", "version": __version__}, + # created_at follows the plan's clock, so --now gives a byte-identical + # plan twice in a row; generated_at keeps the real one for support. + "created_at": moment.isoformat(), + "generated_at": datetime.now().astimezone().isoformat(), + "now": moment.isoformat(), + "lang": cfg.lang, + "lang_source": cfg.lang_source, + "profile": { + "name": cfg.profile_name, + "source": cfg.profile_source, + "dotfile": paths.portable(cfg.dotfile_path, home_dir) if cfg.dotfile_path else None, + "hash": cfg.profile_hash, + }, + "source_root": str(root), + "source_root_portable": paths.portable(root, home_dir), + "managed_dir": str(managed), + "managed_dir_portable": paths.portable(managed, home_dir), + "names": names, + "capabilities": { + "trash_backend": backend, + "finder_tags": _finder_tags_available(), + "permanent_delete_offered": bool(offer_permanent_delete) + and any(action["kind"] == "delete" for action in actions), + }, + "scan_policy": { + "depth": 1, + "hidden_files": "ignored", + "symlinks": "ignored", + "managed_partitions": "skipped", + "regenerable_scan_depth": regen_depth, + "dir_size_limit": {"max_entries": 50000, "time_budget_s": 1.5} + if dir_sizes + else {"max_entries": 0, "time_budget_s": 0}, + "hash_duplicates": bool(hash_duplicates), + "hash_max_bytes": hash_max_bytes, + "allow_referenced": bool(allow_referenced), + "skipped": { + "hidden": notes.skipped_hidden, + "symlinks": notes.skipped_symlinks, + "partitions": notes.skipped_partitions, + "artifacts": notes.skipped_artifacts, + }, + }, + "disk": _disk_status(root, target_percent), + "mess": build_mess(entries, actions, groups), + "summary": { + "entries": len(entries), + "files": sum(1 for entry in entries if not entry.is_dir), + "dirs": sum(1 for entry in entries if entry.is_dir), + "bytes": sum(entry.size_bytes for entry in entries), + "by_tier": dict(sorted(by_tier.items())), + "by_kind": dict(sorted(by_kind_count.items())), + "by_color": color_totals(actions), + "by_destination": dict(sorted(by_destination.items())), + "groups": { + "pair": sum(1 for group in groups if group["kind"] == "pair"), + "duplicate": sum(1 for group in groups if group["kind"] == "duplicate"), + "regenerable": sum(1 for group in groups if group["kind"] == "regenerable"), + }, + "potential_reclaimable_bytes": sum(reclaimable.values()), + "proposed_reclaimed_bytes": 0, + }, + "pinned": [str(item) for item in (profile.get("pinned") or ())], + "actions": actions, + "groups": groups, + # Prose written by an agent on top of the rules, filled in by + # ``carl_file_organizer.notes``. None means nobody wrote any, and the + # report falls back to the rule text. + "notes": None, + "approved_action_ids": [], + "overrides": [], + "approved_at": None, + "approved_by": None, + } + + +def write_plan(plan: Dict[str, Any], output: Path) -> None: + Path(output).parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text( + json.dumps(plan, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def run(args: Any) -> int: + """``carl-file-organizer plan``: a read-only scan, then a plan and a review page.""" + + cfg = getattr(args, "config", None) + if cfg is None: + cfg = config_module.resolve_config( + Path(args.source), + lang=getattr(args, "lang", None), + profile=getattr(args, "profile", None), + ) + + large_mb = int(getattr(args, "large_mb", 500)) + target_percent = float(getattr(args, "target_percent", DEFAULT_TARGET_PERCENT)) + if large_mb < 1: + raise ValueError("--large-mb must be at least 1") + if not 1 <= target_percent <= 99: + raise ValueError("--target-percent must be between 1 and 99") + + where = plan_paths(cfg) + where.managed.mkdir(parents=True, exist_ok=True) + + plan = build_plan( + cfg.root, + config=cfg, + now=getattr(args, "now", None), + hash_duplicates=bool(getattr(args, "hash_duplicates", False)), + large_bytes=large_mb * 1024 * 1024, + target_percent=target_percent, + dir_sizes=bool(getattr(args, "dir_sizes", True)), + offer_permanent_delete=bool(getattr(args, "permanent_delete_options", True)), + allow_referenced=bool(getattr(args, "allow_referenced", False)), + ) + write_plan(plan, where.plan) + print(i18n.t("cli_plan_written", cfg.lang, path=where.plan)) + + # ``scan`` switches this off: it renders one combined report a step later, + # and a tidy-up-only page next to the plan would just be a second thing to + # open and a second thing to keep in step. + if bool(getattr(args, "write_report", True)): + try: + from . import render + + where.report.write_text( + render.render(plan, mode="static", lang=cfg.lang), encoding="utf-8" + ) + except Exception as error: # noqa: BLE001 - the plan itself is the deliverable + print( + i18n.t("cli_report_skipped", cfg.lang, module="build_report.py") + + " ({0})".format(error) + ) + else: + print(i18n.t("cli_report_written", cfg.lang, path=where.report)) + + approvable = sum(1 for action in plan["actions"] if action["approvable"]) + print( + i18n.t( + "cli_scan_summary", + cfg.lang, + entries=plan["summary"]["entries"], + actions=len(plan["actions"]), + approvable=approvable, + ) + ) + print(i18n.t("cli_nothing_moved", cfg.lang)) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py new file mode 100644 index 0000000..54e6e3c --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/preview.py @@ -0,0 +1,1404 @@ +"""The after picture: one knowledge-graph canvas of where everything lands. + +The report ends with a question a table cannot answer well: if I say yes, what +does the folder look like tomorrow? This module answers it as a picture. + +``build_graph`` turns plan.json (optionally plus a storage analysis) into a tiny +graph: a root, five partitions, the second level of folders under them, one node +per subject, and — when a storage analysis is supplied — a second root for the +machine with the cleanup candidates hanging off it. ``render_preview_html`` +turns that graph into a self-contained ``
    ``: canvas, style and script in +one string, no external resource of any kind, safe to paste at the end of the +report or to open on its own from ``file://``. + +Two layouts are computed for every node. In *now* the files are piled evenly +around the root because that is exactly where they are; in *after* they sit +under the folder they are headed for. The switch at the top interpolates +between the two, which is the whole point of the picture. + +The module is deliberately dependency-free (stdlib only, no intra-package +imports) so it can be tested and rendered in isolation. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from typing import Any, Dict, List, Mapping, Optional, Sequence + +__all__ = [ + "MAX_NODES", + "PARTITION_ORDER", + "PREVIEW_TEXT", + "build_graph", + "render_preview_html", + "render_preview_page", + "main", +] + +#: Hard ceiling on nodes. Past this the picture stops being a picture. +MAX_NODES = 400 + +#: The five partitions, in the order they should ring the root. +PARTITION_ORDER = ("inbox", "work", "library", "sensitive", "archive") + +#: Only these two kinds place a subject on the map, in this order of preference. +_KIND_PRECEDENCE = ("move", "hold") + +#: Kinds that free up space rather than move it. +_RECLAIM_KINDS = ("trash", "delete") + +_HOME_RE = re.compile(r"(?:/Users|/home)/[^/\s\"']+") + +# -------------------------------------------------------------------------- +# copy +# -------------------------------------------------------------------------- + +#: Every string the fragment can show, in both languages. Keys are added, never +#: renamed; the zh column avoids straight double quotes. +PREVIEW_TEXT: Dict[str, Dict[str, str]] = { + "zh": { + "title": "整理后长什么样", + "state_now": "现在", + "state_after": "整理后", + "stat_moving": "项要搬", + "stat_staying": "项留下", + "stat_reclaim": "可腾出", + "legend_green": "绿 · 照做就行", + "legend_yellow": "黄 · 看一眼", + "legend_red": "红 · 先别动", + "shape_partition": "分区", + "shape_folder": "目录", + "shape_file": "文件", + "shape_storage": "待清", + "hint": "拖拽节点 · 滚轮缩放 · 点开关看整理后", + "tip_size": "体积", + "tip_from": "现在", + "tip_to": "去向", + "tip_stay": "留在原地", + "tip_clean": "清掉能腾出空间", + "count_unit": "项", + "empty": "这一轮没有要搬的东西", + "machine": "本机", + "rest": "其余 {k} 项", + }, + "en": { + "title": "What it looks like afterwards", + "state_now": "Now", + "state_after": "After", + "stat_moving": "moving", + "stat_staying": "staying", + "stat_reclaim": "frees up", + "legend_green": "Green · just do it", + "legend_yellow": "Yellow · take a look", + "legend_red": "Red · leave it alone", + "shape_partition": "Partition", + "shape_folder": "Folder", + "shape_file": "File", + "shape_storage": "To clear", + "hint": "Drag nodes · scroll to zoom · flip the switch", + "tip_size": "Size", + "tip_from": "Now", + "tip_to": "Goes to", + "tip_stay": "Stays where it is", + "tip_clean": "Clearing this frees space", + "count_unit": "items", + "empty": "Nothing to move this round", + "machine": "This Mac", + "rest": "{k} more items", + }, +} + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def _scrub(value: Any) -> Any: + """Replace any real home directory with ``~`` before it reaches the page.""" + + if isinstance(value, str): + return _HOME_RE.sub("~", value) + if isinstance(value, dict): + return {key: _scrub(item) for key, item in value.items()} + if isinstance(value, list): + return [_scrub(item) for item in value] + return value + + +def _pick_lang(value: Any, lang: str) -> Optional[str]: + if isinstance(value, Mapping): + text = value.get(lang) or value.get("zh") or value.get("en") + return _scrub(text) if isinstance(text, str) else None + if isinstance(value, str): + return _scrub(value) + return None + + +def _relative_destination(action: Mapping[str, Any], roots: Sequence[str]) -> Optional[str]: + """The action's destination as a path relative to the source root.""" + + for key, root in (("destination_portable", roots[0]), ("destination", roots[1])): + dest = action.get(key) + if not isinstance(dest, str) or not dest: + continue + if root and dest.startswith(root.rstrip("/") + "/"): + return dest[len(root.rstrip("/")) + 1 :] + dest = action.get("destination_portable") or action.get("destination") + if isinstance(dest, str) and dest: + # Unanchored destination: keep the tail so the label still reads. + return dest.lstrip("/") + return None + + +def _partition_names(plan: Mapping[str, Any]) -> List[Dict[str, str]]: + names = plan.get("names") or {} + out: List[Dict[str, str]] = [] + seen = set() + for key in PARTITION_ORDER: + label = names.get(key) + if isinstance(label, str) and label and label not in seen: + seen.add(label) + out.append({"key": key, "label": label}) + for key, label in sorted(names.items()): + if "." in key or key in PARTITION_ORDER: + continue + if isinstance(label, str) and label and label not in seen: + seen.add(label) + out.append({"key": key, "label": label}) + return out + + +def _subject_actions(plan: Mapping[str, Any]) -> List[Mapping[str, Any]]: + """One action per subject: a move wins, otherwise a hold, otherwise nothing.""" + + chosen: Dict[str, Mapping[str, Any]] = {} + for action in plan.get("actions") or []: + if not isinstance(action, Mapping): + continue + kind = action.get("kind") + if kind not in _KIND_PRECEDENCE: + continue + subject = action.get("subject_id") or action.get("id") + if not subject: + continue + current = chosen.get(subject) + if current is None: + chosen[subject] = action + continue + if _KIND_PRECEDENCE.index(kind) < _KIND_PRECEDENCE.index(current.get("kind")): + chosen[subject] = action + ordered: List[Mapping[str, Any]] = [] + seen = set() + for action in plan.get("actions") or []: + if not isinstance(action, Mapping): + continue + subject = action.get("subject_id") or action.get("id") + if subject in chosen and subject not in seen and chosen[subject] is action: + seen.add(subject) + ordered.append(action) + return ordered + + +def _reclaim_bytes(plan: Mapping[str, Any]) -> int: + """Bytes the trash and delete actions would free, counted once per subject.""" + + per_subject: Dict[str, int] = {} + for action in plan.get("actions") or []: + if not isinstance(action, Mapping) or action.get("kind") not in _RECLAIM_KINDS: + continue + subject = action.get("subject_id") or action.get("id") or "" + size = action.get("size_bytes") or action.get("bytes") or 0 + try: + size = int(size) + except (TypeError, ValueError): + size = 0 + per_subject[subject] = max(per_subject.get(subject, 0), size) + return sum(per_subject.values()) + + +# -------------------------------------------------------------------------- +# build_graph +# -------------------------------------------------------------------------- + + +def build_graph(plan: Mapping[str, Any], analysis: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]: + """Turn a plan (and optionally a storage analysis) into the preview graph.""" + + if not isinstance(plan, Mapping): + raise TypeError("plan must be a mapping loaded from plan.json") + + lang = plan.get("lang") if plan.get("lang") in ("zh", "en") else "zh" + root_portable = _scrub(plan.get("source_root_portable") or plan.get("source_root") or "~") + root_label = os.path.basename(root_portable.rstrip("/")) or root_portable + + nodes: List[Dict[str, Any]] = [] + links: List[Dict[str, str]] = [] + index: Dict[str, Dict[str, Any]] = {} + + def add_node(node: Dict[str, Any]) -> Dict[str, Any]: + node.setdefault("color", None) + node.setdefault("size_bytes", 0) + node.setdefault("to", None) + node.setdefault("from", None) + node.setdefault("hint", None) + index[node["id"]] = node + nodes.append(node) + return node + + def add_link(source: str, target: str, kind: str) -> None: + links.append({"source": source, "target": target, "kind": kind}) + + add_node( + { + "id": "root", + "label": root_label, + "type": "root", + "depth": 1, + "to": root_portable, + "from": root_portable, + } + ) + + # -- machine layer ------------------------------------------------------ + storage_leaves: List[Dict[str, Any]] = [] + if isinstance(analysis, Mapping): + machine = analysis.get("machine") or {} + host = machine.get("hostname") if isinstance(machine, Mapping) else None + add_node( + { + "id": "machine", + "label": _scrub(host) or PREVIEW_TEXT[lang]["machine"], + "type": "machine", + "depth": 0, + "hint": _pick_lang((analysis.get("overview") or {}).get("headline"), lang), + } + ) + add_link("root", "machine", "after") + for item in analysis.get("items") or []: + if not isinstance(item, Mapping): + continue + item_id = item.get("id") + if not item_id: + continue + path = _scrub(item.get("path_portable") or item.get("path") or "") + parent = path.rsplit("/", 1)[0] if "/" in path else path + try: + size = int(item.get("size_bytes") or 0) + except (TypeError, ValueError): + size = 0 + color = item.get("color") if item.get("color") in ("green", "yellow", "red") else "yellow" + what = _pick_lang(item.get("what"), lang) or "" + hint = what + if color == "green": + hint = (PREVIEW_TEXT[lang]["tip_clean"] + " · " + what).strip(" ·") + node = { + "id": "s:" + str(item_id), + "label": _scrub(item.get("name") or item_id), + "type": "storage", + "depth": 2, + "color": color, + "size_bytes": size, + "from": parent or None, + "to": None, + "hint": hint or None, + "pending_clean": color == "green", + "parent": "machine", + } + storage_leaves.append(node) + + # -- partitions --------------------------------------------------------- + partitions = _partition_names(plan) + partition_by_label: Dict[str, str] = {} + for entry in partitions: + node_id = "p:" + entry["key"] + partition_by_label[entry["label"]] = node_id + add_node( + { + "id": node_id, + "label": entry["label"], + "type": "partition", + "depth": 2, + "to": root_label + "/" + entry["label"], + } + ) + add_link(node_id, "root", "after") + + # -- files -------------------------------------------------------------- + roots = ( + _scrub(plan.get("source_root_portable") or ""), + plan.get("source_root") or "", + ) + folder_ids: Dict[str, str] = {} + file_leaves: List[Dict[str, Any]] = [] + moving = 0 + staying = 0 + + for action in _subject_actions(plan): + subject = str(action.get("subject_id") or action.get("id")) + kind = action.get("kind") + try: + size = int(action.get("size_bytes") or action.get("bytes") or 0) + except (TypeError, ValueError): + size = 0 + color = action.get("color") if action.get("color") in ("green", "yellow", "red") else "yellow" + label = _scrub(action.get("filename") or subject) + hint_map = action.get("reason") if isinstance(action.get("reason"), Mapping) else None + node: Dict[str, Any] = { + "id": "f:" + subject, + "label": label, + "type": "file", + "depth": 4, + "color": color, + "size_bytes": size, + "kind": "dir" if action.get("subject_kind") == "dir" else "file", + "from": root_label, + "hint": _pick_lang(hint_map, lang), + "hint_i18n": _scrub(dict(hint_map)) if hint_map else None, + "moving": kind == "move", + # The report page ticks actions, not subjects. Carrying the action + # id lets a ``cfo:selection`` event find this node without the + # fragment having to know anything about plan.json. + "action_id": str(action.get("id") or ""), + } + + parent = "root" + relative = _relative_destination(action, roots) if kind == "move" else None + if relative: + segments = [part for part in relative.split("/") if part] + directories = segments[:-1] + if directories: + node["to"] = "/".join(directories) + partition_id = partition_by_label.get(directories[0]) + if partition_id is None: + partition_id = "p:" + directories[0] + if partition_id not in index: + add_node( + { + "id": partition_id, + "label": directories[0], + "type": "partition", + "depth": 2, + "to": root_label + "/" + directories[0], + } + ) + add_link(partition_id, "root", "after") + partition_by_label[directories[0]] = partition_id + parent = partition_id + if len(directories) >= 2: + folder_key = directories[0] + "/" + directories[1] + folder_id = folder_ids.get(folder_key) + if folder_id is None: + folder_id = "d:" + folder_key + folder_ids[folder_key] = folder_id + add_node( + { + "id": folder_id, + "label": directories[1], + "type": "folder", + "depth": 3, + "to": folder_key, + } + ) + add_link(folder_id, partition_id, "after") + parent = folder_id + + if node.get("to") is None: + node["to"] = root_label + node["moving"] = False + + node["parent"] = parent + if node["moving"]: + moving += 1 + else: + staying += 1 + file_leaves.append(node) + + # -- cap ---------------------------------------------------------------- + leaves = file_leaves + storage_leaves + structural = len(nodes) + budget = MAX_NODES - structural - 1 + dropped: List[Dict[str, Any]] = [] + if budget < 0: + budget = 0 + if len(leaves) > budget: + leaves = sorted(leaves, key=lambda item: (-int(item.get("size_bytes") or 0), item["id"])) + dropped = leaves[budget:] + leaves = leaves[:budget] + + kept_ids = {node["id"] for node in leaves} + for node in file_leaves + storage_leaves: + if node["id"] not in kept_ids: + continue + parent = node.pop("parent") + add_node(node) + add_link(node["id"], parent, "after") + if node["type"] == "file": + add_link(node["id"], "root", "now") + + if dropped: + rest_bytes = sum(int(item.get("size_bytes") or 0) for item in dropped) + rest_label = { + code: PREVIEW_TEXT[code]["rest"].format(k=len(dropped)) for code in ("zh", "en") + } + add_node( + { + "id": "agg:rest", + "label": rest_label[lang], + "label_i18n": rest_label, + "type": "file", + "depth": 4, + "color": "yellow", + "size_bytes": rest_bytes, + "kind": "file", + "from": root_label, + "to": root_label, + "moving": False, + "aggregate": len(dropped), + } + ) + add_link("agg:rest", "root", "after") + add_link("agg:rest", "root", "now") + + reclaim = _reclaim_bytes(plan) + if isinstance(analysis, Mapping): + for item in analysis.get("items") or []: + if isinstance(item, Mapping) and item.get("color") == "green": + try: + reclaim += int(item.get("size_bytes") or 0) + except (TypeError, ValueError): + pass + + stats = { + "files": len(file_leaves), + "moving": moving, + "staying": staying, + "partitions": len(partitions), + "reclaim_bytes": reclaim, + "storage_items": len(storage_leaves), + "dropped": len(dropped), + "lang": lang, + "root": root_label, + } + return {"nodes": nodes, "links": links, "stats": stats} + + +# -------------------------------------------------------------------------- +# render +# -------------------------------------------------------------------------- + + +def _js_json(payload: Any) -> str: + """JSON that is safe to sit inline in a `` +
    """ + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +def _load(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog="carl_file_organizer.preview", + description="Render the after picture for a plan as a standalone HTML page.", + ) + parser.add_argument("plan", help="path to plan.json") + parser.add_argument("--analysis", default=None, help="path to a storage analysis JSON") + parser.add_argument("-o", "--output", default=None, help="write the page here (default: stdout)") + parser.add_argument("--lang", default=None, choices=["zh", "en"], help="page language") + parser.add_argument("--height", type=int, default=720, help="canvas height in px") + parser.add_argument("--fragment", action="store_true", help="emit only the
    ") + args = parser.parse_args(list(argv) if argv is not None else None) + + plan = _load(args.plan) + analysis = _load(args.analysis) if args.analysis else None + graph = build_graph(plan, analysis) + lang = args.lang or (plan.get("lang") if plan.get("lang") in ("zh", "en") else "zh") + if args.fragment: + html = render_preview_html(graph, lang=lang, height=args.height) + else: + html = render_preview_page(graph, lang=lang, height=args.height) + + if args.output: + directory = os.path.dirname(os.path.abspath(args.output)) + if directory: + os.makedirs(directory, exist_ok=True) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(html) + sys.stderr.write( + "preview: %d nodes, %d links -> %s\n" + % (len(graph["nodes"]), len(graph["links"]), args.output) + ) + else: + sys.stdout.write(html) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py new file mode 100644 index 0000000..0c1b881 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/__init__.py @@ -0,0 +1,10 @@ +"""Built-in profiles. + +Profiles are data, not code. Load them with +``importlib.resources.files("carl_file_organizer.profiles").joinpath("tiered.json")`` +so they keep working from a wheel; never build a path from ``__file__``. +""" + +from __future__ import annotations + +BUILTIN_PROFILES = ("tiered", "simple") diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json new file mode 100644 index 0000000..bee5a7f --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/simple.json @@ -0,0 +1,500 @@ +{ + "profile": "simple", + "profile_version": 1, + "lang": "en", + "managed_dir": { + "zh": "00_File_Organizer", + "en": "00_File_Organizer" + }, + "transient_tag": { + "zh": "文件移动", + "en": "Moved by File Organizer" + }, + "review_log": { + "zh": "review-log.md", + "en": "review-log.md" + }, + "names": { + "documents": { + "zh": "Documents", + "en": "Documents" + }, + "images": { + "zh": "Images", + "en": "Images" + }, + "video": { + "zh": "Video", + "en": "Video" + }, + "audio": { + "zh": "Audio", + "en": "Audio" + }, + "archives": { + "zh": "Archives", + "en": "Archives" + }, + "installers": { + "zh": "Installers", + "en": "Installers" + }, + "code": { + "zh": "Code & Data", + "en": "Code & Data" + }, + "other": { + "zh": "Other", + "en": "Other" + }, + "inbox": { + "zh": "Other", + "en": "Other" + }, + "inbox.pending": { + "zh": "Pending", + "en": "Pending" + }, + "inbox.dup": { + "zh": "Duplicates_Review", + "en": "Duplicates_Review" + }, + "sensitive": { + "zh": "Sensitive", + "en": "Sensitive" + }, + "sensitive.pending": { + "zh": "Pending", + "en": "Pending" + }, + "sensitive.dup": { + "zh": "Duplicates_Review", + "en": "Duplicates_Review" + } + }, + "aging": { + "default_hours": 0, + "archive_hours": 0 + }, + "rules": { + ".doc": { + "dest": "documents", + "reason": "ext_doc" + }, + ".docx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".epub": { + "dest": "documents", + "reason": "ext_doc" + }, + ".key": { + "dest": "documents", + "reason": "ext_doc" + }, + ".md": { + "dest": "documents", + "reason": "ext_doc" + }, + ".numbers": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pages": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pdf": { + "dest": "documents", + "reason": "ext_doc" + }, + ".ppt": { + "dest": "documents", + "reason": "ext_doc" + }, + ".pptx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".rtf": { + "dest": "documents", + "reason": "ext_doc" + }, + ".txt": { + "dest": "documents", + "reason": "ext_doc" + }, + ".xls": { + "dest": "documents", + "reason": "ext_doc" + }, + ".xlsx": { + "dest": "documents", + "reason": "ext_doc" + }, + ".avif": { + "dest": "images", + "reason": "ext_image" + }, + ".gif": { + "dest": "images", + "reason": "ext_image" + }, + ".heic": { + "dest": "images", + "reason": "ext_image" + }, + ".jpeg": { + "dest": "images", + "reason": "ext_image" + }, + ".jpg": { + "dest": "images", + "reason": "ext_image" + }, + ".png": { + "dest": "images", + "reason": "ext_image" + }, + ".svg": { + "dest": "images", + "reason": "ext_image" + }, + ".tif": { + "dest": "images", + "reason": "ext_image" + }, + ".tiff": { + "dest": "images", + "reason": "ext_image" + }, + ".webp": { + "dest": "images", + "reason": "ext_image" + }, + ".avi": { + "dest": "video", + "reason": "ext_video" + }, + ".m4v": { + "dest": "video", + "reason": "ext_video" + }, + ".mkv": { + "dest": "video", + "reason": "ext_video" + }, + ".mov": { + "dest": "video", + "reason": "ext_video" + }, + ".mp4": { + "dest": "video", + "reason": "ext_video" + }, + ".webm": { + "dest": "video", + "reason": "ext_video" + }, + ".aac": { + "dest": "audio", + "reason": "ext_audio" + }, + ".flac": { + "dest": "audio", + "reason": "ext_audio" + }, + ".m4a": { + "dest": "audio", + "reason": "ext_audio" + }, + ".mp3": { + "dest": "audio", + "reason": "ext_audio" + }, + ".ogg": { + "dest": "audio", + "reason": "ext_audio" + }, + ".wav": { + "dest": "audio", + "reason": "ext_audio" + }, + ".7z": { + "dest": "archives", + "reason": "ext_archive" + }, + ".bz2": { + "dest": "archives", + "reason": "ext_archive" + }, + ".gz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".rar": { + "dest": "archives", + "reason": "ext_archive" + }, + ".tar": { + "dest": "archives", + "reason": "ext_archive" + }, + ".tgz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".xz": { + "dest": "archives", + "reason": "ext_archive" + }, + ".zip": { + "dest": "archives", + "reason": "ext_archive" + }, + ".apk": { + "dest": "installers", + "reason": "ext_installer" + }, + ".dmg": { + "dest": "installers", + "reason": "ext_installer" + }, + ".exe": { + "dest": "installers", + "reason": "ext_installer" + }, + ".iso": { + "dest": "installers", + "reason": "ext_installer" + }, + ".msi": { + "dest": "installers", + "reason": "ext_installer" + }, + ".pkg": { + "dest": "installers", + "reason": "ext_installer" + }, + ".c": { + "dest": "code", + "reason": "ext_data" + }, + ".cpp": { + "dest": "code", + "reason": "ext_data" + }, + ".css": { + "dest": "code", + "reason": "ext_data" + }, + ".csv": { + "dest": "code", + "reason": "ext_data" + }, + ".go": { + "dest": "code", + "reason": "ext_data" + }, + ".html": { + "dest": "code", + "reason": "ext_data" + }, + ".ipynb": { + "dest": "code", + "reason": "ext_data" + }, + ".java": { + "dest": "code", + "reason": "ext_data" + }, + ".js": { + "dest": "code", + "reason": "ext_data" + }, + ".json": { + "dest": "code", + "reason": "ext_data" + }, + ".jsx": { + "dest": "code", + "reason": "ext_data" + }, + ".py": { + "dest": "code", + "reason": "ext_data" + }, + ".rb": { + "dest": "code", + "reason": "ext_data" + }, + ".rs": { + "dest": "code", + "reason": "ext_data" + }, + ".sql": { + "dest": "code", + "reason": "ext_data" + }, + ".swift": { + "dest": "code", + "reason": "ext_data" + }, + ".toml": { + "dest": "code", + "reason": "ext_data" + }, + ".ts": { + "dest": "code", + "reason": "ext_data" + }, + ".tsx": { + "dest": "code", + "reason": "ext_data" + }, + ".xml": { + "dest": "code", + "reason": "ext_data" + }, + ".yaml": { + "dest": "code", + "reason": "ext_data" + }, + ".yml": { + "dest": "code", + "reason": "ext_data" + } + }, + "name_patterns": [], + "sensitive": { + "exact_names": [ + ".env", + ".netrc", + "credentials.json", + "id_rsa", + "id_ed25519", + "secrets.json" + ], + "exact_stems": [ + "key", + "keys", + "env", + "pass", + "token", + "secret", + "auth" + ], + "suffixes": [ + ".cer", + ".crt", + ".pem", + ".p12", + ".pfx", + ".mobileprovision", + ".env" + ], + "markers": [ + "credential", + "password", + "passwd", + "apikey", + "api-key", + "api_key", + "private-key", + "recovery", + "secret" + ], + "marker_min_len": 5 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "forbidden": { + "basenames": [ + ".git", + "Library", + "Keychains", + "Messages", + "Mail" + ], + "suffixes": [ + ".photoslibrary", + ".aplibrary", + ".sparsebundle", + ".vmwarevm", + ".pvm" + ], + "markers": [ + ".obsidian" + ], + "glob": [ + "Profile *", + "Photos Library*" + ] + }, + "regenerable": { + "basenames": [ + "node_modules", + ".next", + "dist", + "build", + "coverage", + "__pycache__" + ], + "scan_depth": 3, + "restore": { + "_default": { + "zh": "由构建再生", + "en": "regenerated by the build" + } + } + }, + "derivative_hints": [ + "-copy", + " copy", + "-backup" + ], + "archive_suffixes": [ + ".zip", + ".tar.gz", + ".tgz", + ".tar", + ".7z", + ".rar", + ".gz", + ".xz", + ".bz2" + ], + "installer_suffixes": [ + ".dmg", + ".pkg", + ".iso", + ".exe", + ".msi", + ".apk" + ], + "dedupe": { + "hash_max_bytes": 2147483648, + "copy_patterns": [ + "^(?P.+?) \\((?P\\d+)\\)(?P\\.[^.]+)?$", + "^(?P.+?)_(?P\\d+)(?P\\.[^.]+)?$" + ] + }, + "tags": { + "group_by_user_tags": false + }, + "guard": { + "reference_files": [], + "reference_scan_roots": [ + "$HOME", + "$HOME/bin", + "$HOME/.local/bin", + "/usr/local/bin", + "/opt/homebrew/bin" + ], + "recent_minutes": 10, + "lsof_timeout_s": 5 + }, + "large_bytes": 524288000 +} diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json new file mode 100644 index 0000000..aafbb85 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/profiles/tiered.json @@ -0,0 +1,501 @@ +{ + "profile": "tiered", + "profile_version": 1, + "lang": null, + "managed_dir": { + "zh": "00_下载目录管理", + "en": "00_File_Organizer" + }, + "transient_tag": { + "zh": "文件移动", + "en": "Moved by File Organizer" + }, + "review_log": { + "zh": "人工调整日志.md", + "en": "review-log.md" + }, + "names": { + "inbox": { + "zh": "00_收件箱", + "en": "00_Inbox" + }, + "inbox.pending": { + "zh": "待判断", + "en": "Pending" + }, + "inbox.by_tag": { + "zh": "按标签", + "en": "By_Tag" + }, + "inbox.dup": { + "zh": "重复待确认", + "en": "Duplicates_Review" + }, + "work": { + "zh": "10_工作区", + "en": "10_Workspace" + }, + "work.code": { + "zh": "项目代码", + "en": "Projects" + }, + "work.tools": { + "zh": "工具与导出", + "en": "Tools_Exports" + }, + "work.tools.scripts": { + "zh": "脚本", + "en": "Scripts" + }, + "work.db": { + "zh": "数据库与查询", + "en": "Database" + }, + "work.db.sql": { + "zh": "SQL", + "en": "SQL" + }, + "library": { + "zh": "20_知识库", + "en": "20_Library" + }, + "library.docs": { + "zh": "文档资料", + "en": "Documents" + }, + "library.docs.md": { + "zh": "Markdown", + "en": "Markdown" + }, + "library.docs.pdf": { + "zh": "PDF", + "en": "PDF" + }, + "library.docs.ppt": { + "zh": "PPT", + "en": "PPT" + }, + "library.docs.csv": { + "zh": "CSV", + "en": "CSV" + }, + "library.docs.txt": { + "zh": "TXT", + "en": "TXT" + }, + "library.docs.office": { + "zh": "办公文档", + "en": "Office" + }, + "library.images": { + "zh": "图片素材", + "en": "Images" + }, + "library.images.png": { + "zh": "PNG", + "en": "PNG" + }, + "library.images.jpeg": { + "zh": "JPEG", + "en": "JPEG" + }, + "library.images.gif": { + "zh": "GIF", + "en": "GIF" + }, + "library.images.other": { + "zh": "其他图片", + "en": "Other" + }, + "library.video": { + "zh": "视频素材", + "en": "Video" + }, + "library.video.mp4": { + "zh": "MP4", + "en": "MP4" + }, + "library.video.mov": { + "zh": "MOV", + "en": "MOV" + }, + "library.audio": { + "zh": "音频素材", + "en": "Audio" + }, + "library.web": { + "zh": "网页资料", + "en": "Web" + }, + "library.web.html": { + "zh": "HTML", + "en": "HTML" + }, + "sensitive": { + "zh": "60_敏感信息", + "en": "60_Sensitive" + }, + "sensitive.pending": { + "zh": "待转移", + "en": "Pending" + }, + "sensitive.dup": { + "zh": "重复待确认", + "en": "Duplicates_Review" + }, + "archive": { + "zh": "90_归档", + "en": "90_Archive" + }, + "archive.zip": { + "zh": "压缩包", + "en": "Compressed" + }, + "archive.zip.zip": { + "zh": "ZIP", + "en": "ZIP" + }, + "archive.zip.other": { + "zh": "其他压缩包", + "en": "Other" + }, + "archive.installers": { + "zh": "安装包", + "en": "Installers" + }, + "archive.installers.dmg": { + "zh": "DMG", + "en": "DMG" + }, + "archive.installers.other": { + "zh": "其他安装包", + "en": "Other" + } + }, + "aging": { + "default_hours": 48, + "archive_hours": 168 + }, + "rules": { + ".md": { + "dest": "library.docs.md", + "reason": "ext_doc" + }, + ".pdf": { + "dest": "library.docs.pdf", + "reason": "ext_doc" + }, + ".ppt": { + "dest": "library.docs.ppt", + "reason": "ext_doc" + }, + ".pptx": { + "dest": "library.docs.ppt", + "reason": "ext_doc" + }, + ".doc": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".docx": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".xls": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".xlsx": { + "dest": "library.docs.office", + "reason": "ext_doc" + }, + ".csv": { + "dest": "library.docs.csv", + "reason": "ext_doc" + }, + ".txt": { + "dest": "library.docs.txt", + "reason": "ext_doc" + }, + ".png": { + "dest": "library.images.png", + "reason": "ext_image" + }, + ".jpg": { + "dest": "library.images.jpeg", + "reason": "ext_image" + }, + ".jpeg": { + "dest": "library.images.jpeg", + "reason": "ext_image" + }, + ".gif": { + "dest": "library.images.gif", + "reason": "ext_image" + }, + ".webp": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".heic": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".svg": { + "dest": "library.images.other", + "reason": "ext_image" + }, + ".mp4": { + "dest": "library.video.mp4", + "reason": "ext_video" + }, + ".mov": { + "dest": "library.video.mov", + "reason": "ext_video" + }, + ".mp3": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".m4a": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".wav": { + "dest": "library.audio", + "reason": "ext_audio" + }, + ".html": { + "dest": "library.web.html", + "reason": "ext_web" + }, + ".js": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".sh": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".py": { + "dest": "work.tools.scripts", + "reason": "ext_script" + }, + ".sql": { + "dest": "work.db.sql", + "reason": "ext_data" + }, + ".zip": { + "dest": "archive.zip.zip", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".tar.gz": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".7z": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".rar": { + "dest": "archive.zip.other", + "min_age_hours": 168, + "reason": "ext_archive" + }, + ".dmg": { + "dest": "archive.installers.dmg", + "min_age_hours": 168, + "reason": "ext_installer" + }, + ".pkg": { + "dest": "archive.installers.other", + "min_age_hours": 168, + "reason": "ext_installer" + }, + ".iso": { + "dest": "archive.installers.other", + "min_age_hours": 168, + "reason": "ext_installer" + } + }, + "name_patterns": [], + "sensitive": { + "exact_names": [ + ".env", + ".netrc", + "credentials.json", + "id_rsa", + "id_ed25519", + "secrets.json", + "service-account.json" + ], + "exact_stems": [ + "key", + "keys", + "env", + "pass", + "token", + "tokens", + "secret", + "auth", + "cred", + "creds" + ], + "suffixes": [ + ".cer", + ".crt", + ".key", + ".pem", + ".p12", + ".pfx", + ".mobileprovision", + ".env", + ".keystore", + ".jks" + ], + "markers": [ + "credential", + "password", + "passwd", + "apikey", + "api-key", + "api_key", + "access-token", + "access_token", + "private-key", + "privatekey", + "recovery", + "secret", + "mnemonic", + "seed-phrase", + "密码", + "密钥", + "凭证", + "恢复码", + "身份证", + "银行卡", + "营业执照" + ], + "marker_min_len": 5 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "forbidden": { + "basenames": [ + ".git", + "Library", + "Keychains", + "Messages", + "Mail", + "vm_bundles", + "Time Machine Backups" + ], + "suffixes": [ + ".photoslibrary", + ".aplibrary", + ".musiclibrary", + ".tvlibrary", + ".sparsebundle", + ".vmwarevm", + ".pvm", + ".utm" + ], + "markers": [ + ".obsidian", + "Default/History" + ], + "glob": [ + "Profile *", + "Photos Library*" + ] + }, + "regenerable": { + "basenames": [ + "node_modules", + ".next", + "dist", + "build", + "coverage", + "__pycache__", + ".turbo", + ".cache", + ".pytest_cache", + "target" + ], + "scan_depth": 3, + "restore": { + "node_modules": { + "zh": "重新 npm/pnpm/bun install", + "en": "re-run npm/pnpm/bun install" + }, + ".next": { + "zh": "重新 build 或 dev", + "en": "re-run build or dev" + }, + "__pycache__": { + "zh": "下次运行自动生成", + "en": "regenerated on next run" + }, + "_default": { + "zh": "由构建再生", + "en": "regenerated by the build" + } + } + }, + "derivative_hints": [ + "-fix-", + "-worktree-", + "-scratch", + "-backup", + "-copy", + " copy" + ], + "archive_suffixes": [ + ".zip", + ".tar.gz", + ".tgz", + ".tar", + ".7z", + ".rar", + ".gz", + ".xz", + ".bz2" + ], + "installer_suffixes": [ + ".dmg", + ".pkg", + ".iso", + ".exe", + ".msi", + ".apk" + ], + "dedupe": { + "hash_max_bytes": 2147483648, + "copy_patterns": [ + "^(?P.+?) \\((?P\\d+)\\)(?P\\.[^.]+)?$", + "^(?P.+?)_(?P\\d+)(?P\\.[^.]+)?$", + "^(?P.+?) (?P\\d+)$" + ] + }, + "tags": { + "group_by_user_tags": false + }, + "guard": { + "reference_files": [], + "reference_scan_roots": [ + "$HOME", + "$HOME/projects", + "$HOME/bin", + "$HOME/.local/bin", + "/usr/local/bin", + "/opt/homebrew/bin" + ], + "recent_minutes": 10, + "lsof_timeout_s": 5 + }, + "large_bytes": 524288000 +} diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/render.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/render.py new file mode 100644 index 0000000..35bcbc3 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/render.py @@ -0,0 +1,158 @@ +"""One renderer for both entrances, loaded from the script next door. + +``scripts/build_report.py`` owns the three-colour page: it detects whether it is +looking at a plan.json or an analysis.json, strips every absolute path, lays the +Agent's notes over the rules and fills the template. It is a script rather than +a module of this package because a person runs it by hand +(``python3 scripts/build_report.py analysis.json -o report.html``). + +The review server has to produce byte-for-byte the same page, so it does not +reimplement any of that: this module loads that very file through +:mod:`importlib` and hands its functions on. One renderer, one template, no +second copy to keep in step. + + from . import render + html = render.render(plan, mode="serve", token=token) + +The module is loaded once and cached. ``build_report.py`` sitting next to the +package is part of the skill's layout, so a missing file is a broken install and +raises :class:`RenderError` with a sentence saying so, never an ImportError +traceback in the middle of a review session. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from typing import Any, Dict, Optional, Union + +#: ``carl_file_organizer/render.py`` -> ``scripts/build_report.py`` +BUILD_REPORT_PATH = Path(__file__).resolve().parent.parent / "build_report.py" + +MODULE_NAME = "carl_file_organizer._build_report" + +_MODULE: Optional[Any] = None + + +class RenderError(RuntimeError): + """The renderer could not be loaded or refused the data it was given.""" + + +def module(path: Optional[Union[str, Path]] = None) -> Any: + """The loaded ``build_report`` module, imported at most once per process.""" + + global _MODULE + if _MODULE is not None and path is None: + return _MODULE + target = Path(path) if path is not None else BUILD_REPORT_PATH + if not target.is_file(): + raise RenderError( + "the report renderer is missing: {0}".format(target) + ) + existing = sys.modules.get(MODULE_NAME) + if existing is not None and path is None: + _MODULE = existing + return existing + spec = importlib.util.spec_from_file_location(MODULE_NAME, str(target)) + if spec is None or spec.loader is None: + raise RenderError("cannot load the report renderer from {0}".format(target)) + loaded = importlib.util.module_from_spec(spec) + sys.modules[MODULE_NAME] = loaded + try: + spec.loader.exec_module(loaded) + except Exception as error: # noqa: BLE001 - a broken renderer is one sentence + sys.modules.pop(MODULE_NAME, None) + raise RenderError("the report renderer failed to load: {0}".format(error)) from error + if path is None: + _MODULE = loaded + return loaded + + +def detect_kind(data: Dict[str, Any]) -> str: + """``"organize"`` for a plan.json, ``"storage"`` for an analysis.json.""" + + try: + return str(module().detect_kind(data)) + except RenderError: + raise + except ValueError as error: + raise RenderError(str(error)) from error + + +def sanitize(data: Any, home: Optional[Union[str, Path]] = None) -> Any: + """The renderer's own redaction, so callers do not grow a second one.""" + + return module().sanitize(data, home) + + +def render( + data: Dict[str, Any], + *, + mode: str = "static", + token: str = "", + lang: Optional[str] = None, + notes: Optional[Dict[str, Any]] = None, + kind: Optional[str] = None, + home: Optional[Union[str, Path]] = None, +) -> str: + """The finished page. Same arguments, same output, as ``build_report.render``.""" + + try: + return str( + module().render( + data, mode=mode, token=token, lang=lang, notes=notes, kind=kind, home=home + ) + ) + except RenderError: + raise + except ValueError as error: + raise RenderError(str(error)) from error + + +# -------------------------------------------------------------------------- +# storage-report subcommand +# -------------------------------------------------------------------------- + + +def _read_json(path: Path, what: str) -> Dict[str, Any]: + import json + + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except OSError as error: + raise RenderError("cannot read {0} {1}: {2}".format(what, path, error)) from error + except ValueError as error: + raise RenderError("{0} {1} is not valid JSON: {2}".format(what, path, error)) from error + if not isinstance(data, dict): + raise RenderError("{0} {1} must hold a JSON object".format(what, path)) + return data + + +def run(args: Any) -> int: + """``storage-report [--notes n.json] [-o page.html] [--serve]``.""" + + analysis_path = Path(getattr(args, "analysis")) + analysis = _read_json(analysis_path, "analysis.json") + notes_path = getattr(args, "notes", None) + notes = _read_json(Path(notes_path), "notes.json") if notes_path else None + lang = getattr(args, "lang", None) + + if bool(getattr(args, "serve", False)): + from . import server as server_module + + return server_module.serve_review( + analysis_path, + port=int(getattr(args, "port", 0) or 0), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + open_browser=bool(getattr(args, "open_browser", True)), + notes=notes, + lang=lang, + ) + + html = render(analysis, mode="static", lang=lang, notes=notes) + output = Path(getattr(args, "output", None) or (analysis_path.resolve().parent / "report.html")) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(html, encoding="utf-8") + print(output) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py new file mode 100644 index 0000000..768a314 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/scanner.py @@ -0,0 +1,320 @@ +"""Read-only top level scan: one Entry per visible item, nothing is opened. + +The scan is deliberately shallow. Partition folders and the managed folder are +boundaries, hidden files and symlinks are ignored, and directory sizes are +measured with both an entry cap and a time budget so a runaway folder can never +stall a plan. + +Frozen for every work package:: + + scan_top_level(root, config, *, now, dir_sizes=True) -> (entries, notes) + dir_stats_bounded(path, max_entries=50000, time_budget_s=1.5) -> DirStats + Entry.subject_id # from carl_file_organizer.ids.subject_id + +One rule worth stating loudly: ``Entry.size_bytes`` for a directory is the +walked total, but its ``subject_id`` is computed from the raw ``stat`` size. +Walked totals are capped and would change under ``--no-dir-sizes``, so mixing +them into the identity would invalidate every directory action; the executor +recomputes the same raw-stat identity before it touches anything. +""" + +from __future__ import annotations + +import fnmatch +import os +import plistlib +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, List, Optional, Sequence, Tuple + +from . import config as config_module +from .ids import subject_id as _subject_id +from .paths import DOUBLE_SUFFIXES, split_suffix + +#: Compound suffixes and the split that honours them live in :mod:`paths`, which +#: needs them to build conflict names; they are re-exported here because this is +#: where every caller already looks for them. +__all_reexported__ = ("DOUBLE_SUFFIXES", "split_suffix") + +#: Carl File Organizer's own artefacts, skipped so a downloaded approval file is not +#: treated as a loose document on the next run. +ARTIFACT_GLOBS = ("carl-file-organizer-*.json",) + +TAG_XATTR = "com.apple.metadata:_kMDItemUserTags" + +DEFAULT_MAX_ENTRIES = 50000 +DEFAULT_TIME_BUDGET_S = 1.5 + + +@dataclass +class DirStats: + files: int + bytes: int + truncated: bool + + def as_dict(self) -> dict: + return {"files": self.files, "bytes": self.bytes, "truncated": self.truncated} + + +@dataclass +class Entry: + path: Path + name: str + kind: str # "file" | "dir" + size_bytes: int + modified_ns: int + age_hours: int + suffix: str + tags: List[str] = field(default_factory=list) + dir_stats: Optional[DirStats] = None + subject_id: str = "" + + @property + def is_dir(self) -> bool: + return self.kind == "dir" + + +@dataclass +class ScanNotes: + """What the scan deliberately did not look at.""" + + skipped_hidden: int = 0 + skipped_symlinks: int = 0 + skipped_partitions: int = 0 + skipped_artifacts: int = 0 + unreadable: List[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# names and suffixes +# --------------------------------------------------------------------------- + + +def suffix_of(name: str) -> str: + return split_suffix(name)[1] + + +def stem_of(name: str) -> str: + return split_suffix(name)[0] + + +def is_artifact_name(name: str) -> bool: + if name == config_module.DOTFILE_NAME: + return True + return any(fnmatch.fnmatch(name, pattern) for pattern in ARTIFACT_GLOBS) + + +# --------------------------------------------------------------------------- +# Finder tags +# --------------------------------------------------------------------------- + + +def read_tags(path: Path) -> List[str]: + """Finder tags without their ``\\n`` suffix; ``[]`` off macOS.""" + + if sys.platform != "darwin": + return [] + try: + result = subprocess.run( + ["xattr", "-p", "-x", TAG_XATTR, str(path)], + text=True, + capture_output=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return [] + if result.returncode != 0: + return [] + hex_text = "".join(result.stdout.split()) + if not hex_text: + return [] + try: + tags = plistlib.loads(bytes.fromhex(hex_text)) + except Exception: # noqa: BLE001 - a corrupt xattr is not worth a crash + return [] + if not isinstance(tags, list): + return [] + cleaned = [] + for tag in tags: + text = str(tag).split("\n", 1)[0].strip() + if text: + cleaned.append(text) + return cleaned + + +def user_tags(tags: Sequence[str], transient: Sequence[str]) -> List[str]: + """Drop Carl File Organizer's own temporary tag in every language spelling.""" + + blocked = {str(item).casefold() for item in transient} + return [tag for tag in tags if tag.casefold() not in blocked] + + +# --------------------------------------------------------------------------- +# bounded directory statistics +# --------------------------------------------------------------------------- + + +def dir_stats_bounded( + path: Path, + max_entries: int = DEFAULT_MAX_ENTRIES, + time_budget_s: float = DEFAULT_TIME_BUDGET_S, +) -> DirStats: + """Count files and bytes under ``path`` with a hard entry and time cap.""" + + started = time.monotonic() + files = 0 + total = 0 + seen = 0 + truncated = False + stack: List[str] = [str(path)] + + while stack: + if seen >= max_entries or (time.monotonic() - started) > time_budget_s: + truncated = True + break + current = stack.pop() + try: + iterator = os.scandir(current) + except OSError: + truncated = True + continue + with iterator: + while True: + try: + item = next(iterator) + except StopIteration: + break + except OSError: + truncated = True + break + seen += 1 + if seen >= max_entries or (time.monotonic() - started) > time_budget_s: + truncated = True + break + try: + if item.is_symlink(): + continue + if item.is_dir(follow_symlinks=False): + stack.append(item.path) + continue + stat = item.stat(follow_symlinks=False) + except OSError: + truncated = True + continue + files += 1 + total += stat.st_size + return DirStats(files=files, bytes=total, truncated=truncated) + + +# --------------------------------------------------------------------------- +# the scan itself +# --------------------------------------------------------------------------- + + +def _age_hours(modified_ns: int, now: datetime) -> int: + seconds = now.timestamp() - (modified_ns / 1_000_000_000) + if seconds <= 0: + return 0 + return int(seconds // 3600) + + +def make_entry( + path: Path, + *, + now: datetime, + dir_sizes: bool = True, + stat_result: Any = None, + dir_limits: Optional[dict] = None, + read_finder_tags: bool = True, +) -> Entry: + """Build one :class:`Entry`; ``path`` must already be known to be safe.""" + + stat = stat_result if stat_result is not None else os.stat(str(path), follow_symlinks=False) + kind = "dir" if os.path.isdir(str(path)) else "file" + limits = dir_limits or {} + stats = None + if kind == "dir" and dir_sizes: + stats = dir_stats_bounded( + path, + int(limits.get("max_entries", DEFAULT_MAX_ENTRIES)), + float(limits.get("time_budget_s", DEFAULT_TIME_BUDGET_S)), + ) + size = stats.bytes if stats is not None else (0 if kind == "dir" else stat.st_size) + modified_ns = stat.st_mtime_ns + return Entry( + path=path, + name=path.name, + kind=kind, + size_bytes=size, + modified_ns=modified_ns, + age_hours=_age_hours(modified_ns, now), + suffix=suffix_of(path.name) if kind == "file" else "", + tags=read_tags(path) if read_finder_tags else [], + dir_stats=stats, + # The identity always uses the raw stat size, never the walked size: + # directory totals are capped and would change with --no-dir-sizes. + subject_id=_subject_id(path, kind, stat.st_size, modified_ns), + ) + + +def scan_top_level( + root: Path, + config: Any, + *, + now: datetime, + dir_sizes: bool = True, + read_finder_tags: bool = True, +) -> Tuple[List[Entry], ScanNotes]: + """One level deep, sorted by name; partitions and hidden items are skipped.""" + + base = Path(root) + notes = ScanNotes() + partitions = {str(name) for name in config_module.partition_dir_names(config)} + limits = { + "max_entries": DEFAULT_MAX_ENTRIES, + "time_budget_s": DEFAULT_TIME_BUDGET_S, + } + + entries: List[Entry] = [] + try: + children = sorted(base.iterdir(), key=lambda item: item.name.casefold()) + except OSError as error: + raise ValueError("cannot read {0}: {1}".format(base, error)) from error + + for path in children: + name = path.name + if name.startswith("."): + notes.skipped_hidden += 1 + continue + if path.is_symlink(): + notes.skipped_symlinks += 1 + continue + if name in partitions: + notes.skipped_partitions += 1 + continue + if is_artifact_name(name): + notes.skipped_artifacts += 1 + continue + try: + stat = os.stat(str(path), follow_symlinks=False) + except OSError: + notes.unreadable.append(name) + continue + if not (path.is_dir() or path.is_file()): + notes.unreadable.append(name) + continue + entries.append( + make_entry( + path, + now=now, + dir_sizes=dir_sizes, + stat_result=stat, + dir_limits=limits, + read_finder_tags=read_finder_tags, + ) + ) + return entries, notes diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py new file mode 100644 index 0000000..69ffd01 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/server.py @@ -0,0 +1,693 @@ +"""``review --serve`` and ``storage-report --serve``: the page on 127.0.0.1. + +One server, one page, whichever halves it was handed. A plan.json alone +answers ``/api/apply``, an analysis.json alone answers ``/api/dispose``, and the +combined envelope ``{"plan": ..., "analysis": ...}`` answers both, which is what +``report --serve`` sends. Everything else -- the page itself, the token, the +checks, the reveal endpoint, the shutdown -- is the same in all three cases, +because there is one template and one set of rules about what may leave this +process. + +Security model (see docs/review-page.md): + +* binds ``127.0.0.1`` only, port 0 by default, one random token per run; +* the token is printed to stderr once and handed to the browser in the URL; + the page wipes it from the address bar and sends it as ``X-GN-Token``; +* every request checks the token with ``hmac.compare_digest``, the ``Host`` + header, and for POST the same-origin ``Origin`` header, JSON content type + and a 1 MiB body cap; +* the server validates nothing about the plan or the analysis itself. + ``/api/apply`` fills ``approved_action_ids`` and calls the very same + ``apply_approved_plan`` the CLI uses; ``/api/dispose`` builds a decisions + document and calls the very same ``dispose.apply_decisions``. Every safety + check lives in one place, and it is not this file; +* single-threaded ``HTTPServer`` plus a lock: executions never overlap. +""" + +from __future__ import annotations + +import copy +import hmac +import json +import os +import secrets +import sys +import threading +import webbrowser +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from urllib.parse import parse_qs, urlsplit + +from . import paths +from . import render as render_module + +MAX_BODY_BYTES = 1024 * 1024 +DRAIN_LIMIT_BYTES = 8 * MAX_BODY_BYTES +TOKEN_HEADER = "X-GN-Token" +EXECUTED_STATUSES = ("moved", "trashed", "deleted") +JSON_TYPE = "application/json" + +ORGANIZE = "organize" +STORAGE = "storage" +COMBINED = "combined" + +ApplyFn = Callable[..., Any] +DisposeFn = Callable[..., Any] +RevealFn = Callable[..., bool] + + +class ServerError(RuntimeError): + """Raised when the review server cannot be prepared.""" + + +def _default_apply_fn() -> ApplyFn: + try: + from .executor import apply_approved_plan + except ImportError as error: # pragma: no cover - depends on the executor work package + raise ServerError( + "executor.apply_approved_plan is unavailable ({0}); review --serve needs it".format(error) + ) from error + return apply_approved_plan + + +def _default_dispose_fn() -> DisposeFn: + try: + from .dispose import apply_decisions + except ImportError as error: # pragma: no cover - depends on the dispose module + raise ServerError( + "dispose.apply_decisions is unavailable ({0}); the storage page needs it".format(error) + ) from error + return apply_decisions + + +def _default_reveal_fn() -> RevealFn: + from .opener import reveal + + return reveal + + +def _load_plan(plan_path: Union[str, Path]) -> Dict[str, Any]: + """Read the plan.json or analysis.json this run is about.""" + + path = Path(plan_path) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except OSError as error: + raise ServerError("cannot read plan: {0}".format(error)) from error + except ValueError as error: + raise ServerError("plan is not valid JSON: {0}".format(error)) from error + if not isinstance(data, dict): + raise ServerError("plan must be a JSON object") + return data + + +def detect_kind(data: Dict[str, Any]) -> str: + """Which page this document asks for, or a refusal saying it asks for neither.""" + + try: + return render_module.detect_kind(data) + except render_module.RenderError as error: + raise ServerError(str(error)) from error + + +class ReviewState: + """Everything the handler needs, shared through ``server.gn_state``.""" + + def __init__( + self, + plan: Dict[str, Any], + *, + token: str, + allow_permanent_delete: bool, + apply_fn: Optional[ApplyFn], + kind: Optional[str] = None, + home: Optional[Path] = None, + managed_dir: Optional[Path] = None, + dispose_fn: Optional[DisposeFn] = None, + reveal_fn: Optional[RevealFn] = None, + notes: Optional[Dict[str, Any]] = None, + lang: Optional[str] = None, + ) -> None: + self.plan = plan + self.kind = kind or detect_kind(plan) + self.token = token + self.allow_permanent_delete = allow_permanent_delete + self._apply_fn = apply_fn + self._dispose_fn = dispose_fn + self._reveal_fn = reveal_fn + self.notes = notes + self.lang = lang + self.lock = threading.Lock() + self.executed_ids: List[str] = [] + self.port = 0 + self.home = Path(home) if home is not None else self._home_of(plan) + self.managed_dir = Path(managed_dir) if managed_dir is not None else None + for document in self._documents(): + caps = dict(document.get("capabilities") or {}) + caps["permanent_delete_enabled"] = bool(allow_permanent_delete) + document["capabilities"] = caps + + def _documents(self) -> List[Dict[str, Any]]: + """The one or two documents this run is about, in the order they render.""" + + if self.kind != COMBINED: + return [self.plan] + return [half for half in (self.organize_doc, self.storage_doc) if half] + + @property + def organize_doc(self) -> Dict[str, Any]: + """The tidy-up half, or an empty dict when this run has none.""" + + if self.kind == STORAGE: + return {} + if self.kind == ORGANIZE: + return self.plan + inner = self.plan.get("plan") + return inner if isinstance(inner, dict) else {} + + @property + def storage_doc(self) -> Dict[str, Any]: + """The whole-machine half, or an empty dict when this run has none.""" + + if self.kind == ORGANIZE: + return {} + if self.kind == STORAGE: + return self.plan + inner = self.plan.get("analysis") + return inner if isinstance(inner, dict) else {} + + def _home_of(self, plan: Dict[str, Any]) -> Path: + organize = self.organize_doc + if organize: + return Path(paths.plan_home(organize)) + return Path.home() + + @property + def apply_fn(self) -> ApplyFn: + if self._apply_fn is None: + self._apply_fn = _default_apply_fn() + return self._apply_fn + + @property + def dispose_fn(self) -> DisposeFn: + if self._dispose_fn is None: + self._dispose_fn = _default_dispose_fn() + return self._dispose_fn + + @property + def reveal_fn(self) -> RevealFn: + if self._reveal_fn is None: + self._reveal_fn = _default_reveal_fn() + return self._reveal_fn + + def public_plan(self) -> Dict[str, Any]: + """What leaves this process: the document with every absolute path removed. + + Only the page and ``/api/plan`` get this. ``self.plan`` keeps its + absolute paths and is what ``/api/apply`` and ``/api/dispose`` hand to + the engine, so stripping here costs the acting path nothing. + """ + + if self.kind == ORGANIZE: + data = paths.strip_absolute(self.plan) + elif self.kind == STORAGE: + data = render_module.sanitize(self.plan, self.home) + else: + data = {} + if self.organize_doc: + data["plan"] = paths.strip_absolute(self.organize_doc) + if self.storage_doc: + data["analysis"] = render_module.sanitize(self.storage_doc, self.home) + data["capabilities"] = dict((self.storage_doc or self.organize_doc).get("capabilities") or {}) + data["executed_ids"] = list(self.executed_ids) + return data + + def page(self) -> str: + """The rendered page, from the same renderer ``build_report.py`` uses.""" + + data = copy.deepcopy(self.plan) + data["executed_ids"] = list(self.executed_ids) + try: + return render_module.render( + data, + mode="serve", + token=self.token, + lang=self.lang, + notes=self.notes, + kind=self.kind, + home=self.home, + ) + except render_module.RenderError as error: + raise ServerError(str(error)) from error + + def kinds_for(self, action_ids: List[str]) -> Dict[str, str]: + actions = (self.organize_doc or {}).get("actions") or [] + index = {str(a.get("id")): str(a.get("kind", "")) for a in actions if isinstance(a, dict)} + return {i: index.get(i, "") for i in action_ids} + + +def _normalize_results(report: Any) -> List[Dict[str, Any]]: + """``ApplyReport.results`` or a bare list of dicts -> list of dicts.""" + + results = getattr(report, "results", report) + if results is None: + return [] + out = [] + for item in results: + if isinstance(item, dict): + out.append(item) + else: + out.append( + { + "action_id": getattr(item, "action_id", None), + "status": getattr(item, "status", None), + "detail": getattr(item, "detail", None), + } + ) + return out + + +class ReviewHandler(BaseHTTPRequestHandler): + server_version = "CarlFileOrganizerReview/0.3" + sys_version = "" + protocol_version = "HTTP/1.0" + + # -- plumbing ----------------------------------------------------------- + + @property + def state(self) -> ReviewState: + return self.server.gn_state # type: ignore[attr-defined] + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib signature + # never echo the query string (it carries the token on first load) + path = urlsplit(self.path).path + sys.stderr.write( + "{0} - {1} {2} {3}\n".format(self.address_string(), self.command, path, args[1] if len(args) > 1 else "") + ) + + def _send_json(self, status: int, payload: Any) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", JSON_TYPE + "; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + self.wfile.write(body) + + def _send_html(self, html: str) -> None: + body = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.send_header("Referrer-Policy", "no-referrer") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header( + "Content-Security-Policy", + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; " + "connect-src 'self'; img-src data:; base-uri 'none'; form-action 'none'", + ) + self.end_headers() + self.wfile.write(body) + + def _origin(self) -> str: + return "http://127.0.0.1:{0}".format(self.state.port) + + def _host_ok(self) -> bool: + return self.headers.get("Host", "") == "127.0.0.1:{0}".format(self.state.port) + + def _token_ok(self, presented: Optional[str]) -> bool: + return hmac.compare_digest(str(presented or ""), self.state.token) + + def _guard_common(self) -> bool: + if not self._host_ok(): + self._send_json(400, {"error": "bad Host header"}) + return False + return True + + # -- GET ---------------------------------------------------------------- + + def do_GET(self) -> None: # noqa: N802 - stdlib naming + if not self._guard_common(): + return + parts = urlsplit(self.path) + if parts.path == "/": + token = (parse_qs(parts.query).get("t") or [""])[0] + if not self._token_ok(token): + self._send_json(401, {"error": "missing or wrong token"}) + return + with self.state.lock: + html = self.state.page() + self._send_html(html) + return + if not self._token_ok(self.headers.get(TOKEN_HEADER)): + self._send_json(401, {"error": "missing or wrong token"}) + return + if parts.path == "/api/plan": + with self.state.lock: + self._send_json(200, self.state.public_plan()) + return + self._send_json(404, {"error": "not found"}) + + # -- POST --------------------------------------------------------------- + + def _read_json_body(self) -> Optional[Dict[str, Any]]: + origin = self.headers.get("Origin", "") + if origin != self._origin(): + self._send_json(403, {"error": "Origin must be {0}".format(self._origin())}) + return None + if not self._token_ok(self.headers.get(TOKEN_HEADER)): + self._send_json(401, {"error": "missing or wrong token"}) + return None + content_type = self.headers.get("Content-Type", "").split(";")[0].strip().lower() + if content_type != JSON_TYPE: + self._send_json(415, {"error": "Content-Type must be application/json"}) + return None + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + self._send_json(400, {"error": "bad Content-Length"}) + return None + if length < 0 or length > MAX_BODY_BYTES: + self._drain(length) + self._send_json(413, {"error": "body larger than 1 MiB"}) + return None + raw = self.rfile.read(length) if length else b"" + try: + data = json.loads(raw.decode("utf-8")) if raw else {} + except ValueError: + self._send_json(400, {"error": "body is not valid JSON"}) + return None + if not isinstance(data, dict): + self._send_json(400, {"error": "body must be a JSON object"}) + return None + return data + + def _drain(self, length: int) -> None: + """Read and discard an oversized body (bounded) so the client sees the 413, not a reset.""" + + remaining = min(max(length, 0), DRAIN_LIMIT_BYTES) + while remaining > 0: + chunk = self.rfile.read(min(65536, remaining)) + if not chunk: + break + remaining -= len(chunk) + + def _endpoints(self) -> Dict[str, Callable[[Dict[str, Any]], None]]: + table: Dict[str, Callable[[Dict[str, Any]], None]] = {"/api/reveal": self._reveal} + if self.state.organize_doc: + table["/api/apply"] = self._apply + if self.state.storage_doc: + table["/api/dispose"] = self._dispose + return table + + def do_POST(self) -> None: # noqa: N802 - stdlib naming + if not self._guard_common(): + return + path = urlsplit(self.path).path + table = self._endpoints() + if path not in table and path != "/api/shutdown": + # keep the same checks for unknown paths so probing reveals nothing + if self._read_json_body() is not None: + self._send_json(404, {"error": "not found"}) + return + body = self._read_json_body() + if body is None: + return + if path == "/api/shutdown": + self._send_json(200, {"ok": True, "shutting_down": True}) + threading.Thread(target=self.server.shutdown, daemon=True).start() + return + table[path](body) + + # -- the tidy-up page --------------------------------------------------- + + def _apply(self, body: Dict[str, Any]) -> None: + action_ids = body.get("action_ids") + overrides = body.get("overrides", []) + dry_run = bool(body.get("dry_run", False)) + if not isinstance(action_ids, list) or not all(isinstance(i, str) for i in action_ids): + self._send_json(400, {"error": "action_ids must be a list of strings"}) + return + if not action_ids: + self._send_json(400, {"error": "action_ids is empty"}) + return + if not isinstance(overrides, list) or not all(isinstance(o, dict) for o in overrides): + self._send_json(400, {"error": "overrides must be a list of objects"}) + return + state = self.state + with state.lock: + repeated = sorted(set(action_ids) & set(state.executed_ids)) + if repeated: + self._send_json(409, {"error": "already executed", "action_ids": repeated}) + return + kinds = state.kinds_for(action_ids) + deletes = sorted(i for i, kind in kinds.items() if kind == "delete") + if deletes and not state.allow_permanent_delete: + self._send_json( + 403, + { + "error": "permanent delete is off; restart with --allow-permanent-delete", + "action_ids": deletes, + }, + ) + return + plan = copy.deepcopy(state.organize_doc) + plan["approved_action_ids"] = list(action_ids) + plan["overrides"] = list(overrides) + plan["approved_at"] = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + plan["approved_by"] = "serve" + try: + report = state.apply_fn( + plan, dry_run=dry_run, allow_permanent_delete=state.allow_permanent_delete + ) + except Exception as error: # noqa: BLE001 - executor refusals become one JSON line + self._send_json(422, {"error": "{0}: {1}".format(type(error).__name__, error)}) + return + results = _normalize_results(report) + newly_done = [] + if not dry_run: + for item in results: + if item.get("status") in EXECUTED_STATUSES and item.get("action_id"): + aid = str(item["action_id"]) + if aid not in state.executed_ids: + state.executed_ids.append(aid) + newly_done.append(aid) + payload: Dict[str, Any] = { + "ok": True, + "dry_run": dry_run, + "results": results, + "executed_ids": list(state.executed_ids), + "newly_executed": newly_done, + } + for key in ("run_id", "manifest", "audit"): + value = getattr(report, key, None) + if value is not None: + payload[key] = str(value) + self._send_json(200, payload) + + # -- the whole-machine page --------------------------------------------- + + def _dispose(self, body: Dict[str, Any]) -> None: + item_ids = body.get("item_ids") + action = body.get("action", "trash") + if not isinstance(item_ids, list) or not all(isinstance(i, str) for i in item_ids): + self._send_json(400, {"error": "item_ids must be a list of strings"}) + return + if not item_ids: + self._send_json(400, {"error": "item_ids is empty"}) + return + if action not in ("trash", "delete"): + self._send_json(400, {"error": 'action must be "trash" or "delete"'}) + return + state = self.state + with state.lock: + if action == "delete" and not state.allow_permanent_delete: + self._send_json( + 403, + { + "error": "permanent delete is off; restart with --allow-permanent-delete", + "item_ids": sorted(item_ids), + }, + ) + return + repeated = sorted(set(item_ids) & set(state.executed_ids)) + if repeated: + self._send_json(409, {"error": "already executed", "item_ids": repeated}) + return + decisions = { + "schema": "carl-file-organizer/storage-decisions", + "schema_version": 1, + "item_ids": list(item_ids), + "actions": {i: action for i in item_ids}, + "decided_at": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"), + "decided_by": "serve", + } + try: + report = state.dispose_fn( + copy.deepcopy(state.storage_doc), + decisions, + dry_run=False, + allow_permanent_delete=state.allow_permanent_delete, + home=state.home, + managed_dir=state.managed_dir, + ) + except Exception as error: # noqa: BLE001 - a refusal becomes one JSON line + self._send_json(422, {"error": "{0}: {1}".format(type(error).__name__, error)}) + return + results = _normalize_results(report) + newly_done = [] + for item in results: + if item.get("status") in EXECUTED_STATUSES and item.get("item_id"): + iid = str(item["item_id"]) + if iid not in state.executed_ids: + state.executed_ids.append(iid) + newly_done.append(iid) + payload: Dict[str, Any] = { + "ok": True, + "dry_run": False, + "action": action, + "results": results, + "executed_ids": list(state.executed_ids), + "newly_executed": newly_done, + } + for key in ("run_id", "manifest", "audit"): + value = getattr(report, key, None) + if value is not None: + payload[key] = str(value) + self._send_json(200, payload) + + # -- both pages --------------------------------------------------------- + + def _reveal(self, body: Dict[str, Any]) -> None: + """Put a folder on screen. Reads nothing, writes nothing, moves nothing.""" + + portable = body.get("path_portable") + if not isinstance(portable, str) or not portable.strip(): + self._send_json(400, {"error": "path_portable must be a non-empty string"}) + return + state = self.state + target = paths.expand_portable(portable, state.home) + if not paths.inside_root(target, state.home): + self._send_json(403, {"error": "that path is outside the home directory"}) + return + if not os.path.lexists(str(target)): + self._send_json(404, {"error": "that path is not there any more"}) + return + try: + shown = bool(state.reveal_fn(target)) + except Exception as error: # noqa: BLE001 - a file manager is never worth a 500 + self._send_json(200, {"ok": False, "error": str(error)}) + return + self._send_json(200, {"ok": shown, "path_portable": portable}) + + +# -------------------------------------------------------------------------- +# public API +# -------------------------------------------------------------------------- + + +def build_server( + plan: Union[Dict[str, Any], str, Path], + *, + host: str = "127.0.0.1", + port: int = 0, + allow_permanent_delete: bool = False, + apply_fn: Optional[ApplyFn] = None, + dispose_fn: Optional[DisposeFn] = None, + reveal_fn: Optional[RevealFn] = None, + token: Optional[str] = None, + home: Optional[Path] = None, + managed_dir: Optional[Path] = None, + notes: Optional[Dict[str, Any]] = None, + lang: Optional[str] = None, +) -> Tuple[HTTPServer, str]: + """Prepare (but do not run) the review server. Returns ``(server, token)``.""" + + data = _load_plan(plan) if not isinstance(plan, dict) else copy.deepcopy(plan) + token = token or secrets.token_urlsafe(32) + state = ReviewState( + data, + token=token, + allow_permanent_delete=allow_permanent_delete, + apply_fn=apply_fn, + dispose_fn=dispose_fn, + reveal_fn=reveal_fn, + home=home, + managed_dir=managed_dir, + notes=notes, + lang=lang, + ) + server = HTTPServer((host, port), ReviewHandler) + state.port = server.server_address[1] + server.gn_state = state # type: ignore[attr-defined] + return server, token + + +def serve_review( + plan_path: Union[str, Path, Dict[str, Any]], + *, + host: str = "127.0.0.1", + port: int = 0, + allow_permanent_delete: bool = False, + open_browser: bool = True, + apply_fn: Optional[ApplyFn] = None, + dispose_fn: Optional[DisposeFn] = None, + home: Optional[Path] = None, + managed_dir: Optional[Path] = None, + notes: Optional[Dict[str, Any]] = None, + lang: Optional[str] = None, +) -> int: + """Run the review server until Ctrl-C or ``POST /api/shutdown``.""" + + server, token = build_server( + plan_path, + host=host, + port=port, + allow_permanent_delete=allow_permanent_delete, + apply_fn=apply_fn, + dispose_fn=dispose_fn, + home=home, + managed_dir=managed_dir, + notes=notes, + lang=lang, + ) + state = server.gn_state # type: ignore[attr-defined] + # fail early with a clear line instead of a 422 on the first click + if state.organize_doc and apply_fn is None: + state.apply_fn # noqa: B018 + if state.storage_doc and dispose_fn is None: + state.dispose_fn # noqa: B018 + bound_port = server.server_address[1] + url = "http://127.0.0.1:{0}/?t={1}".format(bound_port, token) + sys.stderr.write("review page (token inside, only on this line): {0}\n".format(url)) + sys.stderr.write("page kind: {0}\n".format(state.kind)) + if allow_permanent_delete: + sys.stderr.write("permanent delete is ENABLED for this session\n") + sys.stderr.write("press Ctrl-C to stop\n") + if open_browser: + try: + webbrowser.open(url) + except Exception: # noqa: BLE001 - a browser failure is not fatal + pass + try: + server.serve_forever(poll_interval=0.5) + except KeyboardInterrupt: + return 130 + finally: + server.server_close() + return 0 + + +def run(args: Any) -> int: + """CLI entry: ``review --serve [--port N] [--no-open] [--allow-permanent-delete]``.""" + + return serve_review( + args.plan, + port=int(getattr(args, "port", 0) or 0), + allow_permanent_delete=bool(getattr(args, "allow_permanent_delete", False)), + open_browser=bool(getattr(args, "open_browser", True)), + ) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py new file mode 100644 index 0000000..e44a161 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/status.py @@ -0,0 +1,241 @@ +"""Two answers: when this folder was last tidied, and whether it is messy again. + +The first answer comes only from the manifests in the managed directory. A +folder's modification time changes every time anything is downloaded into it, +so it says nothing about whether anyone ever organized it; the manifests are +the only honest record. + +The second answer comes from a fresh read-only plan. When the planner is not +available the status still prints, and simply says the recheck could not run. +""" + +from __future__ import annotations + +import sys +import unicodedata +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from . import i18n +from . import manifest as manifest_module + +TEXT = { + "never": {"zh": "还没整理过", "en": "not tidied yet"}, + "last_line": { + "zh": "{when}，动过 {count} 项（清单 {name}）", + "en": "{when}, {count} item(s) (manifest {name})", + }, + "clean": {"zh": "顶层是干净的", "en": "the top level is clear"}, + "dirty": { + "zh": "散件 {loose} · 静置中 {aging} · 待判断 {pending} · 敏感待转移 {sensitive}", + "en": "loose {loose} · settling {aging} · undecided {pending} · sensitive {sensitive}", + }, + "no_recheck": { + "zh": "无法复查（{error}），下面的数字这次没有算出来", + "en": "recheck unavailable ({error}); the counts below were not computed", + }, + "planner_missing": { + "zh": "扫描模块还没接上", + "en": "the planner is not wired up yet", + }, + "root_line": {"zh": "目录", "en": "Folder"}, +} + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class StatusReport: + root: Path + lang: str = "en" + last_run: Optional[manifest_module.RunInfo] = None + loose: int = 0 + aging: int = 0 + pending: int = 0 + sensitive_pending: int = 0 + recheck_actions: Optional[int] = None + recheck_error: Optional[str] = None + managed: Optional[Path] = None + notes: List[str] = field(default_factory=list) + + +#: Rules that mean the item is waiting for a person to decide where it goes. +PENDING_RULES = ("unknown-ext", "dir:pending", "dir:derivative") + + +def _plan_counts(plan: Dict[str, Any], report: StatusReport) -> None: + actions = plan.get("actions") or [] + loose = set() + aging = set() + pending = set() + sensitive = set() + approvable = 0 + + for action in actions: + if not isinstance(action, dict): + continue + subject = str(action.get("subject_id") or action.get("id") or "") + kind = str(action.get("kind") or "") + tier = str(action.get("tier") or "") + rule = str(action.get("rule") or "") + if action.get("approvable", kind != "hold"): + approvable += 1 + if kind == "hold": + if tier == "aging" or rule == "hold:aging": + aging.add(subject) + continue + loose.add(subject) + if rule in PENDING_RULES or str(action.get("destination_key") or "").startswith( + "inbox.pending" + ): + pending.add(subject) + if tier == "sensitive": + sensitive.add(subject) + + report.loose = len(loose) + report.aging = len(aging) + report.pending = len(pending) + report.sensitive_pending = len(sensitive) + report.recheck_actions = approvable + + +def _default_plan_fn(root: Path, config: Any, now: Optional[datetime]) -> Dict[str, Any]: + """Call ``planner.build_plan`` with whichever keywords it actually accepts.""" + + import inspect + + from . import planner as planner_module + + build = getattr(planner_module, "build_plan", None) + if build is None: + raise RuntimeError(text("planner_missing", "en")) + + options: Dict[str, Any] = {} + try: + parameters = inspect.signature(build).parameters + except (TypeError, ValueError): + parameters = {} + if "config" in parameters: + options["config"] = config + if "now" in parameters and now is not None: + options["now"] = now + if "dir_sizes" in parameters: + options["dir_sizes"] = False + if "hash_duplicates" in parameters: + options["hash_duplicates"] = False + return build(root, **options) + + +def compute_status( + root: Path, + *, + config: Any, + now: Optional[datetime] = None, + plan_fn: Optional[Callable[[], Dict[str, Any]]] = None, +) -> StatusReport: + """Last run from the manifests, current mess from a fresh read-only plan.""" + + from . import config as config_module + + folder = Path(root) + report = StatusReport(root=folder, lang=getattr(config, "lang", "en")) + managed = config_module.managed_dir(config) + report.managed = managed + report.last_run = manifest_module.latest_run(managed) + + try: + plan = plan_fn() if plan_fn is not None else _default_plan_fn(folder, config, now) + except Exception as error: # noqa: BLE001 - status must print even when scanning fails + report.recheck_error = str(error) or error.__class__.__name__ + return report + + if not isinstance(plan, dict): + report.recheck_error = "the planner returned no plan" + return report + + _plan_counts(plan, report) + return report + + +def _display_width(text_value: str) -> int: + """Column width of a label, counting wide CJK characters as two cells.""" + + return sum(2 if unicodedata.east_asian_width(char) in ("W", "F") else 1 for char in text_value) + + +def _pad(text_value: str, width: int) -> str: + return text_value + " " * max(0, width - _display_width(text_value)) + + +def format_status(report: StatusReport, lang: Optional[str] = None) -> str: + """Two labelled lines: when it was last done, and how it looks now.""" + + speech = lang or report.lang or "en" + label_root = text("root_line", speech) + label_last = i18n.t("status_last_run", speech) + label_now = i18n.t("status_dirty_now", speech) + width = max(_display_width(item) for item in (label_root, label_last, label_now)) + + if report.last_run is None: + last = text("never", speech) + else: + stamp = report.last_run.moved_at or "" + if report.last_run.timestamp is not None and not stamp: + stamp = report.last_run.timestamp.strftime("%Y-%m-%d %H:%M:%S") + last = text( + "last_line", + speech, + when=stamp, + count=report.last_run.done, + name=report.last_run.path.name, + ) + + if report.recheck_error is not None: + now_line = text("no_recheck", speech, error=report.recheck_error) + elif not (report.loose or report.aging or report.pending or report.sensitive_pending): + now_line = text("clean", speech) + else: + now_line = text( + "dirty", + speech, + loose=report.loose, + aging=report.aging, + pending=report.pending, + sensitive=report.sensitive_pending, + ) + + lines = [ + "{0} {1}".format(_pad(label_root, width), report.root), + "{0} {1}".format(_pad(label_last, width), last), + "{0} {1}".format(_pad(label_now, width), now_line), + ] + return "\n".join(lines) + + +def run(args: Any) -> int: + """``carl-file-organizer status [dir] [--lang zh|en]``.""" + + config = getattr(args, "config", None) + if config is None: + from . import config as config_module + from . import paths + + root = paths.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + config = config_module.resolve_config(root, lang=getattr(args, "lang", None)) + + report = compute_status(config.root, config=config) + print(format_status(report, config.lang)) + if report.recheck_error: + print(report.recheck_error, file=sys.stderr) + return 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py new file mode 100644 index 0000000..d28ea23 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/tags.py @@ -0,0 +1,302 @@ +"""macOS Finder tags: read, write, add, remove, and clear the temporary one. + +The temporary tag is how a person sees what one run touched: open Finder, click +the tag, and there is the whole batch. Tagging is a courtesy, never a +precondition, so every failure here warns on stderr and returns False instead +of stopping a batch that already moved files. + +Everywhere except macOS these functions are honest no-ops. + +This module also owns the ``clear-tags`` subcommand. +""" + +from __future__ import annotations + +import os +import plistlib +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence + +TAG_XATTR = "com.apple.metadata:_kMDItemUserTags" + +#: Finder stores a colour as a second line inside the tag string. +COLOR_SEPARATOR = "\n" + +TAG_LIST_GLOB = "*-move-tags-*.txt" + +MESSAGES = { + "cleared": {"zh": "已清除标签 {tag}：{count} 项", "en": "Cleared tag {tag} from {count} item(s)"}, + "dry_run": { + "zh": "预演：会清除标签 {tag}，共 {count} 项，没有真的改动", + "en": "Dry run: would clear tag {tag} from {count} item(s); nothing changed", + }, + "unsupported": { + "zh": "这台机器不支持 Finder 标签，没有可清除的东西", + "en": "Finder tags are not available on this machine; nothing to clear", + }, + "no_lists": { + "zh": "管理目录里没有标签清单（{glob}），可以用 --from 指定一个", + "en": "No tag list ({glob}) in the managed folder; pass --from to name one", + }, + "missing": {"zh": "找不到：{path}", "en": "missing: {path}"}, + "summary": { + "zh": "检查 {checked} 项，清除 {cleared} 项，跳过 {skipped} 项，失败 {failed} 项", + "en": "checked {checked}, cleared {cleared}, skipped {skipped}, failed {failed}", + }, +} + + +@dataclass +class ClearReport: + tag: str + dry_run: bool + checked: int = 0 + cleared: int = 0 + skipped: int = 0 + missing: int = 0 + failed: int = 0 + lists: List[Path] = field(default_factory=list) + details: List[Dict[str, Any]] = field(default_factory=list) + + +def message(key: str, lang: str = "en", **kw: Any) -> str: + entry = MESSAGES.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def _warn(text: str) -> None: + print("tag_warning: {0}".format(text), file=sys.stderr) + + +def supported() -> bool: + """True only on macOS with the ``xattr`` tool available.""" + + return sys.platform == "darwin" and bool(shutil.which("xattr")) + + +# -------------------------------------------------------------------------- +# raw tag access +# -------------------------------------------------------------------------- + + +def read_tags(path: Path) -> List[str]: + """Every Finder tag on ``path``, colour suffix included, or an empty list.""" + + if not supported(): + return [] + try: + result = subprocess.run( + ["xattr", "-p", "-x", TAG_XATTR, str(path)], + text=True, + capture_output=True, + ) + except (OSError, subprocess.SubprocessError) as error: + _warn("{0}\t{1}".format(path, error)) + return [] + if result.returncode != 0: + return [] + hex_text = "".join(result.stdout.split()) + if not hex_text: + return [] + try: + tags = plistlib.loads(bytes.fromhex(hex_text)) + except Exception: # noqa: BLE001 - a malformed xattr is not worth a traceback + return [] + if not isinstance(tags, list): + return [] + return [str(tag) for tag in tags] + + +def write_tags(path: Path, tags: Sequence[str]) -> bool: + """Replace the tag list on ``path``; warn and return False on any failure.""" + + if not supported(): + return False + try: + payload = plistlib.dumps([str(tag) for tag in tags], fmt=plistlib.FMT_BINARY).hex() + subprocess.run( + ["xattr", "-w", "-x", TAG_XATTR, payload, str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.SubprocessError) as error: + _warn("{0}\t{1}".format(path, error)) + return False + return True + + +def add_tag(path: Path, tag: str) -> bool: + """Add one tag, keeping the others. False means nothing was written.""" + + if not supported(): + return False + current = read_tags(path) + if any(_tag_name(item) == tag for item in current): + return False + return write_tags(path, list(current) + [tag]) + + +def remove_tag(path: Path, tag: str) -> bool: + """Drop one tag, keeping the others. False means nothing was written.""" + + if not supported(): + return False + current = read_tags(path) + remaining = [item for item in current if _tag_name(item) != tag] + if len(remaining) == len(current): + return False + return write_tags(path, remaining) + + +def _tag_name(tag: str) -> str: + """``"Red\\n6"`` -> ``"Red"``: strip the colour line Finder appends.""" + + return str(tag).split(COLOR_SEPARATOR, 1)[0] + + +def user_tags(source: Any, transient: Iterable[str] = ()) -> List[str]: + """Tags a person put there: colours stripped, our temporary tag removed. + + ``source`` is either a path to read or an already read list of tags. + """ + + if isinstance(source, (str, os.PathLike)): + raw: Iterable[str] = read_tags(Path(source)) + else: + raw = source or [] + drop = {str(item) for item in transient} + names: List[str] = [] + for tag in raw: + name = _tag_name(tag) + if not name or name in drop or name in names: + continue + names.append(name) + return names + + +# -------------------------------------------------------------------------- +# clear-tags +# -------------------------------------------------------------------------- + + +def clear_tags(paths: Iterable[Path], tag: str, *, dry_run: bool = False) -> ClearReport: + """Remove ``tag`` from every path that still has it.""" + + report = ClearReport(tag=tag, dry_run=dry_run) + for item in paths: + target = Path(item) + report.checked += 1 + if not os.path.lexists(str(target)): + report.missing += 1 + report.details.append({"path": str(target), "status": "missing"}) + continue + current = read_tags(target) + if not any(_tag_name(entry) == tag for entry in current): + report.skipped += 1 + report.details.append({"path": str(target), "status": "untagged"}) + continue + if dry_run: + report.cleared += 1 + report.details.append({"path": str(target), "status": "would-clear"}) + continue + if remove_tag(target, tag): + report.cleared += 1 + report.details.append({"path": str(target), "status": "cleared"}) + else: + report.failed += 1 + report.details.append({"path": str(target), "status": "failed"}) + return report + + +def find_tag_lists(managed: Path) -> List[Path]: + """Every ``*-move-tags-*.txt`` written by past runs, oldest first.""" + + folder = Path(managed) + if not folder.is_dir(): + return [] + return sorted(folder.glob(TAG_LIST_GLOB)) + + +def paths_from_lists(lists: Sequence[Path], *, home: Optional[Path] = None) -> List[Path]: + """Expand the ``$HOME/...`` lines of the given tag lists, in order, deduped.""" + + from . import manifest as manifest_module + from . import paths as paths_module + + seen = set() + found: List[Path] = [] + for source in lists: + for line in manifest_module.read_tag_list(Path(source)): + expanded = paths_module.expand_portable(line, home) + key = str(expanded) + if key in seen: + continue + seen.add(key) + found.append(expanded) + return found + + +def run(args: Any) -> int: + """``carl-file-organizer clear-tags [dir] [--from list]... [--all-lists] [--tag NAME]``.""" + + from . import config as config_module + from . import paths as paths_module + + root = paths_module.refuse_root( + args.source, allow_outside_home=getattr(args, "allow_outside_home", False) + ) + cfg = config_module.resolve_config(root, lang=getattr(args, "lang", None)) + lang = cfg.lang + managed = config_module.managed_dir(cfg) + tag = getattr(args, "tag", None) or config_module.transient_tag(cfg) + + chosen: List[Path] = [] + explicit = list(getattr(args, "from_lists", None) or []) + if explicit: + chosen.extend(Path(item) for item in explicit) + if getattr(args, "all_lists", False) or not explicit: + for candidate in find_tag_lists(managed): + if candidate not in chosen: + chosen.append(candidate) + + missing_lists = [item for item in chosen if not Path(item).is_file()] + for item in missing_lists: + print(message("missing", lang, path=item), file=sys.stderr) + chosen = [item for item in chosen if Path(item).is_file()] + + if not chosen: + print(message("no_lists", lang, glob=TAG_LIST_GLOB), file=sys.stderr) + return 2 + + if not supported(): + print(message("unsupported", lang)) + return 0 + + targets = paths_from_lists(chosen) + report = clear_tags(targets, tag, dry_run=bool(getattr(args, "dry_run", False))) + report.lists = chosen + + for entry in report.details: + print("{0}\t{1}".format(entry["status"], entry["path"])) + key = "dry_run" if report.dry_run else "cleared" + print(message(key, lang, tag=tag, count=report.cleared)) + print( + message( + "summary", + lang, + checked=report.checked, + cleared=report.cleared, + skipped=report.skipped + report.missing, + failed=report.failed, + ) + ) + return 2 if report.failed else 0 diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py new file mode 100644 index 0000000..1f44901 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/trash.py @@ -0,0 +1,270 @@ +"""Move things to the system trash, or refuse; never fall back to deleting. + +macOS goes through Finder, Linux through ``gio``, Windows through the shell's +own recycle bin call. When none of them is available the backend is ``none`` +and every trash request raises :class:`TrashError`, which is the whole point: a +failed trash must never quietly turn into a permanent delete. + +The three backends are not symmetric. Finder and gio are ordinary commands, so +they run through :func:`subprocess.run` and take the ``runner`` seam. Windows +has no command that reliably reaches the recycle bin, so it calls +``SHFileOperationW`` through ctypes and takes the ``win_runner`` seam instead. +Both seams exist for the same reason: this module has to be testable on a +machine that is not the one being tested for. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Callable, List, Optional + +FINDER = "finder" +GIO = "gio" +WINDOWS = "win32" +NONE = "none" + +DEFAULT_TIMEOUT_S = 30.0 + +#: AppleScript error raised when the terminal is not allowed to drive Finder. +AUTOMATION_DENIED = "-1743" + +MESSAGES = { + "no_backend": { + "zh": "这台机器没有可用的废纸篓后端，拒绝处置：{path}", + "en": "No trash backend on this machine; refusing to dispose of {path}", + }, + "still_there": { + "zh": "废纸篓命令跑完了，但 {path} 还在原地，当作失败处理", + "en": "The trash command finished but {path} is still there; treating it as a failure", + }, + "automation_denied": { + "zh": ( + "Finder 拒绝了自动化请求（-1743）。去系统设置 > 隐私与安全性 > 自动化，" + "把终端对 Finder 的开关打开，再跑一次。" + ), + "en": ( + "Finder refused the automation request (-1743). Open System Settings > " + "Privacy & Security > Automation and allow your terminal to control Finder, " + "then run this again." + ), + }, + "failed": { + "zh": "废纸篓命令失败（退出码 {code}）：{detail}", + "en": "The trash command failed (exit code {code}): {detail}", + }, + "windows_unavailable": { + "zh": "这台 Windows 上调不到回收站接口（shell32），拒绝处置：{path}", + "en": "The recycle bin interface (shell32) is unreachable on this Windows; refusing to dispose of {path}", + }, + "windows_failed": { + "zh": "回收站调用失败（SHFileOperationW 返回 {code}）", + "en": "The recycle bin call failed (SHFileOperationW returned {code})", + }, + "windows_aborted": { + "zh": "回收站调用被中止，条目留在原地", + "en": "The recycle bin call was aborted; the item stayed where it was", + }, + "timeout": { + "zh": "废纸篓命令超时（{seconds} 秒），可能有确认框在等你点", + "en": "The trash command timed out after {seconds}s; a confirmation dialog may be waiting", + }, +} + + +class TrashError(RuntimeError): + """The item did not reach the trash. Nothing else is attempted.""" + + +def message(key: str, lang: str = "en", **kw: Any) -> str: + entry = MESSAGES.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +def detect_backend() -> str: + """``finder`` on macOS, ``win32`` on Windows, ``gio`` elsewhere, else ``none``.""" + + if sys.platform == "darwin" and shutil.which("osascript"): + return FINDER + if sys.platform == "win32": + return WINDOWS if windows_available() else NONE + if shutil.which("gio"): + return GIO + return NONE + + +def applescript_quote(text: str) -> str: + """Escape a path for an AppleScript string literal (backslash first).""" + + return str(text).replace("\\", "\\\\").replace('"', '\\"') + + +def finder_command(path: Path) -> List[str]: + """The exact argv used for the Finder backend; never handed to a shell.""" + + absolute = os.path.abspath(str(path)) + script = 'tell application "Finder" to delete POSIX file "{0}"'.format( + applescript_quote(absolute) + ) + return ["osascript", "-e", script] + + +def gio_command(path: Path) -> List[str]: + return ["gio", "trash", "--", os.path.abspath(str(path))] + + +# --------------------------------------------------------------------------- +# Windows: the recycle bin through SHFileOperationW +# --------------------------------------------------------------------------- + +#: ``FO_DELETE``: the operation itself. +FO_DELETE = 0x0003 + +#: Send it to the recycle bin rather than unlinking it. Without this one flag +#: the very same call is a permanent delete, which is the outcome this whole +#: module exists to prevent, so it is never optional. +FOF_ALLOWUNDO = 0x0040 + +#: Do not put a yes/no dialog in front of a script that nobody is watching. +FOF_NOCONFIRMATION = 0x0010 + +#: No progress window. +FOF_SILENT = 0x0004 + +#: No modal error box either. A dialog nobody can click would hang the process +#: forever, and the return code already tells us everything the box would say. +FOF_NOERRORUI = 0x0400 + +#: The exact flag word this tool uses. One constant, so a reviewer can read the +#: policy in one line instead of reassembling it from call sites. +WINDOWS_FLAGS = FOF_ALLOWUNDO | FOF_NOCONFIRMATION | FOF_SILENT | FOF_NOERRORUI + + +def windows_path_argument(path: Path) -> str: + """The ``pFrom`` string: one absolute path, terminated by two NULs. + + ``SHFileOperationW`` reads a list of paths separated by NUL and ended by an + empty entry, so a single path still needs the second NUL. Building it here + rather than at the call site means the shape is testable on any platform. + """ + + return os.path.abspath(str(path)) + "\0\0" + + +def windows_available() -> bool: + """True when ``shell32`` can actually be reached from this interpreter.""" + + if sys.platform != "win32": + return False + try: + import ctypes + + return ctypes.windll.shell32 is not None # type: ignore[attr-defined] + except (ImportError, AttributeError, OSError): + return False + + +def windows_send_to_recycle_bin(path: Path) -> Any: + """Call ``SHFileOperationW`` for one path; return ``(code, aborted)``. + + Nothing here is imported until it is called, so the module still imports + cleanly on macOS and Linux, where ``ctypes.wintypes`` does not exist. + """ + + import ctypes + from ctypes import wintypes + + class SHFILEOPSTRUCTW(ctypes.Structure): + _fields_ = [ + ("hwnd", wintypes.HWND), + ("wFunc", wintypes.UINT), + ("pFrom", wintypes.LPCWSTR), + ("pTo", wintypes.LPCWSTR), + ("fFlags", ctypes.c_uint16), + ("fAnyOperationsAborted", wintypes.BOOL), + ("hNameMappings", ctypes.c_void_p), + ("lpszProgressTitle", wintypes.LPCWSTR), + ] + + operation = SHFILEOPSTRUCTW() + operation.hwnd = None + operation.wFunc = FO_DELETE + operation.pFrom = windows_path_argument(path) + operation.pTo = None + operation.fFlags = WINDOWS_FLAGS + operation.fAnyOperationsAborted = False + operation.hNameMappings = None + operation.lpszProgressTitle = None + + shell32 = ctypes.windll.shell32 # type: ignore[attr-defined] + shell32.SHFileOperationW.argtypes = [ctypes.POINTER(SHFILEOPSTRUCTW)] + shell32.SHFileOperationW.restype = ctypes.c_int + code = int(shell32.SHFileOperationW(ctypes.byref(operation))) + return code, bool(operation.fAnyOperationsAborted) + + +def _windows_trash( + target: Path, + lang: str, + win_runner: Optional[Callable[[Path], Any]], +) -> None: + call = win_runner if win_runner is not None else windows_send_to_recycle_bin + if win_runner is None and not windows_available(): + raise TrashError(message("windows_unavailable", lang, path=target)) + try: + code, aborted = call(target) + except OSError as error: + raise TrashError(str(error)) from error + if int(code) != 0: + raise TrashError(message("windows_failed", lang, code=int(code))) + if aborted: + raise TrashError(message("windows_aborted", lang)) + + +def move_to_trash( + path: Path, + *, + backend: str, + timeout_s: float = DEFAULT_TIMEOUT_S, + runner: Optional[Callable[..., Any]] = None, + win_runner: Optional[Callable[[Path], Any]] = None, + lang: str = "en", +) -> None: + """Send one item to the trash or raise; there is no third outcome.""" + + target = Path(path) + if backend not in (FINDER, GIO, WINDOWS): + raise TrashError(message("no_backend", lang, path=target)) + + if backend == WINDOWS: + _windows_trash(target, lang, win_runner) + if os.path.lexists(str(target)): + raise TrashError(message("still_there", lang, path=target)) + return + + argv = finder_command(target) if backend == FINDER else gio_command(target) + call = runner if runner is not None else subprocess.run + + try: + result = call(argv, capture_output=True, text=True, timeout=timeout_s) + except subprocess.TimeoutExpired as error: + raise TrashError(message("timeout", lang, seconds=timeout_s)) from error + except OSError as error: + raise TrashError(str(error)) from error + + code = int(getattr(result, "returncode", 0) or 0) + stderr = (getattr(result, "stderr", "") or "").strip() + if code != 0: + if AUTOMATION_DENIED in stderr: + raise TrashError(message("automation_denied", lang)) + raise TrashError(message("failed", lang, code=code, detail=stderr or "no output")) + + if os.path.lexists(str(target)): + raise TrashError(message("still_there", lang, path=target)) diff --git a/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py new file mode 100644 index 0000000..b19aa61 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/carl_file_organizer/undo.py @@ -0,0 +1,444 @@ +"""Put things back where they came from, in reverse order. + +Undo reads either the audit log or a TSV manifest, walks it backwards, and +moves each item back to the path it started from. It is deliberately timid: +if the original path is occupied again, or the item is no longer where the +record says it is, that row is skipped and reported rather than forced. + +Trash and permanent delete cannot be undone from here. Trashed items are put +back from the Trash by the person; permanently deleted ones are gone, and +saying so plainly is more useful than pretending. +""" + +from __future__ import annotations + +import os +import shutil +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import manifest as manifest_module +from . import paths +from . import tags as tags_module + +#: Record statuses that describe a completed move, in both spellings. +UNDOABLE_STATUSES = ("moved", "MOVED", "restored", "RESTORED") + +TEXT = { + "trashed": { + "zh": "这一项进了废纸篓，在废纸篓里选放回原处就行，这里不动它", + "en": "This one went to the trash; put it back from there, not from here", + }, + "deleted": { + "zh": "这一项已永久删除，没法还原", + "en": "This one was permanently deleted and cannot be restored", + }, + "not_moved": { + "zh": "这条记录不是一次成功的移动（{status}），跳过", + "en": "This record is not a completed move ({status}); skipped", + }, + "no_paths": { + "zh": "这条记录缺少原路径或当前路径，跳过", + "en": "This record has no original or current path; skipped", + }, + "occupied": { + "zh": "原来的位置已经有同名的东西了，不覆盖，跳过", + "en": "Something is at the original path again; not overwriting, skipped", + }, + "missing": { + "zh": "现在的位置上找不到它，可能已经被人挪走了，跳过", + "en": "It is not where the record says it is; skipped", + }, + "outside": { + "zh": "路径不在整理目录里，拒绝移动", + "en": "That path is outside the folder; refused", + }, + "symlink": {"zh": "路径里有软链，拒绝移动", "en": "A symlink sits in this path; refused"}, + "restored": {"zh": "已放回原处", "en": "put back"}, + "dry_run": {"zh": "预演：会放回原处", "en": "dry run: would put it back"}, + "error": {"zh": "放回失败：{error}", "en": "putting it back failed: {error}"}, + "log": { + "zh": "撤销了 {count} 项移动，已放回原处；清单：`{manifest}`。", + "en": "Undid {count} move(s) and put them back; manifest: `{manifest}`.", + }, + "summary": { + "zh": "共 {total} 条：放回 {restored}，跳过 {skipped}，失败 {failed}", + "en": "{total} record(s): restored {restored}, skipped {skipped}, failed {failed}", + }, + "no_records": { + "zh": "这个文件里没有可撤销的记录：{path}", + "en": "No undoable record in {path}", + }, + "unknown_root": { + "zh": "看不出这些记录属于哪个整理目录：{path}", + "en": "Cannot tell which folder these records belong to: {path}", + }, +} + + +def text(key: str, lang: str = "en", **kw: Any) -> str: + entry = TEXT.get(key, {}) + template = entry.get(lang) or entry.get("en") or key + try: + return template.format(**kw) + except (KeyError, IndexError, ValueError): + return template + + +@dataclass +class UndoStep: + record: Dict[str, Any] + kind: str + status: str + ok: bool + detail: str = "" + action_id: str = "" + original: Optional[Path] = None + current: Optional[Path] = None + size_bytes: int = 0 + + +@dataclass +class UndoReport: + run_id: str + dry_run: bool + results: List[Dict[str, Any]] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + manifest: Optional[Path] = None + audit: Optional[Path] = None + review_log: Optional[Path] = None + + +# -------------------------------------------------------------------------- +# reading records +# -------------------------------------------------------------------------- + + +def load_records(path: Path) -> List[Dict[str, Any]]: + """Read an ``audit.jsonl`` or a manifest ``.tsv`` into one common shape.""" + + source = Path(path).expanduser() + if not source.is_file(): + raise FileNotFoundError(str(source)) + suffix = source.suffix.casefold() + if suffix in (".jsonl", ".json", ".ndjson"): + return manifest_module.read_audit(source) + if suffix == ".tsv": + return [_from_manifest_row(row) for row in manifest_module.read_manifest(source)] + raise ValueError( + "undo reads an audit .jsonl or a manifest .tsv, not {0}".format(source.name) + ) + + +def _from_manifest_row(row: Dict[str, str]) -> Dict[str, Any]: + """A manifest row wearing the audit record's field names.""" + + target = row.get("target_path") or "" + try: + size = int(row.get("size_bytes") or 0) + except ValueError: + size = 0 + return { + "timestamp": row.get("moved_at", ""), + "action_id": row.get("action_id", ""), + "kind": row.get("kind") or ("move" if target else ""), + "source": row.get("original_path", ""), + "destination": target, + "status": row.get("status", ""), + "size_bytes": size, + "detail": row.get("reason", ""), + "origin": "manifest", + } + + +# -------------------------------------------------------------------------- +# planning the undo +# -------------------------------------------------------------------------- + + +def plan_undo( + records: Sequence[Dict[str, Any]], + *, + root: Path, + home: Optional[Path] = None, + lang: str = "en", +) -> List[UndoStep]: + """Reverse order, and one step per record so nothing disappears silently.""" + + steps: List[UndoStep] = [] + for record in reversed(list(records)): + kind = str(record.get("kind") or "") + status = str(record.get("status") or "") + action_id = str(record.get("action_id") or "") + + if kind == "trash" or status in ("trashed", "TRASHED"): + steps.append( + UndoStep(record, kind or "trash", "skipped", False, text("trashed", lang), action_id) + ) + continue + if kind == "delete" or status in ("deleted", "DELETED"): + steps.append( + UndoStep(record, kind or "delete", "skipped", False, text("deleted", lang), action_id) + ) + continue + if kind and kind != "move": + steps.append( + UndoStep(record, kind, "skipped", False, text("not_moved", lang, status=status), action_id) + ) + continue + if status not in UNDOABLE_STATUSES: + steps.append( + UndoStep(record, "move", "skipped", False, text("not_moved", lang, status=status), action_id) + ) + continue + + original_text = str(record.get("source") or "") + current_text = str(record.get("destination") or "") + if not original_text or not current_text: + steps.append( + UndoStep(record, "move", "skipped", False, text("no_paths", lang), action_id) + ) + continue + + original = paths.expand_portable(original_text, home) + current = paths.expand_portable(current_text, home) + try: + size = int(record.get("size_bytes") or 0) + except (TypeError, ValueError): + size = 0 + steps.append( + UndoStep( + record, + "move", + "ready", + True, + "", + action_id, + original=original, + current=current, + size_bytes=size, + ) + ) + return steps + + +# -------------------------------------------------------------------------- +# running it +# -------------------------------------------------------------------------- + + +def run_undo( + steps: Sequence[UndoStep], + *, + dry_run: bool = False, + now: Optional[datetime] = None, + root: Optional[Path] = None, + home: Optional[Path] = None, + config: Any = None, + lang: Optional[str] = None, + managed: Optional[Path] = None, + transient_tag: Optional[str] = None, +) -> UndoReport: + """Move each ready step back, then write its own manifest, audit and log.""" + + from . import config as config_module + + moment = now or datetime.now().astimezone() + run_id = manifest_module.timestamp_text(moment) + moved_at = moment.isoformat(timespec="seconds") + speech = lang or (getattr(config, "lang", None) or "en") + home_dir = Path(home) if home is not None else Path.home() + + if managed is None and config is not None: + managed = config_module.managed_dir(config) + if transient_tag is None and config is not None: + transient_tag = config_module.transient_tag(config) + + report = UndoReport(run_id=run_id, dry_run=dry_run) + rows: List[Dict[str, Any]] = [] + + for step in steps: + status, detail = _restore_one(step, root=root, dry_run=dry_run, lang=speech) + if status == "restored" and not dry_run and transient_tag and step.original: + try: + tags_module.remove_tag(step.original, transient_tag) + except Exception as error: # noqa: BLE001 - tagging never stops an undo + print("tag_warning: {0}\t{1}".format(step.original, error), file=sys.stderr) + + record = { + "timestamp": moved_at, + "run_id": run_id, + "action_id": step.action_id, + "subject_id": step.record.get("subject_id"), + "kind": "move", + "source": paths.portable(step.current, home_dir) if step.current else None, + "destination": paths.portable(step.original, home_dir) if step.original else None, + "status": "moved" if status == "restored" else status, + "detail": detail, + "size_bytes": step.size_bytes, + "manifest": "", + "tagged": False, + "trash_backend": "", + "undo": True, + } + report.results.append(record) + + if step.original and step.current: + rows.append( + { + "moved_at": moved_at, + "status": "RESTORED" if status == "restored" else status.upper(), + "original_path": paths.portable(step.current, home_dir), + "target_path": paths.portable(step.original, home_dir), + "size_bytes": step.size_bytes, + "reason": detail, + "restore_method": 'mv "{0}" "{1}"'.format( + paths.portable(step.original, home_dir), + paths.portable(step.current, home_dir), + ), + "kind": "move", + "action_id": step.action_id, + } + ) + + counts: Dict[str, int] = {"total": len(report.results)} + for record in report.results: + key = str(record.get("status")) + counts[key] = counts.get(key, 0) + 1 + report.counts = counts + + restored = counts.get("moved", 0) + if not dry_run and managed is not None and rows: + managed = Path(managed) + managed.mkdir(parents=True, exist_ok=True) + root_name = Path(root).name if root is not None else managed.parent.name + manifest_file = manifest_module.manifest_path(managed, root_name, "undo", moment) + manifest_module.write_manifest(manifest_file, rows) + report.manifest = manifest_file + manifest_text = paths.portable(manifest_file, home_dir) + for record in report.results: + record["manifest"] = manifest_text + report.audit = manifest_module.append_audit( + report.results, manifest_module.audit_path(managed) + ) + if restored: + report.review_log = manifest_module.append_review_log( + managed, + speech, + moment.date(), + [text("log", speech, count=restored, manifest=manifest_text)], + name=config_module.review_log_name(config) if config is not None else None, + ) + + return report + + +def _restore_one( + step: UndoStep, *, root: Optional[Path], dry_run: bool, lang: str +) -> Tuple[str, str]: + if not step.ok: + return step.status, step.detail + original = step.original + current = step.current + if original is None or current is None: + return "skipped", text("no_paths", lang) + + if os.path.lexists(str(original)): + return "skipped", text("occupied", lang) + if not os.path.lexists(str(current)): + return "skipped", text("missing", lang) + + if root is not None: + base = Path(root) + if not paths.inside_root(current, base) or not paths.inside_root(original.parent, base): + return "refused", text("outside", lang) + if paths.has_symlink_component(current, base) or paths.has_symlink_component( + original.parent, base + ): + return "refused", text("symlink", lang) + + if dry_run: + return "dry-run", text("dry_run", lang) + + try: + original.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(current), str(original)) + except (OSError, shutil.Error) as error: + return "failed", text("error", lang, error=error) + return "restored", text("restored", lang) + + +# -------------------------------------------------------------------------- +# undo subcommand +# -------------------------------------------------------------------------- + + +def infer_root(record_path: Path, steps: Sequence[UndoStep]) -> Optional[Path]: + """The folder these records belong to: the managed directory's parent.""" + + candidate = paths.realpath(Path(record_path)).parent.parent + for step in steps: + if step.original is not None and paths.inside_root(step.original.parent, candidate): + return candidate + originals = [str(step.original.parent) for step in steps if step.original is not None] + if not originals: + return candidate if candidate.is_dir() else None + try: + common = Path(os.path.commonpath(originals)) + except ValueError: + return None + return common if common.is_dir() else None + + +def run(args: Any) -> int: + """``carl-file-organizer undo [--dry-run]``.""" + + from . import config as config_module + + record_path = Path(args.record).expanduser() + records = load_records(record_path) + if not records: + print(text("no_records", "en", path=record_path), file=sys.stderr) + return 2 + + steps = plan_undo(records, root=Path("/"), home=None) + root = infer_root(record_path, steps) + if root is None: + print(text("unknown_root", "en", path=record_path), file=sys.stderr) + return 2 + + config = config_module.resolve_config(root) + lang = config.lang + steps = plan_undo(records, root=root, home=None, lang=lang) + + report = run_undo( + steps, + dry_run=bool(args.dry_run), + root=root, + config=config, + lang=lang, + managed=config_module.managed_dir(config), + ) + + for record in report.results: + print( + "{0}\t{1}\t{2}".format( + record.get("status"), record.get("source") or "", record.get("detail") + ) + ) + print( + text( + "summary", + lang, + total=report.counts.get("total", 0), + restored=report.counts.get("moved", 0) + report.counts.get("dry-run", 0), + skipped=report.counts.get("skipped", 0) + report.counts.get("refused", 0), + failed=report.counts.get("failed", 0), + ) + ) + if report.manifest: + print("manifest: {0}".format(report.manifest)) + return 2 if report.counts.get("failed", 0) else 0 diff --git a/skills/ops/carl-file-organizer/scripts/demo_fixture.py b/skills/ops/carl-file-organizer/scripts/demo_fixture.py new file mode 100644 index 0000000..0744353 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/demo_fixture.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Build a fake, throwaway "Downloads"-like directory for CI smoke tests, +manual command walkthroughs, and documentation screenshots. + +Zero third-party dependencies. Every path and filename here is invented; +nothing about it should ever resemble a real user's machine or files. + +Usage (paths are relative to this skill folder, and the target is yours to pick): + python3 scripts/demo_fixture.py [--now ISO] [--partitions zh|en] + python3 scripts/organize.py plan --lang zh + +Nothing outside is written, and is created if missing. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# --------------------------------------------------------------------------- +# time helpers +# --------------------------------------------------------------------------- + + +def _parse_now(value: str | None) -> datetime: + if value is None: + return datetime.now(timezone.utc) + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +_UTIME_SUPPORTS_SYMLINKS = os.utime in os.supports_follow_symlinks + + +def set_mtime(path: Path, when: datetime, *, is_symlink: bool = False) -> None: + ts = when.timestamp() + if is_symlink: + if _UTIME_SUPPORTS_SYMLINKS: + os.utime(path, (ts, ts), follow_symlinks=False) + # else: leave the symlink's own mtime alone (platform can't set it). + return + os.utime(path, (ts, ts)) + + +# --------------------------------------------------------------------------- +# small content writers +# --------------------------------------------------------------------------- + + +def write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def write_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def write_repeated_bytes(path: Path, size: int, pattern: bytes = b"GN-demo-") -> None: + """Write `size` real (non-sparse) bytes, cheaply, without pulling in any + third-party dependency.""" + path.parent.mkdir(parents=True, exist_ok=True) + chunk = (pattern * (65536 // len(pattern) + 1))[:65536] + with path.open("wb") as fh: + written = 0 + while written < size: + take = min(len(chunk), size - written) + fh.write(chunk[:take]) + written += take + + +def write_sparse_file(path: Path, size: int) -> None: + """Create a file that *reports* `size` bytes but only consumes disk + space for the one block actually written (a sparse file).""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("wb") as fh: + if size > 0: + fh.seek(size - 1) + fh.write(b"\0") + else: + fh.truncate(0) + + +def make_symlink(link_path: Path, target_name: str) -> None: + link_path.parent.mkdir(parents=True, exist_ok=True) + if link_path.exists() or link_path.is_symlink(): + link_path.unlink() + link_path.symlink_to(target_name) + + +# --------------------------------------------------------------------------- +# fixture assembly +# --------------------------------------------------------------------------- + +PARTITION_NAMES = { + "zh": { + "inbox": "00_收件箱", + "library_pdf": "20_知识库/文档资料/PDF", + }, + "en": { + "inbox": "00_Inbox", + "library_pdf": "20_Library/Documents/PDF", + }, +} + + +def build(target: Path, now: datetime, partitions: str) -> None: + target.mkdir(parents=True, exist_ok=True) + ago = lambda **kw: now - timedelta(**kw) # noqa: E731 + + # -- settled documents (past the document aging window) ----------------- + write_text(target / "quarterly-report.pdf", "%PDF-1.4\n% demo placeholder\n") + set_mtime(target / "quarterly-report.pdf", ago(days=5)) + + docs = { + "notes.md": "# Demo notes\n\nJust placeholder text.\n", + "slides.pptx": "PK\x03\x04 demo placeholder pptx bytes\n", + "data.csv": "col_a,col_b\n1,2\n3,4\n", + "readme.txt": "This is a placeholder readme for the demo fixture.\n", + } + for name, content in docs.items(): + write_text(target / name, content) + set_mtime(target / name, ago(days=3)) + + # -- fresh, still inside the aging window -------------------------------- + write_text(target / "fresh-draft.md", "# Draft\n\nStill warm.\n") + set_mtime(target / "fresh-draft.md", ago(hours=1)) + + write_bytes(target / "just-downloaded.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "just-downloaded.zip", ago(hours=2)) + + # -- image / video / web page (settled) ---------------------------------- + media = { + "photo.png": b"\x89PNG\r\n\x1a\n" + b"demo-placeholder-bytes", + "screenshot.jpg": b"\xff\xd8\xff\xe0" + b"demo-placeholder-bytes", + "clip.mp4": b"\x00\x00\x00\x18ftypmp42" + b"demo-placeholder-bytes", + "page.html": b"demo

    placeholder

    ", + } + for name, data in media.items(): + write_bytes(target / name, data) + set_mtime(target / name, ago(days=3)) + + # -- cold archives / installers (past the 7-day archive window) --------- + write_bytes(target / "old-tool.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "old-tool.zip", ago(days=10)) + + write_repeated_bytes(target / "installer.dmg", 2 * 1024 * 1024) + set_mtime(target / "installer.dmg", ago(days=12)) + + # -- paired archive + extracted directory -------------------------------- + write_bytes(target / "Archive.zip", b"PK\x05\x06" + b"\x00" * 18) + set_mtime(target / "Archive.zip", ago(days=3)) + + write_text(target / "Archive" / "readme-inside.txt", "Extracted placeholder file one.\n") + write_text(target / "Archive" / "data-inside.txt", "Extracted placeholder file two.\n") + for p in (target / "Archive" / "readme-inside.txt", target / "Archive" / "data-inside.txt"): + set_mtime(p, ago(days=3)) + set_mtime(target / "Archive", ago(days=3)) + + # -- duplicates ----------------------------------------------------------- + same_content = "Invoice placeholder, both copies are byte-identical.\n" + write_text(target / "invoice.pdf", same_content) + write_text(target / "invoice (1).pdf", same_content) + set_mtime(target / "invoice.pdf", ago(days=3)) + set_mtime(target / "invoice (1).pdf", ago(days=3)) + + write_bytes(target / "export.png", b"\x89PNG\r\n\x1a\n" + b"export-A-placeholder") + write_bytes(target / "export_1.png", b"\x89PNG\r\n\x1a\n" + b"export-B-different-bytes") + set_mtime(target / "export.png", ago(days=3)) + set_mtime(target / "export_1.png", ago(days=3)) + + # -- sensitive naming (content is placeholder, never a real credential) -- + write_text(target / "api-credential.txt", "PLACEHOLDER_NOT_A_REAL_CREDENTIAL\n") + write_text( + target / "server.pem", + "-----BEGIN PLACEHOLDER-----\nthis is not a real certificate\n-----END PLACEHOLDER-----\n", + ) + write_text(target / "prod.env", "EXAMPLE_KEY=placeholder-not-real\n") + for name in ("api-credential.txt", "server.pem", "prod.env"): + set_mtime(target / name, ago(days=2)) + + # -- a project-shaped directory ------------------------------------------- + proj = target / "my-project" + write_text(proj / "package.json", '{"name": "demo-project", "version": "0.0.0"}\n') + write_text(proj / "src" / "index.js", "console.log('demo');\n") + write_text(proj / "node_modules" / "left-pad" / "index.js", "module.exports = function leftPad() {};\n") + write_text(proj / ".git" / "HEAD", "ref: refs/heads/main\n") + for p in ( + proj / "package.json", + proj / "src" / "index.js", + proj / "node_modules" / "left-pad" / "index.js", + proj / ".git" / "HEAD", + proj / "src", + proj / "node_modules" / "left-pad", + proj / "node_modules", + proj / ".git", + proj, + ): + set_mtime(p, ago(days=2)) + + # -- a worktree-shaped directory (.git is a *file*, not a directory) ----- + worktree = target / "my-project-worktree-fix" + write_text(worktree / ".git", "gitdir: /tmp/example/.git/worktrees/fix\n") + set_mtime(worktree / ".git", ago(days=2)) + set_mtime(worktree, ago(days=2)) + + # -- one big (sparse) file ------------------------------------------------- + write_sparse_file(target / "big-video.mov", 600 * 1024 * 1024) + set_mtime(target / "big-video.mov", ago(days=10)) + + # -- unknown extension / no extension --------------------------------------- + write_bytes(target / "mystery.blob", b"\x00\x01\x02demo-placeholder-bytes") + set_mtime(target / "mystery.blob", ago(days=5)) + + write_text(target / "README", "Placeholder file with no extension.\n") + set_mtime(target / "README", ago(days=5)) + + # -- hidden file + symlink --------------------------------------------------- + write_bytes(target / ".DS_Store", b"\x00\x00\x00\x01Bud1demo-placeholder") + set_mtime(target / ".DS_Store", ago(days=1)) + + make_symlink(target / "link-to-notes.md", "notes.md") + set_mtime(target / "link-to-notes.md", ago(days=3), is_symlink=True) + + # -- already-existing partition directories (established structure) ------- + names = PARTITION_NAMES[partitions] + inbox = target / names["inbox"] + inbox.mkdir(parents=True, exist_ok=True) + set_mtime(inbox, ago(days=30)) + + pdf_dir = target / names["library_pdf"] + write_text(pdf_dir / "quarterly-report.pdf", "%PDF-1.4\n% already-filed placeholder copy\n") + set_mtime(pdf_dir / "quarterly-report.pdf", ago(days=20)) + # walk the partition chain and set an established mtime on each level + parts = names["library_pdf"].split("/") + cur = target + for part in parts: + cur = cur / part + set_mtime(cur, ago(days=30)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("target_dir", type=Path, help="directory to fill with fixture files") + parser.add_argument("--now", type=str, default=None, help="ISO 8601 timestamp to treat as 'now'") + parser.add_argument( + "--partitions", + choices=("zh", "en"), + default="zh", + help="language for the pre-existing partition directory names (default: zh)", + ) + args = parser.parse_args(argv) + + now = _parse_now(args.now) + build(args.target_dir.resolve(), now, args.partitions) + print(f"Demo fixture written to: {args.target_dir.resolve()}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/hygiene.sh b/skills/ops/carl-file-organizer/scripts/hygiene.sh new file mode 100755 index 0000000..fb718ec --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/hygiene.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Repository hygiene: nothing that ships should carry one person's machine, one +# person's private project names, or borrowed vocabulary. +# +# The patterns live in this one file, and this file excludes itself from every +# search. Spelling them inline in the CI workflow made the workflow match +# itself, so the check failed on a clean tree. +# +# Two scopes, because the four rules are not the same kind of rule: +# +# repo the whole collection. A private path or an unattributed source is a +# publishing problem wherever it sits, so every skill is searched. +# skill this skill's own folder. Traffic-light wording and the product names +# this tool grew out of are house style here; another skill in the +# collection has its own history and its own vocabulary. +# +# tests/ is left out of the private-path rule on purpose. Nothing under tests/ +# is installed or published, and a test that proves a path gets redacted has to +# spell out the path being redacted. This rule protects what people receive. +# +# Usage: bash skills/ops/carl-file-organizer/scripts/hygiene.sh +# (exit 0 clean, exit 1 with the offending lines; run it from anywhere) + +set -uo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +skill_dir="$(cd "$here/.." && pwd)" +repo_root="$(cd "$skill_dir/../../.." && pwd)" +cd "$repo_root" + +skill_rel="${skill_dir#"$repo_root"/}" + +COMMON=(-r -n -E + --exclude-dir=.git + --exclude-dir=fixtures + --exclude-dir=__pycache__ + --exclude-dir=node_modules + --exclude-dir=.venv + --exclude-dir=dist + --exclude-dir=build + --exclude-dir=.gstack + --exclude=hygiene.sh +) + +failed=0 + +report() { + echo "::error::$1" + failed=1 +} + +# 1. No absolute home directories and no one machine's account name, anywhere +# that ships. See the note above about tests/. +# +# The path pattern wants /Users/ followed by something that looks like a real +# account, so code and prose that handle the prefix itself -- a redaction +# regex, a startswith check, a sentence listing the shapes -- read as what +# they are. Two rules make that work: an account name starts with a letter +# or digit, and it is longer than one character. Which makes /Users/x the +# house placeholder: use it when a document has to show the shape. +if grep "${COMMON[@]}" --exclude-dir=tests '/Users/[A-Za-z0-9_-]{2,}|carl2077|\$CODEX_HOME' .; then + report "private path pattern found; write \$HOME/... instead, or /Users/x when a document has to show the shape" +fi + +# 2. The acknowledgements table in the READMEs is the only place a source is +# named, and that holds for every skill in the collection. +matches=$(grep "${COMMON[@]}" -l '卡兹克|Khazix|storage-analyzer' . 2>/dev/null || true) +if [ -n "$matches" ]; then + outside=$(printf '%s\n' "$matches" | grep -vE '(^|/)README(\.en)?\.md$' || true) + if [ -n "$outside" ]; then + report "source named outside the README acknowledgements: $outside" + fi +fi + +# 3. Borrowed traffic-light vocabulary, in this skill's own writing. +if grep "${COMMON[@]}" '红灯|黄灯|绿灯' "$skill_rel"; then + report "traffic-light terminology found" +fi + +# 4. Private product and service names from the notes this tool grew out of, +# in this skill's own writing. +if grep "${COMMON[@]}" -i 'tikhub|deepseek|goodcase|订阅链接' "$skill_rel"; then + report "private product name found; use an invented name instead" +fi + +if [ "$failed" -eq 0 ]; then + echo "hygiene: clean" +fi +exit "$failed" diff --git a/skills/ops/carl-file-organizer/scripts/organize.py b/skills/ops/carl-file-organizer/scripts/organize.py new file mode 100755 index 0000000..ee3ed74 --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/organize.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Entry point for the file organizer: tidy one folder, with approval in between. + +The engine is the ``carl_file_organizer`` package sitting next to this file. +Nothing is installed, nothing is on PYTHONPATH, so this script puts its own +directory at the front of ``sys.path`` and hands over to the package CLI. That +is the whole job; every flag and every subcommand lives in +``carl_file_organizer/cli.py``. + +Three commands do the job (any Python 3.9 or newer; there are no dependencies): + + python3 organize.py scan ~/Downloads --lang zh + python3 organize.py report ~/Downloads/00_下载目录管理 + python3 organize.py apply --dry-run + +``scan`` reads the folder and, unless you pass ``--no-storage``, the machine, +and writes plan.json and storage-scan.json into the folder's managed directory. +The Agent then writes notes.json and analysis.json next to them. ``report`` +turns whatever that directory holds into one page: the cleanup and the moves in +the same three colour bands, the after picture below them, and one button that +exports one decisions file (or, with ``--serve``, runs the lot straight away). +``apply`` acts on that file, the moves first and the cleanup second. + +The same work split into steps, for when you want one piece on its own: + + python3 organize.py plan ~/Downloads --lang zh + python3 organize.py build --notes notes.json --report + python3 organize.py review --serve + python3 organize.py storage-report [--serve] + python3 organize.py dispose --dry-run + python3 organize.py undo + python3 organize.py status ~/Downloads + python3 organize.py clear-tags ~/Downloads + python3 organize.py --version + +``scan``, ``report``, ``plan``, ``status`` and ``storage-report`` only read. +``build`` only rewrites the plan's prose. ``apply`` and ``dispose`` are the two +that touch anything, and they touch only the ids you approved. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +if HERE not in sys.path: + sys.path.insert(0, HERE) + +from carl_file_organizer.cli import main # noqa: E402 - the path shim has to run first + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/ops/carl-file-organizer/scripts/storage_scan.py b/skills/ops/carl-file-organizer/scripts/storage_scan.py new file mode 100644 index 0000000..beb38ab --- /dev/null +++ b/skills/ops/carl-file-organizer/scripts/storage_scan.py @@ -0,0 +1,1188 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""整机只读盘点扫描器（macOS + Windows）。 + +它只做两件事：给这台机器拍一张体检照，再把最占地方的那些目录按组量出来， +写成一份 storage-scan.json 交给 Agent 去读。它不移动、不删除、不改权限， +连一个文件的内容都不打开，只看 stat 里的大小和时间。 + +用法： + + python3 storage_scan.py [--out storage-scan.json] [--budget-seconds 60] + [--top-files 30] [--home PATH] + +尺寸口径：目录体积等于其下所有文件 st_size 之和，也就是逻辑大小。跟 du 的 +块大小口径会有出入，稀疏文件和 APFS 压缩过的文件会偏大。要的是量级判断， +不是审计账。 + +预算口径：--budget-seconds 是整轮硬上限，按组、按条目层层分摊。到点就停， +量到一半的条目带 partial=True，顶层 budget_hit=True。宁可标不准，不报假数。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time + +SCHEMA = "storage-scan/1" + +DEFAULT_BUDGET_SECONDS = 60.0 +DEFAULT_TOP_FILES = 30 +DEFAULT_MIN_FILE_MB = 200 # 进 top_files 的门槛 +DEFAULT_MIN_SIZE_MB = 50 # 进 groups 的门槛，比它小的条目不进输出 +SUBPROCESS_TIMEOUT = 5.0 # 每个体检子进程的硬超时 +MAX_ENTRIES_PER_ITEM = 400000 # 单个条目最多遍历的目录项数 +MEMO_MAX_DEPTH = 5 # 只缓存浅层目录体积，避免吃内存 +TIME_CHECK_EVERY = 128 # 每遍历多少项看一次表 + +# 有些组天生就没那么大，用全局门槛会把它们全滤没。安装包尤其明显。 +GROUP_MIN_SIZE_OVERRIDE = { + "downloads_installers": 10 * 1024 * 1024, + "trash": 1024 * 1024, + "recycle_bin": 1024 * 1024, +} + +GB = 1024 ** 3 + +# ---------------------------------------------------------------- 卷的取舍 + +#: macOS 的 df 会把一台机器的全部卷都摊出来：只读的系统卷、Preboot、VM、 +#: Recovery、每个挂上来的磁盘镜像。它们跟人要腾的空间没关系，列出来只会把 +#: 真正该看的那一行淹掉。所以磁盘列表只留两种卷：装着用户数据的那一个，和 +#: 真外接上来的盘。其余的收进 system_volumes 里存个底，不进正文。 +PRIMARY_DATA_MOUNT = "/System/Volumes/Data" +PRIMARY_DATA_NAME = "Macintosh HD 数据卷" +EXTERNAL_MOUNT_PREFIX = "/Volumes/" + +#: 名字长这样的都是系统自己的卷或者本地快照，哪怕挂在 /Volumes 下也不算外接盘。 +SYSTEM_VOLUME_NAMES = ( + "com.apple.timemachine", + "recovery", + "preboot", + "vm", + "update", + "xarts", + "iscpreboot", + "hardware", +) + +#: 比这还小的挂载点不是能放东西的盘，是脚本挂上来的临时镜像或者 RAM 盘。 +#: 真的外接盘没有小于 1 GiB 的，所以这条门槛只会误伤假挂载。 +MIN_EXTERNAL_VOLUME_BYTES = GB + +# ---------------------------------------------------------------- 分类词表 + +REGENERABLE_DIR_NAMES = ( + "node_modules", ".next", "dist", "build", "target", "coverage", + "__pycache__", ".turbo", ".nuxt", ".svelte-kit", ".parcel-cache", +) + +INSTALLER_EXTS = ( + ".dmg", ".pkg", ".zip", ".iso", ".exe", ".msi", ".msix", ".appimage", + ".deb", ".rpm", ".xip", +) + +MEDIA_EXTS = ( + ".mp4", ".mov", ".m4v", ".mkv", ".avi", ".webm", ".mp3", ".wav", ".aiff", + ".flac", ".png", ".jpg", ".jpeg", ".heic", ".gif", ".tiff", ".psd", ".ai", + ".sketch", ".fig", ".key", ".pptx", +) + +# 命中即 dev_cache。写成路径段序列，小写、正斜杠。 +DEV_CACHE_PATHS = ( + ".npm", ".pnpm-store", ".yarn", ".cargo", ".gradle", ".m2", ".nuget", + ".cache", ".bun", ".deno", "go/pkg", "go-build", + "library/caches/pip", "library/caches/uv", "library/caches/ms-playwright", + "library/caches/homebrew", "library/caches/com.apple.dt.xcode", + "library/developer/xcode/deriveddata", + "library/developer/xcode/archives", + "library/developer/xcode/ios devicesupport", + "library/developer/coresimulator", + "appdata/local/pip/cache", "appdata/local/npm-cache", + "appdata/local/yarn", "appdata/local/nuget", "appdata/local/uv", + "appdata/local/ms-playwright", "appdata/local/temp", + "appdata/local/go-build", "appdata/local/microsoft/vscode", +) + +# 禁刀区。删了要么丢账号、要么丢聊天记录、要么丢一整个库。 +RED_PATHS = ( + "vm_bundles", "virtual machines", "parallels", "hyper-v", + "docker-desktop-data", "wsl", "user data", "profiles", + "library/safari", + "library/application support/google/chrome", + "library/application support/chromium", + "library/application support/firefox", + "library/application support/bravesoftware", + "library/application support/microsoft edge", + "library/keychains", "library/messages", "library/mail", + "library/mobile documents", "library/calendars", "library/contacts", + "library/containers/com.apple.mail", + "library/containers/com.apple.messages", + "library/group containers/group.com.apple.notes", + "com.apple.photos", ".git", ".obsidian", "onedrive", "dropbox", + "icloud drive", "appdata/roaming/microsoft/crypto", + "appdata/local/microsoft/edge/user data", + "appdata/local/google/chrome/user data", + "appdata/roaming/mozilla/firefox/profiles", +) + +# 目录名以这些结尾的一律 red。 +RED_SUFFIXES = ( + ".photoslibrary", ".utm", ".pvm", ".vmwarevm", ".sparsebundle", + ".musiclibrary", ".tvlibrary", ".imovielibrary", ".fcpbundle", +) + +# 遍历时撞见这些目录名，就说明条目里面埋着禁刀区，整体不能当删除单元。 +# 名字必须足够特异：messages、mail、profiles、.git 这种词随便哪个代码库里 +# 都有，拿来下沉会把 Downloads、Documents、projects 一起染红，三色就废了。 +RED_DIR_NAMES = frozenset(( + "vm_bundles", "user data", "virtual machines", "hyper-v", + "docker-desktop-data", +)) + +# 这些只作提示，不改颜色：值得 Agent 知道，但不足以判整个条目死刑。 +NOTABLE_DIR_NAMES = frozenset(( + ".git", ".obsidian", "keychains", "messages", "mail", "profiles", +)) + +CACHE_SEGMENTS = ("caches", "cache", "temp", "tmp", ".trash", "$recycle.bin") + +APP_DATA_SEGMENTS = ("application support", "containers", "group containers", + "appdata") + +MEDIA_SEGMENTS = ("movies", "music", "pictures") + +WORKSPACE_CANDIDATES = ( + "projects", "agent-workbench", "workspace", "work", "worktrees", + "dev", "src", "code", "repos", "Developer", +) + +# ---------------------------------------------------------------- 小工具 + + +def human(num_bytes): + """字节数转成人能念出来的字符串。""" + if num_bytes is None: + return "?" + value = float(num_bytes) + for unit in ("B", "KB", "MB", "GB", "TB", "PB"): + if abs(value) < 1024.0 or unit == "PB": + if unit == "B": + return "%d B" % int(value) + return "%.1f %s" % (value, unit) + value /= 1024.0 + return "%.1f PB" % value + + +def iso_time(timestamp): + try: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + except Exception: + return None + + +def run_cmd(args, timeout=SUBPROCESS_TIMEOUT): + """跑一个只读命令。超时、没这个命令、报错，都当没采到。""" + try: + proc = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=timeout, + ) + except Exception: + return "" + try: + return proc.stdout.decode("utf-8", "replace") + except Exception: + return "" + + +def path_segments(path): + """拆成小写路径段，正反斜杠都认。""" + return [seg for seg in path.replace("\\", "/").lower().split("/") if seg] + + +def haystack(path): + """给段序列匹配用的干草堆，两头都带斜杠。""" + return "/" + "/".join(path_segments(path)) + "/" + + +def matches_any(path, needles): + """路径里是否出现过这些段序列之一。按段匹配，.git 不会命中 .gitignore。""" + hay = haystack(path) + for needle in needles: + if ("/" + needle.strip("/") + "/") in hay: + return True + return False + + +def _windows_drive_letters(): + """列出 Windows 上所有盘符。测试里会把这个函数整体换掉。""" + try: + import ctypes + + bitmask = ctypes.windll.kernel32.GetLogicalDrives() # type: ignore[attr-defined] + except Exception: + return [] + return [chr(ord("A") + i) for i in range(26) if bitmask >> i & 1] + + +# ---------------------------------------------------------------- 扫描器 + + +class StorageScanner(object): + """只读扫描器，一个实例负责一轮扫描。""" + + def __init__( + self, + home=None, + platform_name=None, + budget_seconds=DEFAULT_BUDGET_SECONDS, + top_files=DEFAULT_TOP_FILES, + min_file_bytes=DEFAULT_MIN_FILE_MB * 1024 * 1024, + min_size_bytes=DEFAULT_MIN_SIZE_MB * 1024 * 1024, + with_system=True, + ): + self.platform = platform_name or sys.platform + self.is_windows = self.platform.startswith("win") + default_home = os.environ.get( + "USERPROFILE" if self.is_windows else "HOME" + ) or os.path.expanduser("~") + self.home = os.path.abspath(home or default_home) + self.budget_seconds = float(budget_seconds) + self.top_files_limit = int(top_files) + self.min_file_bytes = int(min_file_bytes) + self.min_size_bytes = int(min_size_bytes) + self.with_system = with_system + + self.started_monotonic = time.monotonic() + self.deadline = self.started_monotonic + self.budget_seconds + self.budget_hit = False + + self._memo = {} + self._memo_red = {} + self._denied = {} + self._big_files = {} + self._entries_this_item = 0 + self._red_inside = [] + self._notable_inside = [] + self._tick = 0 + self._abort = False + self.dropped_small = 0 + self._system_volumes = [] + + # -------------------------------------------------- 路径 + + @property + def home_token(self): + return "%USERPROFILE%" if self.is_windows else "$HOME" + + @property + def sep(self): + return "\\" if self.is_windows else "/" + + def join(self, *parts): + return os.path.join(*parts) + + def portable(self, path): + """绝对路径写成 $HOME/... 或 %USERPROFILE%\\...，不落用户名。""" + try: + rel = os.path.relpath(path, self.home) + except ValueError: + return path + if rel == os.curdir: + return self.home_token + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return path + return self.home_token + self.sep + rel.replace(os.sep, self.sep) + + def scoped(self, path): + """分类只看 home 以下的那一段。 + + 词表里的每一条都是从 home 数起的：``library/caches/pip``、 + ``appdata/local/temp``、``.trash``。可 home 上面还有别的目录名，而它 + 们不该说话——一个 home 造在 ``/tmp`` 底下，整台机器就会因为路径里有 + 一个 ``tmp`` 段被判成缓存，三色一起塌成 green。所以匹配前先把 home + 之上的部分减掉。home 以外的路径（外接盘、系统目录）原样返回，那里 + 本来就没有 home 相对口径可言。 + """ + try: + rel = os.path.relpath(path, self.home) + except ValueError: + return path + if rel == os.curdir: + return "" + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return path + return rel.replace(os.sep, "/").replace("\\", "/") + + # -------------------------------------------------- 预算 + + def time_left(self): + return self.deadline - time.monotonic() + + def out_of_time(self): + if self.time_left() <= 0: + self.budget_hit = True + return True + return False + + def slice_deadline(self, share_of_remaining): + """给一段工作切一块时间，最少 0.3 秒。""" + remaining = max(self.time_left(), 0.0) + return time.monotonic() + max(remaining * share_of_remaining, 0.3) + + # -------------------------------------------------- 遍历 + + def note_denied(self, path, reason, counted_bytes=0): + if path in self._denied: + return + self._denied[path] = { + "path": path, + "path_portable": self.portable(path), + "reason": reason, + "counted_bytes": counted_bytes, + } + + def note_big_file(self, path, size, mtime): + if size < self.min_file_bytes or path in self._big_files: + return + self._big_files[path] = { + "path": path, + "path_portable": self.portable(path), + "name": os.path.basename(path), + "size_bytes": size, + "size_human": human(size), + "mtime": mtime, + "kind": self.guess_kind(path, is_dir=False), + } + + @staticmethod + def dir_marker(name): + """这个目录名是禁刀区标志物（red），还是只值得一提（note）。""" + low = name.lower() + if low in RED_DIR_NAMES: + return ("red", low) + for suffix in RED_SUFFIXES: + if low.endswith(suffix): + return ("red", low) + if low in NOTABLE_DIR_NAMES: + return ("note", low) + return None + + def walk_size(self, path, deadline, depth=0): + """递归量一个目录，返回 (bytes, partial, marks)。只读，不跟 symlink。 + + marks 是一串 (severity, name)，记下这棵树里埋着什么值得注意的东西。 + """ + memoized = self._memo.get(path) + if memoized is not None: + return memoized, False, self._memo_red.get(path, frozenset()) + + total = 0 + partial = False + marks = set() + try: + iterator = os.scandir(path) + except PermissionError: + self.note_denied(path, "permission_denied") + return 0, True, marks + except OSError as exc: + self.note_denied(path, "os_error:" + exc.__class__.__name__) + return 0, True, marks + + try: + with iterator as entries: + for entry in entries: + # 超时是全栈的：一层喊停，所有祖先层立刻跟着停， + # 否则每一层都要再攒够一轮才轮到看表，越深漂得越远。 + if self._abort: + partial = True + break + self._entries_this_item += 1 + self._tick += 1 + if self._tick % TIME_CHECK_EVERY == 0 and time.monotonic() > deadline: + self._abort = True + partial = True + break + if self._entries_this_item > MAX_ENTRIES_PER_ITEM: + self._abort = True + partial = True + break + try: + if entry.is_symlink(): + continue + if entry.is_dir(follow_symlinks=False): + marker = self.dir_marker(entry.name) + if marker and len(marks) < 12: + marks.add(marker) + sub, sub_partial, sub_marks = self.walk_size( + entry.path, deadline, depth + 1 + ) + total += sub + partial = partial or sub_partial + if sub_marks and len(marks) < 12: + marks.update(sub_marks) + else: + stat = entry.stat(follow_symlinks=False) + total += stat.st_size + if stat.st_size >= self.min_file_bytes: + self.note_big_file( + entry.path, stat.st_size, iso_time(stat.st_mtime) + ) + except PermissionError: + self.note_denied(entry.path, "permission_denied") + partial = True + except OSError: + continue + except PermissionError: + self.note_denied(path, "permission_denied", total) + partial = True + except OSError: + partial = True + + if not partial and depth <= MEMO_MAX_DEPTH: + self._memo[path] = total + self._memo_red[path] = frozenset(marks) + return total, partial, marks + + def measure(self, path, deadline): + """量一个路径：目录递归，文件 stat。返回 (bytes, partial, mtime)。""" + self._entries_this_item = 0 + self._red_inside = [] + self._notable_inside = [] + self._abort = False + try: + stat = os.stat(path) + except PermissionError: + self.note_denied(path, "permission_denied") + return None, True, None + except OSError: + return None, False, None + mtime = iso_time(stat.st_mtime) + if os.path.isdir(path): + size, partial, marks = self.walk_size(path, deadline) + self._red_inside = sorted(n for sev, n in marks if sev == "red") + self._notable_inside = sorted(n for sev, n in marks if sev == "note") + return size, partial, mtime + self.note_big_file(path, stat.st_size, mtime) + return stat.st_size, False, mtime + + # -------------------------------------------------- 分类 + + def guess_kind(self, path, is_dir=True): + inside = self.scoped(path) + segs = path_segments(inside) + name = segs[-1] if segs else "" + + if is_dir and name in REGENERABLE_DIR_NAMES: + return "build_artifact" + if matches_any(inside, DEV_CACHE_PATHS): + return "dev_cache" + if not is_dir: + for ext in INSTALLER_EXTS: + if name.endswith(ext): + return "installer" + for ext in MEDIA_EXTS: + if name.endswith(ext): + return "media" + for seg in CACHE_SEGMENTS: + if seg in segs: + return "cache" + for seg in APP_DATA_SEGMENTS: + if seg in segs: + return "app_data" + for seg in MEDIA_SEGMENTS: + if seg in segs: + return "media" + for workspace in WORKSPACE_CANDIDATES: + if workspace.lower() in segs: + return "project" + if is_dir: + try: + if os.path.isdir(os.path.join(path, ".git")): + return "project" + except OSError: + pass + return "unknown" + + def is_red(self, path): + inside = self.scoped(path) + for seg in path_segments(inside): + for suffix in RED_SUFFIXES: + if seg.endswith(suffix): + return True + if matches_any(inside, RED_PATHS): + return True + # 目录里有 .obsidian 就是知识库主库，别碰。 + try: + if os.path.isdir(os.path.join(path, ".obsidian")): + return True + except OSError: + pass + return False + + def suggest_color(self, path, kind, group, red_inside=()): + """三色只是建议，最终解释权在 Agent 手里，但绿的门槛必须硬。 + + 判定顺序：自己在禁刀区就是 red；纯缓存和可再生产物是 green，哪怕里面 + 混着一个叫 profiles 的目录也照样可再生；剩下的，里面埋着禁刀区的算 + red，其余一律 yellow，交给人看一眼。 + """ + if self.is_red(path): + return "red" + if group in ("trash", "recycle_bin"): + return "green" + if kind in ("cache", "dev_cache", "build_artifact"): + return "green" + if red_inside: + return "red" + return "yellow" + + def restore_hint(self, path, kind, group): + name = (path_segments(path) or [""])[-1] + if name == "node_modules": + return "在项目根目录重跑 npm install / pnpm install / bun install" + if name in (".next", "dist", "build", "target", ".turbo", "coverage"): + return "重跑一次构建或 dev server 就会重新生成" + if name == "__pycache__": + return "下次导入模块时自动重建" + if group in ("trash", "recycle_bin") or name in (".trash", "$recycle.bin"): + return "清空之后不可恢复，先确认里面没有要留的东西" + if kind == "dev_cache": + return "下次装依赖或构建时会重新下载，第一次会慢一点" + if kind == "cache": + return "应用自己会重建，可能第一次启动稍慢" + if kind == "installer": + return "需要时去官网重新下载，老版本安装包一般不必留" + if kind == "app_data": + return "应用的账号、配置或本地数据，删了要重新登录或重新配置" + if kind == "project": + return "有远端仓库的可以重新 clone，没有的就是唯一副本" + if kind == "media": + return "素材本体，删了只能靠备份找回" + return "没有自动恢复途径，动之前先确认别处有没有副本" + + def make_item(self, path, group, size, partial, mtime, is_dir=True): + kind = self.guess_kind(path, is_dir=is_dir) + red_inside = list(self._red_inside) if is_dir else [] + item = { + "name": (path_segments(path) or [path])[-1], + "path": path, + "path_portable": self.portable(path), + "size_bytes": size, + "size_human": human(size), + "kind": kind, + "suggested_color": self.suggest_color(path, kind, group, red_inside), + "restore_hint": self.restore_hint(path, kind, group), + "mtime": mtime, + "group": group, + "partial": bool(partial), + } + if item["suggested_color"] == "red": + if self.is_red(path): + item["red_reason"] = "路径本身属于禁刀区" + elif red_inside: + item["red_reason"] = "里面埋着 " + "、".join(red_inside[:4]) + notable = list(self._notable_inside) if is_dir else [] + if notable: + item["contains_notable"] = notable[:6] + return item + + # -------------------------------------------------- 组 + + def children_of(self, path): + """一层子项，目录和文件都算，symlink 跳过。""" + out = [] + try: + with os.scandir(path) as entries: + for entry in entries: + try: + if entry.is_symlink(): + continue + except OSError: + continue + out.append(entry.path) + except PermissionError: + self.note_denied(path, "permission_denied") + except OSError: + pass + return sorted(out) + + def installers_in(self, path): + """Downloads 顶层的安装包，只看扩展名不进子目录。""" + out = [] + try: + with os.scandir(path) as entries: + for entry in entries: + try: + if entry.is_symlink() or entry.is_dir(follow_symlinks=False): + continue + except OSError: + continue + low = entry.name.lower() + if any(low.endswith(ext) for ext in INSTALLER_EXTS): + out.append(entry.path) + except PermissionError: + self.note_denied(path, "permission_denied") + except OSError: + pass + return sorted(out) + + def regenerable_under(self, roots, max_depth=3): + """工作区下找可再生目录，命中即当叶子，不再往里走。""" + found = [] + for root in roots: + if not os.path.isdir(root): + continue + stack = [(root, 0)] + while stack: + if self.out_of_time(): + return sorted(found) + current, depth = stack.pop() + try: + with os.scandir(current) as entries: + for entry in entries: + try: + if entry.is_symlink(): + continue + if not entry.is_dir(follow_symlinks=False): + continue + except OSError: + continue + if entry.name in REGENERABLE_DIR_NAMES: + found.append(entry.path) + continue + if entry.name.startswith(".") or depth + 1 >= max_depth: + continue + stack.append((entry.path, depth + 1)) + except PermissionError: + self.note_denied(current, "permission_denied") + except OSError: + continue + return sorted(found) + + def group_specs(self): + if self.is_windows: + return self.windows_group_specs(self.home) + return self.macos_group_specs(self.home) + + def macos_group_specs(self, home): + """顺序是有讲究的：先量深层，再量浅层，浅层就能吃到 memo。""" + library = self.join(home, "Library") + return [ + ("dev_caches", [ + self.join(home, ".npm"), + self.join(home, ".pnpm-store"), + self.join(home, ".cache"), + self.join(home, ".cargo"), + self.join(home, ".gradle"), + self.join(home, ".m2"), + self.join(home, ".nuget"), + self.join(home, ".bun"), + self.join(home, "go", "pkg"), + self.join(library, "Caches", "pip"), + self.join(library, "Caches", "uv"), + self.join(library, "Caches", "ms-playwright"), + self.join(library, "Caches", "Homebrew"), + self.join(library, "Caches", "com.apple.dt.Xcode"), + ]), + ("downloads_installers", ("installers", self.join(home, "Downloads"))), + ("trash", [self.join(home, ".Trash")]), + ("developer", [ + self.join(library, "Developer", "Xcode", "DerivedData"), + self.join(library, "Developer", "Xcode", "Archives"), + self.join(library, "Developer", "Xcode", "iOS DeviceSupport"), + self.join(library, "Developer", "CoreSimulator"), + ]), + ("caches", ("children", self.join(library, "Caches"))), + ("app_support", ("children", self.join(library, "Application Support"))), + ("containers", ("children", self.join(library, "Containers"))), + ("group_containers", ("children", self.join(library, "Group Containers"))), + ("logs", ("children", self.join(library, "Logs"))), + ("projects_regenerable", ("regenerable", [ + self.join(home, name) for name in WORKSPACE_CANDIDATES + ])), + ("library", ("children", library)), + ("home_top", ("children", home)), + ] + + def windows_group_specs(self, home): + local = self.join(home, "AppData", "Local") + roaming = self.join(home, "AppData", "Roaming") + system_drive = os.environ.get("SystemDrive", "C:") + return [ + ("dev_caches", [ + self.join(home, ".cache"), + self.join(home, ".npm"), + self.join(home, ".gradle"), + self.join(home, ".m2"), + self.join(home, ".nuget"), + self.join(home, ".cargo"), + ]), + ("downloads_installers", ("installers", self.join(home, "Downloads"))), + ("recycle_bin", [system_drive + "\\$Recycle.Bin"]), + ("appdata_local", ("children_plus", (local, [ + self.join(local, "Temp"), + self.join(local, "pip", "Cache"), + self.join(local, "npm-cache"), + self.join(local, "Yarn"), + self.join(local, "NuGet"), + self.join(local, "uv"), + self.join(local, "ms-playwright"), + self.join(local, "Google", "Chrome", "User Data"), + self.join(local, "Microsoft", "Edge", "User Data"), + self.join(local, "Mozilla", "Firefox", "Profiles"), + ]))), + ("appdata_roaming", ("children", roaming)), + ("projects_regenerable", ("regenerable", [ + self.join(home, name) for name in WORKSPACE_CANDIDATES + ])), + ("user_top", ("children", home)), + ] + + def resolve_spec(self, spec): + """把组的声明解析成一串待测路径。""" + if isinstance(spec, list): + return [path for path in spec if os.path.exists(path)] + mode, payload = spec + if mode == "children": + return self.children_of(payload) + if mode == "children_plus": + base, extras = payload + paths = self.children_of(base) + known = set(paths) + for extra in extras: + if extra not in known and os.path.exists(extra): + paths.append(extra) + return sorted(paths) + if mode == "installers": + return self.installers_in(payload) + if mode == "regenerable": + return self.regenerable_under(payload) + return [] + + def scan_one_group(self, name, spec, group_deadline): + paths = self.resolve_spec(spec) + floor = self.min_size_bytes + override = GROUP_MIN_SIZE_OVERRIDE.get(name) + if override is not None: + floor = min(floor, override) + items = [] + for index, path in enumerate(paths): + if os.path.islink(path): + continue + remaining_items = len(paths) - index + now = time.monotonic() + if now > group_deadline or self.out_of_time(): + self.budget_hit = True + break + share = max((group_deadline - now) / remaining_items, 0.3) + item_deadline = min(now + share, self.deadline) + is_dir = os.path.isdir(path) + size, partial, mtime = self.measure(path, item_deadline) + if size is None: + continue + if size < floor and not partial: + self.dropped_small += 1 + continue + items.append(self.make_item(path, name, size, partial, mtime, is_dir)) + items.sort(key=lambda entry: entry["size_bytes"], reverse=True) + return items + + def scan_groups(self): + specs = self.group_specs() + groups = {} + skipped = [] + for index, (name, spec) in enumerate(specs): + if self.out_of_time(): + groups[name] = [] + skipped.append(name) + continue + remaining_groups = len(specs) - index + group_deadline = self.slice_deadline(1.0 / remaining_groups) + groups[name] = self.scan_one_group(name, spec, group_deadline) + groups["system_notes"] = self.system_notes() + return groups, skipped + + def system_notes(self): + """只写不扫。这些地方要空间得走系统自己的路子，不能直接删。""" + if self.is_windows: + notes = [ + ("WinSxS 组件存储", "C:\\Windows\\WinSxS", + "系统组件的历史版本，看着大，里面大量是硬链接，直接删会让更新和修复失败。", + "管理员执行 DISM /Online /Cleanup-Image /StartComponentCleanup"), + ("Windows 更新缓存", "C:\\Windows\\SoftwareDistribution\\Download", + "已下载的更新包，装完就没用了，但更新服务在跑的时候删会卡住。", + "用磁盘清理里的 Windows 更新清理，或先停 wuauserv 再清"), + ("休眠文件 hiberfil.sys", "C:\\hiberfil.sys", + "内存镜像，通常跟物理内存一个量级，删掉等于关掉休眠和快速启动。", + "管理员执行 powercfg /hibernate off"), + ("虚拟内存 pagefile.sys", "C:\\pagefile.sys", + "系统页面文件，直接删要么蓝屏要么立刻被重建。", + "在系统属性的虚拟内存里改大小或换盘"), + ("系统还原点", "C:\\System Volume Information", + "还原点和卷影副本，占的是保留配额，资源管理器里看不到。", + "系统保护里调低磁盘使用比例，或删掉旧还原点"), + ] + else: + notes = [ + ("可清除空间 Purgeable", "/System/Volumes/Data", + "系统标记为可清除的部分，df 里算在已用，磁盘紧张时系统会自己回收。", + "真的缺空间时自动释放，也可以在存储设置里查看"), + ("时间机器本地快照", "/Volumes/com.apple.TimeMachine.localsnapshots", + "本地快照挂在系统卷上，用户目录里看不见，却实实在在占地方。", + "tmutil listlocalsnapshots / 由系统按空间压力自动回收"), + ("系统级缓存", "/Library/Caches", + "全机共用的缓存，属于系统卷，普通用户改不动。", + "需要管理员权限，一般不建议手动清"), + ] + out = [] + for title, path, what, how in notes: + out.append({ + "name": title, + "path": path, + "path_portable": path, + "size_bytes": None, + "size_human": "未测量", + "kind": "system", + "suggested_color": "red", + "restore_hint": how, + "mtime": None, + "group": "system_notes", + "partial": False, + "scanned": False, + "note": what, + }) + return out + + # -------------------------------------------------- 体检 + + def collect_disks(self): + if self.is_windows: + return self.collect_disks_windows() + return self.collect_disks_macos() + + @staticmethod + def volume_role(mount, total_bytes): + """一个挂载点是 primary（数据卷）、external（外接盘），还是 system。""" + if mount == PRIMARY_DATA_MOUNT: + return "primary" + if not mount.startswith(EXTERNAL_MOUNT_PREFIX): + return "system" + name = mount[len(EXTERNAL_MOUNT_PREFIX):].split("/")[0] + low = name.lower() + for marker in SYSTEM_VOLUME_NAMES: + if low == marker or low.startswith(marker + "."): + return "system" + if total_bytes < MIN_EXTERNAL_VOLUME_BYTES: + return "system" + return "external" + + def collect_disks_macos(self): + disks = [] + system_volumes = [] + output = run_cmd(["df", "-k"]) + for line in output.splitlines()[1:]: + parts = line.split(None, 8) + if len(parts) < 9: + continue + device, blocks, used, avail, mount = ( + parts[0], parts[1], parts[2], parts[3], parts[8] + ) + if not device.startswith("/dev/"): + continue + try: + total_b = int(blocks) * 1024 + used_b = int(used) * 1024 + free_b = int(avail) * 1024 + except ValueError: + continue + role = self.volume_role(mount, total_b) + disk = self.make_disk( + mount, device, total_b, used_b, free_b, "df", + name=PRIMARY_DATA_NAME if role == "primary" else None, + primary=role == "primary", + ) + if role == "system": + system_volumes.append(disk) + else: + disks.append(disk) + disks.sort(key=lambda entry: (not entry["primary"], entry["mount"])) + self._system_volumes = system_volumes + if not disks: + fallback = self.fallback_disk("/") + if fallback: + disks.append(fallback) + return disks + + def collect_disks_windows(self): + disks = [] + for letter in _windows_drive_letters(): + root = letter + ":\\" + try: + total, used, free = shutil.disk_usage(root) + except Exception: + continue + disks.append( + self.make_disk(root, root, total, used, free, "shutil.disk_usage") + ) + if not disks: + fallback = self.fallback_disk(os.environ.get("SystemDrive", "C:") + "\\") + if fallback: + disks.append(fallback) + return disks + + def fallback_disk(self, root): + try: + total, used, free = shutil.disk_usage(root) + except Exception: + return None + return self.make_disk(root, root, total, used, free, "shutil.disk_usage") + + def make_disk(self, mount, device, total, used, free, source, + name=None, primary=False): + capacity = 0.0 + if used + free > 0: + capacity = round(100.0 * used / float(used + free), 1) + flags = [] + if capacity >= 85.0: + flags.append("high_usage") + if free < 30 * GB: + flags.append("low_free") + return { + "name": name or os.path.basename(mount.rstrip("/\\")) or mount, + "primary": bool(primary), + "mount": mount, + "device": device, + "total_bytes": total, + "used_bytes": used, + "free_bytes": free, + "total_human": human(total), + "used_human": human(used), + "free_human": human(free), + "capacity_percent": capacity, + "flags": flags, + "source": source, + } + + def collect_system_info(self): + info = { + "platform": self.platform, + "home_token": self.home_token, + "python": "%d.%d.%d" % sys.version_info[:3], + "cpu_count": os.cpu_count(), + } + if self.is_windows: + import platform as platform_mod + + info["os"] = platform_mod.system() + " " + platform_mod.release() + info["os_build"] = platform_mod.version() + info["arch"] = os.environ.get( + "PROCESSOR_ARCHITECTURE", platform_mod.machine() + ) + return info + + product = run_cmd(["sw_vers", "-productVersion"]).strip() + if product: + info["os"] = "macOS " + product + build = run_cmd(["sw_vers", "-buildVersion"]).strip() + if build: + info["os_build"] = build + arch = run_cmd(["uname", "-m"]).strip() + if arch: + info["arch"] = arch + brand = run_cmd(["sysctl", "-n", "machdep.cpu.brand_string"]).strip() + if brand: + info["cpu"] = brand + memsize = run_cmd(["sysctl", "-n", "hw.memsize"]).strip() + if memsize.isdigit(): + info["memory_bytes"] = int(memsize) + info["memory_human"] = human(int(memsize)) + pressure = run_cmd(["memory_pressure"]) + if pressure: + info["memory_pressure_head"] = "\n".join(pressure.splitlines()[:8]) + vm_stat = run_cmd(["vm_stat"]) + if vm_stat: + info["vm_stat_head"] = "\n".join(vm_stat.splitlines()[:6]) + uptime = run_cmd(["uptime"]).strip() + if uptime: + info["uptime"] = uptime + return info + + # -------------------------------------------------- 汇总 + + def top_files(self): + files = sorted( + self._big_files.values(), + key=lambda entry: entry["size_bytes"], + reverse=True, + ) + return files[: self.top_files_limit] + + def run(self): + if self.with_system: + system = self.collect_system_info() + disks = self.collect_disks() + else: + system = {"platform": self.platform, "home_token": self.home_token} + disks = [] + + groups, skipped = self.scan_groups() + top = self.top_files() + denied = sorted(self._denied.values(), key=lambda entry: entry["path"]) + + measured = 0 + item_count = 0 + partial_count = 0 + for items in groups.values(): + for item in items: + if item.get("size_bytes"): + measured += item["size_bytes"] + item_count += 1 + if item.get("partial"): + partial_count += 1 + + denied_note = "" + if denied: + denied_note = ( + "有 %d 个目录读不到，体积没有计入，实际占用比这份数字更大。" + % len(denied) + ) + + return { + "schema": SCHEMA, + "generated_at": time.strftime("%Y-%m-%d %H:%M:%S %z"), + "platform": self.platform, + "home_token": self.home_token, + "system": system, + "disks": disks, + "system_volumes": self._system_volumes, + "groups": groups, + "top_files": top, + "denied": denied, + "elapsed_seconds": round(time.monotonic() - self.started_monotonic, 2), + "budget_seconds": self.budget_seconds, + "budget_hit": self.budget_hit, + "skipped_groups": skipped, + "summary": { + "group_count": len(groups), + "item_count": item_count, + "partial_item_count": partial_count, + "measured_bytes": measured, + "measured_human": human(measured), + "sizes_are_lower_bounds": partial_count > 0, + "partial_note": ( + "有 %d 个条目没量完就到点了，它们的体积是下限，实际更大。" + % partial_count + ) if partial_count else "", + "dropped_below_floor": self.dropped_small, + "min_size_bytes": self.min_size_bytes, + "min_file_bytes": self.min_file_bytes, + "denied_count": len(denied), + "denied_note": denied_note, + }, + } + + +# ---------------------------------------------------------------- 入口 + + +def run_scan(**kwargs): + """跑一轮扫描，返回 storage-scan.json 的内容（dict）。""" + return StorageScanner(**kwargs).run() + + +def format_summary(data): + lines = ["扫描用时 %.2fs%s" % ( + data["elapsed_seconds"], "，撞到预算上限" if data["budget_hit"] else "" + )] + for disk in data.get("disks", []): + lines.append("盘 %s：%s 已用 / %s 可用 / %s%%" % ( + disk["mount"], disk["used_human"], disk["free_human"], + disk["capacity_percent"], + )) + for name in sorted(data["groups"]): + items = data["groups"][name] + if items: + lines.append(" %-22s %d 项" % (name, len(items))) + top = data.get("top_files", [])[:5] + if top: + lines.append("最大的几个文件：") + for entry in top: + lines.append(" %10s %s" % (entry["size_human"], entry["path_portable"])) + for key in ("partial_note", "denied_note"): + if data["summary"].get(key): + lines.append(data["summary"][key]) + return "\n".join(lines) + + +def forgiving_console(): + """摘要里全是中文，而 Windows 上重定向出去的 stdout 是老代码页。 + + 不管它的话，一次量完的扫描会在打印摘要那一行抛 UnicodeEncodeError， + 盘点结果明明已经落盘了，退出码却是 1。JSON 自己带 encoding="utf-8"， + 从来不受影响；转义的只是给人看的那几行。 + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(errors="backslashreplace") + except (ValueError, OSError): + continue + + +def main(argv=None): + forgiving_console() + parser = argparse.ArgumentParser( + description="整机只读盘点扫描（macOS + Windows），只量大小，不动文件。" + ) + parser.add_argument("--out", default="storage-scan.json", + help="输出文件路径，写 - 就打到标准输出") + parser.add_argument("--budget-seconds", type=float, + default=DEFAULT_BUDGET_SECONDS, help="整轮时间上限") + parser.add_argument("--top-files", type=int, default=DEFAULT_TOP_FILES, + help="大文件榜取前几名") + parser.add_argument("--min-file-mb", type=float, default=DEFAULT_MIN_FILE_MB, + help="进大文件榜的门槛，单位 MB") + parser.add_argument("--min-size-mb", type=float, default=DEFAULT_MIN_SIZE_MB, + help="进分组清单的门槛，单位 MB") + parser.add_argument("--home", default=None, help="指定要扫的家目录，默认当前用户") + parser.add_argument("--quiet", action="store_true", help="不打人类可读摘要") + args = parser.parse_args(argv) + + data = run_scan( + home=args.home, + budget_seconds=args.budget_seconds, + top_files=args.top_files, + min_file_bytes=int(args.min_file_mb * 1024 * 1024), + min_size_bytes=int(args.min_size_mb * 1024 * 1024), + ) + + payload = json.dumps(data, ensure_ascii=False, indent=2) + if args.out == "-": + sys.stdout.write(payload + "\n") + else: + out_path = os.path.abspath(args.out) + parent = os.path.dirname(out_path) + if parent and not os.path.isdir(parent): + os.makedirs(parent, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as handle: + handle.write(payload + "\n") + if not args.quiet: + sys.stdout.write("已写入 %s\n" % out_path) + if not args.quiet: + sys.stdout.write(format_summary(data) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/cfo_helpers.py b/tests/cfo_helpers.py new file mode 100644 index 0000000..9fe04e4 --- /dev/null +++ b/tests/cfo_helpers.py @@ -0,0 +1,85 @@ +"""Shared scaffolding for the engine tests. + +Two rules keep these tests honest: time is always injected, and the safety net +never touches the real machine. ``GUARD_OFFLINE`` gives every planner test an +empty reference list, an empty symlink scan root and no lsof at all, so a test +run cannot depend on what happens to be open on the developer's laptop. +""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +TZ = timezone(timedelta(hours=9)) +NOW = datetime(2026, 9, 2, 12, 0, 0, tzinfo=TZ) + +#: The safety net, wired to nothing. +GUARD_OFFLINE: Dict[str, Any] = { + "reference_files": [], + "scan_roots": [], + "lsof_enabled": False, +} + + +def guard_with(**overrides: Any) -> Dict[str, Any]: + options = dict(GUARD_OFFLINE) + options.update(overrides) + return options + + +def write(path: Path, content: str = "placeholder\n") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def age(path: Path, hours: float, now: datetime = NOW) -> Path: + """Backdate ``path`` so it reads as ``hours`` old relative to ``now``.""" + + when = (now - timedelta(hours=hours)).timestamp() + os.utime(str(path), (when, when)) + return path + + +def fake_lsof(mapping: Optional[Dict[str, str]] = None): + """An lsof stand-in: path substring -> ``command,pid``.""" + + table = mapping or {} + + class _Result(object): + def __init__(self, stdout: str, returncode: int = 0) -> None: + self.stdout = stdout + self.returncode = returncode + + def runner(argv, timeout_s): # noqa: ANN001 - mirrors subprocess.run's shape + lines = [] + for argument in argv: + for needle, who in table.items(): + if needle in str(argument): + command, pid = who.split(",") + lines.append("p{0}".format(pid)) + lines.append("c{0}".format(command)) + lines.append("n{0}".format(argument)) + return _Result("\n".join(lines) + ("\n" if lines else ""), 0 if lines else 1) + + return runner + + +def action_for(plan: Dict[str, Any], filename: str, kind: str = "move") -> Dict[str, Any]: + for action in plan["actions"]: + if action["filename"] == filename and action["kind"] == kind: + return action + raise AssertionError( + "no {0} action for {1}; saw {2}".format( + kind, + filename, + sorted({(a["filename"], a["kind"]) for a in plan["actions"]}), + ) + ) + + +def actions_for(plan: Dict[str, Any], filename: str): + return [action for action in plan["actions"] if action["filename"] == filename] diff --git a/tests/cfo_path.py b/tests/cfo_path.py new file mode 100644 index 0000000..2d83c40 --- /dev/null +++ b/tests/cfo_path.py @@ -0,0 +1,24 @@ +"""Put the file organizer's scripts directory on ``sys.path``. + +The engine lives inside a skill folder (``skills/ops/carl-file-organizer/scripts``) +rather than in an installed package, so every test that says +``import carl_file_organizer`` imports this module first. Importing the whole +package once is much cheaper than loading twenty modules by file path, and it +keeps the intra-package imports (``from . import guard``) working. + +Importing this module twice is free; the insert is guarded. +""" + +from __future__ import annotations + +import os +import sys + +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(TESTS_DIR) +SCRIPTS_DIR = os.path.join(REPO_ROOT, "skills", "ops", "carl-file-organizer", "scripts") +FIXTURES_DIR = os.path.join(TESTS_DIR, "fixtures", "carl_file_organizer") + +for _entry in (SCRIPTS_DIR, TESTS_DIR): + if _entry not in sys.path: + sys.path.insert(0, _entry) diff --git a/tests/fixtures/carl_file_organizer/plan-v2-sample.json b/tests/fixtures/carl_file_organizer/plan-v2-sample.json new file mode 100644 index 0000000..c82f8ca --- /dev/null +++ b/tests/fixtures/carl_file_organizer/plan-v2-sample.json @@ -0,0 +1,1230 @@ +{ + "schema_version": 2, + "generator": { + "name": "carl-file-organizer", + "version": "0.3.0" + }, + "created_at": "2026-09-02T10:15:00+09:00", + "now": "2026-09-02T10:15:00+09:00", + "lang": "zh", + "lang_source": "existing-dirs", + "profile": { + "name": "tiered", + "source": "builtin+dotfile", + "dotfile": "$HOME/Downloads/.carl-file-organizer.json", + "hash": "3f9a1c0e2b77" + }, + "source_root": "/Users/example/Downloads", + "source_root_portable": "$HOME/Downloads", + "managed_dir": "/Users/example/Downloads/00_下载目录管理", + "managed_dir_portable": "$HOME/Downloads/00_下载目录管理", + "names": { + "inbox": "00_收件箱", + "inbox.pending": "00_收件箱/待判断", + "inbox.by_tag": "00_收件箱/按标签", + "inbox.dup": "00_收件箱/重复待确认", + "work": "10_工作区", + "work.tools": "10_工作区/工具与导出", + "work.tools.scripts": "10_工作区/工具与导出/脚本", + "library": "20_知识库", + "library.docs": "20_知识库/文档资料", + "library.docs.pdf": "20_知识库/文档资料/PDF", + "library.docs.txt": "20_知识库/文档资料/TXT", + "sensitive": "60_敏感信息", + "sensitive.pending": "60_敏感信息/待转移", + "sensitive.dup": "60_敏感信息/重复待确认", + "archive": "90_归档", + "archive.zip": "90_归档/压缩包", + "archive.zip.zip": "90_归档/压缩包/ZIP", + "archive.installers": "90_归档/安装包", + "archive.installers.dmg": "90_归档/安装包/DMG" + }, + "capabilities": { + "trash_backend": "finder", + "finder_tags": true, + "permanent_delete_offered": true + }, + "scan_policy": { + "depth": 1, + "hidden_files": "ignored", + "symlinks": "ignored", + "managed_partitions": "skipped", + "regenerable_scan_depth": 3, + "dir_size_limit": { + "max_entries": 50000, + "time_budget_s": 1.5 + }, + "hash_duplicates": true, + "hash_max_bytes": 2147483648, + "allow_referenced": false + }, + "disk": { + "total_bytes": 994662584320, + "used_bytes": 815394881536, + "free_bytes": 179267702784, + "used_percent": 81.98, + "target_percent": 90.0, + "bytes_to_remove_for_target": 0, + "note": "Moves on this volume reclaim 0 bytes." + }, + "mess": { + "score": 52, + "color": "yellow", + "color_source": "rule", + "counts": { + "loose": 12, + "aging": 1, + "pending": 2, + "groups": 3, + "top_dirs": 3 + }, + "reason": { + "zh": "散件 12 件，其中 2 件要你看一眼，还没到失控的地步", + "en": "12 loose items, 2 of them want a look from you; not out of hand yet" + } + }, + "summary": { + "entries": 12, + "files": 9, + "dirs": 3, + "bytes": 49115218011, + "by_tier": { + "aging": 1, + "duplicate": 9, + "forbidden": 1, + "in_use": 1, + "referenced": 1, + "regenerable": 2, + "routine": 3, + "sensitive": 1 + }, + "by_kind": { + "delete": 2, + "hold": 4, + "move": 8, + "trash": 5 + }, + "by_color": { + "green": { + "actions": 8, + "bytes": 419153355 + }, + "yellow": { + "actions": 7, + "bytes": 277336072 + }, + "red": { + "actions": 4, + "bytes": 48844385348 + } + }, + "by_destination": { + "00_收件箱/待判断": 2, + "00_收件箱/待判断/Archive": 2, + "20_知识库/文档资料/PDF": 3, + "60_敏感信息/待转移": 1 + }, + "groups": { + "pair": 1, + "duplicate": 1, + "regenerable": 1 + }, + "potential_reclaimable_bytes": 218429399, + "proposed_reclaimed_bytes": 0 + }, + "pinned": [ + "carl-file-organizer-*.json" + ], + "actions": [ + { + "id": "9c1f0a7b2d3e4f55", + "subject_id": "1a2b3c4d5e6f7a8b", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/report.pdf", + "source_portable": "$HOME/Downloads/report.pdf", + "filename": "report.pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/report.pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 2411008, + "modified_ns": 1756700000000000000, + "age_hours": 71, + "rule": "ext:.pdf", + "reason": { + "zh": "文档按类型归档到 20_知识库/文档资料/PDF", + "en": "Document filed by type into 20_知识库/文档资料/PDF" + }, + "detail": { + "zh": "静置 71 小时，已满 48 小时", + "en": "aged 71h, past the 48h threshold" + }, + "confidence": 0.98, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "0b7e4411aa93c2d1", + "subject_id": "2b3c4d5e6f7a8b9c", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/api-key-backup.txt", + "source_portable": "$HOME/Downloads/api-key-backup.txt", + "filename": "api-key-backup.txt", + "destination": "/Users/example/Downloads/60_敏感信息/待转移/api-key-backup.txt", + "destination_portable": "$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt", + "destination_key": "sensitive.pending", + "size_bytes": 1204, + "modified_ns": 1756500000000000000, + "age_hours": 126, + "rule": "sensitive:marker", + "reason": { + "zh": "名字看着像凭证，先单独隔离，未读取内容", + "en": "The name looks like a credential, so it is isolated; contents were never read" + }, + "detail": { + "zh": "命中敏感词 api-key，不看静置期", + "en": "matched marker api-key; the settling period is skipped" + }, + "confidence": 0.9, + "tier": "sensitive", + "group_id": null, + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/60_敏感信息/待转移/api-key-backup.txt\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "a1b2c30d4e5f6071", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": "/Users/example/Downloads/00_收件箱/待判断/Archive/Archive.zip", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip", + "destination_key": "inbox.pending", + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive.zip\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "b2c3d40e5f607182", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 8915621, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "c3d4e50f60718293", + "subject_id": "3c4d5e6f7a8b9c0d", + "kind": "delete", + "subject_kind": "file", + "source": "/Users/example/Downloads/Archive.zip", + "source_portable": "$HOME/Downloads/Archive.zip", + "filename": "Archive.zip", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 8915621, + "modified_ns": 1756100000000000000, + "age_hours": 240, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [ + "allow-permanent-delete" + ], + "reclaims_bytes": 8915621, + "restore_method": { + "zh": "删掉就没了，只对可再生的构建产物用这一项", + "en": "Permanent; only use this for regenerable build output" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "d4e5f60718293a4b", + "subject_id": "4d5e6f7a8b9c0d1e", + "kind": "move", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Archive", + "source_portable": "$HOME/Downloads/Archive", + "filename": "Archive", + "destination": "/Users/example/Downloads/00_收件箱/待判断/Archive/Archive", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/Archive/Archive", + "destination_key": "inbox.pending", + "size_bytes": 9104882, + "modified_ns": 1756110000000000000, + "age_hours": 237, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/Archive/Archive\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 42, + "bytes": 9104882, + "truncated": false + }, + "color": "green" + }, + { + "id": "e5f60718293a4b5c", + "subject_id": "4d5e6f7a8b9c0d1e", + "kind": "trash", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Archive", + "source_portable": "$HOME/Downloads/Archive", + "filename": "Archive", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 9104882, + "modified_ns": 1756110000000000000, + "age_hours": 237, + "rule": "pair", + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "detail": null, + "confidence": 0.8, + "tier": "duplicate", + "group_id": "pair-7d2c1e9f", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 9104882, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 42, + "bytes": 9104882, + "truncated": false + }, + "color": "yellow" + }, + { + "id": "f60718293a4b5c6d", + "subject_id": "5e6f7a8b9c0d1e2f", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide.pdf", + "source_portable": "$HOME/Downloads/guide.pdf", + "filename": "guide.pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/guide.pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 1048576, + "modified_ns": 1756300000000000000, + "age_hours": 160, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide.pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "0718293a4b5c6d7e", + "subject_id": "5e6f7a8b9c0d1e2f", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide.pdf", + "source_portable": "$HOME/Downloads/guide.pdf", + "filename": "guide.pdf", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 1048576, + "modified_ns": 1756300000000000000, + "age_hours": 160, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 1048576, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "18293a4b5c6d7e8f", + "subject_id": "6f7a8b9c0d1e2f30", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide (1).pdf", + "source_portable": "$HOME/Downloads/guide (1).pdf", + "filename": "guide (1).pdf", + "destination": "/Users/example/Downloads/20_知识库/文档资料/PDF/guide (1).pdf", + "destination_portable": "$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf", + "destination_key": "library.docs.pdf", + "size_bytes": 1048576, + "modified_ns": 1756310000000000000, + "age_hours": 157, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/20_知识库/文档资料/PDF/guide (1).pdf\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "green" + }, + { + "id": "293a4b5c6d7e8f90", + "subject_id": "6f7a8b9c0d1e2f30", + "kind": "trash", + "subject_kind": "file", + "source": "/Users/example/Downloads/guide (1).pdf", + "source_portable": "$HOME/Downloads/guide (1).pdf", + "filename": "guide (1).pdf", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 1048576, + "modified_ns": 1756310000000000000, + "age_hours": 157, + "rule": "dup:sha256", + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "detail": null, + "confidence": 0.95, + "tier": "duplicate", + "group_id": "dup-3ba91c04", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 1048576, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + }, + { + "id": "3a4b5c6d7e8f9001", + "subject_id": "7a8b9c0d1e2f3041", + "kind": "move", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype", + "source_portable": "$HOME/Downloads/web-prototype", + "filename": "web-prototype", + "destination": "/Users/example/Downloads/00_收件箱/待判断/web-prototype", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/web-prototype", + "destination_key": "inbox.pending", + "size_bytes": 214748364, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "dir:pending", + "reason": { + "zh": "整个目录保留原名，先进待判断", + "en": "Whole folder keeps its name and waits in Pending" + }, + "detail": null, + "confidence": 0.6, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": true, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/web-prototype\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/web-prototype\" \"$HOME/Downloads/\"" + }, + "tags": [], + "hints": [ + "git-repo" + ], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [ + "git-dir", + "node-project" + ] + }, + "dir_stats": { + "files": 18422, + "bytes": 214748364, + "truncated": false + }, + "color": "yellow" + }, + { + "id": "4b5c6d7e8f900112", + "subject_id": "8b9c0d1e2f304152", + "kind": "trash", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype/node_modules", + "source_portable": "$HOME/Downloads/web-prototype/node_modules", + "filename": "node_modules", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 198311744, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "regenerable:node_modules", + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "detail": null, + "confidence": 0.99, + "tier": "regenerable", + "group_id": "regen-5c1d77ab", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 198311744, + "restore_method": { + "zh": "在废纸篓里选放回原处", + "en": "Put Back from the Trash" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 18120, + "bytes": 198311744, + "truncated": false + }, + "color": "green" + }, + { + "id": "5c6d7e8f90011223", + "subject_id": "8b9c0d1e2f304152", + "kind": "delete", + "subject_kind": "dir", + "source": "/Users/example/Downloads/web-prototype/node_modules", + "source_portable": "$HOME/Downloads/web-prototype/node_modules", + "filename": "node_modules", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 198311744, + "modified_ns": 1756400000000000000, + "age_hours": 132, + "rule": "regenerable:node_modules", + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "detail": null, + "confidence": 0.99, + "tier": "regenerable", + "group_id": "regen-5c1d77ab", + "approvable": true, + "reroutable": false, + "default_selected": false, + "requires": [ + "allow-permanent-delete" + ], + "reclaims_bytes": 198311744, + "restore_method": { + "zh": "删掉就没了，只对可再生的构建产物用这一项", + "en": "Permanent; only use this for regenerable build output" + }, + "tags": [], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 18120, + "bytes": 198311744, + "truncated": false + }, + "color": "green" + }, + { + "id": "6d7e8f9001122334", + "subject_id": "9c0d1e2f30415263", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/installer-2026.dmg", + "source_portable": "$HOME/Downloads/installer-2026.dmg", + "filename": "installer-2026.dmg", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 612368384, + "modified_ns": 1756750000000000000, + "age_hours": 57, + "rule": "hold:aging", + "reason": { + "zh": "还在静置期，已放 57 小时，满 168 小时再动", + "en": "Still settling: 57h of the required 168h" + }, + "detail": null, + "confidence": 1.0, + "tier": "aging", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "还在静置期，已放 57 小时，满 168 小时再动", + "en": "Still settling: 57h of the required 168h", + "ready_at": "2026-09-07T19:15:00+09:00" + }, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "7e8f900112233445", + "subject_id": "0d1e2f3041526374", + "kind": "hold", + "subject_kind": "dir", + "source": "/Users/example/Downloads/Photos Library.photoslibrary", + "source_portable": "$HOME/Downloads/Photos Library.photoslibrary", + "filename": "Photos Library.photoslibrary", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 48231923712, + "modified_ns": 1756600000000000000, + "age_hours": 98, + "rule": "hold:forbidden", + "reason": { + "zh": "禁刀区，只展示不处理（forbidden:suffix:.photoslibrary）", + "en": "No-go zone, shown but never touched (forbidden:suffix:.photoslibrary)" + }, + "detail": null, + "confidence": 1.0, + "tier": "forbidden", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "系统媒体库，任何情况下都不动", + "en": "A system media library; never touched", + "label": "forbidden:suffix:.photoslibrary" + }, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": { + "files": 0, + "bytes": 48231923712, + "truncated": true + }, + "color": "red" + }, + { + "id": "8f90011223344556", + "subject_id": "1e2f304152637485", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/session.log", + "source_portable": "$HOME/Downloads/session.log", + "filename": "session.log", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 91204, + "modified_ns": 1756790000000000000, + "age_hours": 6, + "rule": "hold:in_use", + "reason": { + "zh": "现在有进程开着它：tail(48213)，先不动", + "en": "A process still has it open: tail(48213)" + }, + "detail": null, + "confidence": 1.0, + "tier": "in_use", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "进程 tail 正在读它，等它退出再整理", + "en": "tail is reading it; tidy up after it exits" + }, + "guard": { + "open_by": [ + { + "command": "tail", + "pid": 48213 + } + ], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "900112233445566a", + "subject_id": "2f30415263748596", + "kind": "hold", + "subject_kind": "file", + "source": "/Users/example/Downloads/backup-tool.sh", + "source_portable": "$HOME/Downloads/backup-tool.sh", + "filename": "backup-tool.sh", + "destination": null, + "destination_portable": null, + "destination_key": null, + "size_bytes": 2048, + "modified_ns": 1756200000000000000, + "age_hours": 210, + "rule": "hold:referenced", + "reason": { + "zh": "有配置或脚本引用了这个路径：$HOME/Library/LaunchAgents/com.example.backup.plist:14，动它之前先改引用", + "en": "A config or script references this path: $HOME/Library/LaunchAgents/com.example.backup.plist:14; fix the reference first" + }, + "detail": null, + "confidence": 1.0, + "tier": "referenced", + "group_id": null, + "approvable": false, + "reroutable": false, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": null, + "tags": [], + "hints": [], + "hold_reason": { + "zh": "一个 launchd 任务写死了这个路径", + "en": "A launchd job hard-codes this path" + }, + "guard": { + "open_by": [], + "referenced_in": [ + { + "file": "$HOME/Library/LaunchAgents/com.example.backup.plist", + "line": 14, + "form": "$HOME/Downloads/backup-tool.sh" + }, + { + "file": "$HOME/.zshrc", + "line": 82, + "form": "~/Downloads/backup-tool.sh" + } + ], + "incoming_symlinks": [ + "$HOME/.local/bin/backup-tool" + ], + "shape": [ + "executable", + "shebang" + ] + }, + "dir_stats": null, + "color": "red" + }, + { + "id": "0112233445566a7b", + "subject_id": "30415263748596a7", + "kind": "move", + "subject_kind": "file", + "source": "/Users/example/Downloads/dataset.parquet", + "source_portable": "$HOME/Downloads/dataset.parquet", + "filename": "dataset.parquet", + "destination": "/Users/example/Downloads/00_收件箱/待判断/dataset.parquet", + "destination_portable": "$HOME/Downloads/00_收件箱/待判断/dataset.parquet", + "destination_key": "inbox.pending", + "size_bytes": 33554432, + "modified_ns": 1756450000000000000, + "age_hours": 118, + "rule": "unknown-ext", + "reason": { + "zh": "扩展名不在规则表里，先进待判断，可以在页面上改去向", + "en": "Extension is not in the rule table; it waits in Pending and can be rerouted" + }, + "detail": null, + "confidence": 0.55, + "tier": "routine", + "group_id": null, + "approvable": true, + "reroutable": true, + "default_selected": false, + "requires": [], + "reclaims_bytes": 0, + "restore_method": { + "zh": "mv \"$HOME/Downloads/00_收件箱/待判断/dataset.parquet\" \"$HOME/Downloads/\"", + "en": "mv \"$HOME/Downloads/00_收件箱/待判断/dataset.parquet\" \"$HOME/Downloads/\"" + }, + "tags": [ + "数据" + ], + "hints": [], + "hold_reason": null, + "guard": { + "open_by": [], + "referenced_in": [], + "incoming_symlinks": [], + "shape": [] + }, + "dir_stats": null, + "color": "yellow" + } + ], + "groups": [ + { + "group_id": "pair-7d2c1e9f", + "kind": "pair", + "tier": "duplicate", + "stem": "archive", + "evidence": "name", + "sha256": null, + "members": [ + "4d5e6f7a8b9c0d1e", + "3c4d5e6f7a8b9c0d" + ], + "member_labels": { + "4d5e6f7a8b9c0d1e": "Archive/", + "3c4d5e6f7a8b9c0d": "Archive.zip" + }, + "reason": { + "zh": "压缩包和解压目录是一对，默认建议留目录", + "en": "Archive and extracted folder are a pair; keeping the folder is suggested" + }, + "options": [ + { + "key": "keep_all", + "label": { + "zh": "都保留，一起搬进待判断", + "en": "Keep both, move them together" + }, + "action_ids": [ + "d4e5f60718293a4b", + "a1b2c30d4e5f6071" + ], + "requires": [] + }, + { + "key": "keep:4d5e6f7a8b9c0d1e", + "label": { + "zh": "留目录，压缩包进废纸篓", + "en": "Keep the folder, trash the archive" + }, + "action_ids": [ + "d4e5f60718293a4b", + "b2c3d40e5f607182" + ], + "requires": [] + }, + { + "key": "keep:3c4d5e6f7a8b9c0d", + "label": { + "zh": "留压缩包，目录进废纸篓", + "en": "Keep the archive, trash the folder" + }, + "action_ids": [ + "a1b2c30d4e5f6071", + "e5f60718293a4b5c" + ], + "requires": [] + }, + { + "key": "keep:4d5e6f7a8b9c0d1e:delete", + "label": { + "zh": "留目录，压缩包永久删除", + "en": "Keep the folder, permanently delete the archive" + }, + "action_ids": [ + "d4e5f60718293a4b", + "c3d4e50f60718293" + ], + "requires": [ + "allow-permanent-delete" + ] + } + ], + "default_choice": "keep:4d5e6f7a8b9c0d1e", + "potential_bytes": 8915621, + "color": "yellow" + }, + { + "group_id": "dup-3ba91c04", + "kind": "duplicate", + "tier": "duplicate", + "stem": "guide.pdf", + "evidence": "sha256", + "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + "members": [ + "5e6f7a8b9c0d1e2f", + "6f7a8b9c0d1e2f30" + ], + "member_labels": { + "5e6f7a8b9c0d1e2f": "guide.pdf", + "6f7a8b9c0d1e2f30": "guide (1).pdf" + }, + "reason": { + "zh": "内容完全相同（sha256 一致），留一份就够", + "en": "Byte-identical (same sha256); one copy is enough" + }, + "options": [ + { + "key": "keep_all", + "label": { + "zh": "都保留", + "en": "Keep both" + }, + "action_ids": [ + "f60718293a4b5c6d", + "18293a4b5c6d7e8f" + ], + "requires": [] + }, + { + "key": "keep:5e6f7a8b9c0d1e2f", + "label": { + "zh": "留 guide.pdf，另一份进废纸篓", + "en": "Keep guide.pdf, trash the other" + }, + "action_ids": [ + "f60718293a4b5c6d", + "293a4b5c6d7e8f90" + ], + "requires": [] + }, + { + "key": "keep:6f7a8b9c0d1e2f30", + "label": { + "zh": "留 guide (1).pdf，另一份进废纸篓", + "en": "Keep guide (1).pdf, trash the other" + }, + "action_ids": [ + "18293a4b5c6d7e8f", + "0718293a4b5c6d7e" + ], + "requires": [] + } + ], + "default_choice": "keep:5e6f7a8b9c0d1e2f", + "potential_bytes": 1048576, + "color": "yellow" + }, + { + "group_id": "regen-5c1d77ab", + "kind": "regenerable", + "tier": "regenerable", + "stem": "node_modules", + "evidence": "basename", + "sha256": null, + "members": [ + "8b9c0d1e2f304152" + ], + "member_labels": { + "8b9c0d1e2f304152": "web-prototype/node_modules" + }, + "reason": { + "zh": "构建产物，删掉还能再生成（重新 npm/pnpm/bun install）", + "en": "Build output; it can be regenerated (re-run npm/pnpm/bun install)" + }, + "options": [ + { + "key": "trash_all", + "label": { + "zh": "全部进废纸篓", + "en": "Trash all of them" + }, + "action_ids": [ + "4b5c6d7e8f900112" + ], + "requires": [] + }, + { + "key": "delete_all", + "label": { + "zh": "全部永久删除", + "en": "Permanently delete all of them" + }, + "action_ids": [ + "5c6d7e8f90011223" + ], + "requires": [ + "allow-permanent-delete" + ] + } + ], + "default_choice": "trash_all", + "potential_bytes": 198311744, + "color": "green" + } + ], + "notes": null, + "approved_action_ids": [], + "overrides": [], + "approved_at": null, + "approved_by": null +} diff --git a/tests/fixtures/carl_file_organizer/storage-analysis-sample.json b/tests/fixtures/carl_file_organizer/storage-analysis-sample.json new file mode 100644 index 0000000..45d9025 --- /dev/null +++ b/tests/fixtures/carl_file_organizer/storage-analysis-sample.json @@ -0,0 +1,245 @@ +{ + "schema": "carl-file-organizer/storage-analysis", + "schema_version": 1, + "generator": {"name": "carl-file-organizer", "version": "0.3.0"}, + "created_at": "2026-09-02T14:05:00+09:00", + "lang": "zh", + "scan": {"source": "storage-scan.json", "elapsed_s": 41.6, "denied": ["$HOME/Library/Mail", "$HOME/Library/Messages"], "top_n": 5}, + "machine": { + "hostname": "carl-macbook", + "os": "macOS 15.6", + "arch": "arm64", + "cpu": "Apple M2 Pro", + "memory_bytes": 34359738368, + "uptime_days": 12, + "home_portable": "$HOME" + }, + "disks": [ + {"mount": "/", "name": "Macintosh HD", "filesystem": "apfs", "total_bytes": 994662584320, "used_bytes": 815394881536, "free_bytes": 179267702784, "used_percent": 81.98, "primary": true}, + {"mount": "/Volumes/T7", "name": "T7", "filesystem": "exfat", "total_bytes": 2000398934016, "used_bytes": 612345678901, "free_bytes": 1388053255115, "used_percent": 30.61, "primary": false} + ], + "summary": { + "scanned_bytes": 401234567890, + "reclaimable_bytes": 61239487232, + "by_color": { + "green": {"count": 4, "bytes": 38654705664}, + "yellow": {"count": 3, "bytes": 22584781568}, + "red": {"count": 3, "bytes": 96636764160} + } + }, + "overview": { + "headline": "系统盘用了 82%，其中 36 GB 是能直接清的开发缓存，21 GB 要你看一眼，剩下的别碰。", + "priority": [ + {"text": "先清 Xcode DerivedData 和 pip、Homebrew 缓存，三项加起来 31 GB，删了下次构建自动重建。", "item_ids": ["st-derived-data", "st-pip-cache", "st-brew-cache"]}, + {"text": "Downloads 里 6 个安装包已经装过了，看一眼没用的丢废纸篓。", "item_ids": ["st-dl-installers"]}, + {"text": "Chrome 用户数据 9 GB，里面有登录态，别清，改在浏览器里清缓存。", "item_ids": ["st-chrome-profile"]} + ], + "long_term": [ + "把 ~/projects 里超过半年没动的仓库整个搬到 T7，node_modules 不用跟着走。", + "Time Machine 本地快照占着 40 GB，接上备份盘跑一次备份会自动瘦下去。", + "Photos 图库 45 GB 建议开 iCloud 优化存储，本地只留缩略图。" + ] + }, + "top5": [ + {"id": "st-photos", "name": "Photos Library.photoslibrary", "path_portable": "$HOME/Pictures/Photos Library.photoslibrary", "size_bytes": 48318382080, "kind": "media", "color": "red", "note": "系统媒体库，只能在照片 app 里管理"}, + {"id": "st-tm-snapshots", "name": "Time Machine 本地快照", "path_portable": "/System/Volumes/Data/.MobileBackups", "size_bytes": 42949672960, "kind": "system", "color": "red", "note": "系统自己管，接上备份盘就会释放"}, + {"id": "st-derived-data", "name": "DerivedData", "path_portable": "$HOME/Library/Developer/Xcode/DerivedData", "size_bytes": 23622320128, "kind": "build", "color": "green", "note": "Xcode 编译中间产物"}, + {"id": "st-chrome-profile", "name": "Chrome", "path_portable": "$HOME/Library/Application Support/Google/Chrome", "size_bytes": 9663676416, "kind": "app-data", "color": "yellow", "note": "浏览器配置和登录态"}, + {"id": "st-old-node-modules", "name": "node_modules（旧项目）", "path_portable": "$HOME/projects/legacy-dashboard/node_modules", "size_bytes": 8589934592, "kind": "build", "color": "green", "note": "半年没动的项目依赖"} + ], + "items": [ + { + "id": "st-derived-data", + "name": "DerivedData", + "path_portable": "$HOME/Library/Developer/Xcode/DerivedData", + "size_bytes": 23622320128, + "kind": "build", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "Xcode 每次编译留下的中间产物和索引。", + "why": "只要项目源码还在，这些东西下次打开工程会自动重建。", + "if_removed": "第一次编译会慢几分钟，之后一切照旧。", + "action": "trash", + "trash_paths": ["$HOME/Library/Developer/Xcode/DerivedData"], + "restore": "不需要恢复，重新编译即可。", + "commands": ["rm -rf \"$HOME/Library/Developer/Xcode/DerivedData\""], + "kill_processes": ["Xcode"], + "open_by": [] + }, + { + "id": "st-pip-cache", + "name": "pip 缓存", + "path_portable": "$HOME/Library/Caches/pip", + "size_bytes": 4294967296, + "kind": "cache", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "pip 下载过的 wheel 包缓存。", + "why": "纯缓存，装包时命中会快一点，没有也能从 PyPI 重新拉。", + "if_removed": "下次 pip install 多花一点下载时间。", + "action": "trash", + "trash_paths": ["$HOME/Library/Caches/pip"], + "restore": "不需要恢复。", + "commands": ["pip cache purge"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-brew-cache", + "name": "Homebrew 缓存", + "path_portable": "$HOME/Library/Caches/Homebrew", + "size_bytes": 3221225472, + "kind": "cache", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "brew 下载过的安装包。", + "why": "已经装好的软件不依赖它，brew 自己也建议定期清。", + "if_removed": "重装同一个包时要重新下载。", + "action": "trash", + "trash_paths": ["$HOME/Library/Caches/Homebrew"], + "restore": "不需要恢复。", + "commands": ["brew cleanup --prune=all"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-old-node-modules", + "name": "node_modules（旧项目）", + "path_portable": "$HOME/projects/legacy-dashboard/node_modules", + "size_bytes": 8589934592, + "kind": "build", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "legacy-dashboard 项目的依赖目录，项目本身 7 个月没改过。", + "why": "package-lock.json 还在，随时能 npm install 回来。", + "if_removed": "再打开这个项目前要先装一遍依赖。", + "action": "trash", + "trash_paths": ["$HOME/projects/legacy-dashboard/node_modules"], + "restore": "cd 到项目目录跑 npm install。", + "commands": ["cd \"$HOME/projects/legacy-dashboard\" && npm install"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-chrome-profile", + "name": "Chrome 用户数据", + "path_portable": "$HOME/Library/Application Support/Google/Chrome", + "size_bytes": 9663676416, + "kind": "app-data", + "tier": "user-data", + "color": "yellow", + "disk": "/", + "what": "Chrome 的配置、扩展、历史、登录态和缓存都在这里。", + "why": "里面有你所有网站的登录状态，整目录删掉等于重装浏览器。", + "if_removed": "所有网站要重新登录，扩展要重装，书签如果没同步就没了。", + "action": "manual", + "trash_paths": [], + "restore": "只能靠 Google 账号同步找回一部分。", + "commands": [], + "kill_processes": [], + "open_by": [], + "note": "想腾空间就在 Chrome 设置里清缓存，别动这个目录。" + }, + { + "id": "st-dl-installers", + "name": "Downloads 里的安装包", + "path_portable": "$HOME/Downloads", + "size_bytes": 6442450944, + "kind": "installer", + "tier": "cold", + "color": "yellow", + "disk": "/", + "what": "6 个 dmg 和 pkg，对应的软件都已经在应用程序里了。", + "why": "装过就没用了，但也许你想留着某个特定版本。", + "if_removed": "以后要重装得重新下载。", + "action": "trash", + "trash_paths": ["$HOME/Downloads/Docker.dmg", "$HOME/Downloads/Cursor-0.52.dmg", "$HOME/Downloads/Obsidian-1.8.pkg"], + "restore": "废纸篓里放回原处。", + "commands": [], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-ios-backups", + "name": "iPhone 备份", + "path_portable": "$HOME/Library/Application Support/MobileSync/Backup", + "size_bytes": 6479380480, + "kind": "app-data", + "tier": "user-data", + "color": "yellow", + "disk": "/", + "what": "Finder 给 iPhone 做的本地整机备份，最近一次是 3 个月前。", + "why": "如果你已经用 iCloud 备份，这份就是多余的；如果没有，它是唯一的备份。", + "if_removed": "手机出问题时没有本地备份可以恢复。", + "action": "manual", + "trash_paths": [], + "restore": "无法恢复，只能重新备份。", + "commands": [], + "kill_processes": [], + "open_by": [], + "note": "在 Finder 里点手机，进管理备份，确认过再删。" + }, + { + "id": "st-photos", + "name": "Photos Library.photoslibrary", + "path_portable": "$HOME/Pictures/Photos Library.photoslibrary", + "size_bytes": 48318382080, + "kind": "media", + "tier": "forbidden", + "color": "red", + "disk": "/", + "what": "照片 app 的整个图库，原图、缩略图和数据库都在里面。", + "why": "禁刀区。在 Finder 里动它任何一部分都会把图库弄坏。", + "if_removed": "照片全丢，iCloud 没开优化存储的话没得找回。", + "action": "keep", + "trash_paths": [], + "restore": "无法恢复。", + "commands": [], + "kill_processes": [], + "open_by": [{"command": "photolibraryd", "pid": 512}] + }, + { + "id": "st-tm-snapshots", + "name": "Time Machine 本地快照", + "path_portable": "/System/Volumes/Data/.MobileBackups", + "size_bytes": 42949672960, + "kind": "system", + "tier": "forbidden", + "color": "red", + "disk": "/", + "what": "备份盘不在身边时 macOS 自己留的本地快照。", + "why": "系统管理的空间，磁盘紧张时会自动腾出来，手动删没意义。", + "if_removed": "不该手动删。真要释放，接上备份盘跑一次备份。", + "action": "keep", + "trash_paths": [], + "restore": "不适用。", + "commands": ["tmutil listlocalsnapshots /"], + "kill_processes": [], + "open_by": [] + }, + { + "id": "st-mail-containers", + "name": "邮件 app 数据", + "path_portable": "$HOME/Library/Containers/com.apple.mail", + "size_bytes": 5368709120, + "kind": "app-data", + "tier": "in_use", + "color": "red", + "disk": "/", + "what": "邮件 app 下载到本地的邮件和附件。", + "why": "现在有进程开着它，而且里面是你的邮件，删了要重新从服务器拉。", + "if_removed": "邮件 app 要重新同步所有账号，POP 账号的邮件直接没了。", + "action": "keep", + "trash_paths": [], + "restore": "只能从邮件服务器重新同步。", + "commands": [], + "kill_processes": [], + "open_by": [{"command": "Mail", "pid": 2210}] + } + ], + "executed_ids": [] +} diff --git a/tests/test_cfo_carl_file_organizer.py b/tests/test_cfo_carl_file_organizer.py new file mode 100644 index 0000000..2332ea7 --- /dev/null +++ b/tests/test_cfo_carl_file_organizer.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import tempfile +import unittest +from datetime import datetime, timedelta +from pathlib import Path + +from carl_file_organizer.classifier import classify +from carl_file_organizer.config import resolve_config +from carl_file_organizer.executor import PlanError, apply_approved_plan +from carl_file_organizer.planner import build_plan +from carl_file_organizer.render import render + +# The safety net has nowhere to look in these tests: no launch agents, no +# symlink scan roots, no lsof. Behaviour must not depend on the machine. +GUARD_OFFLINE = {"reference_files": [], "scan_roots": [], "lsof_enabled": False} + + +def simple_config(root: Path): + return resolve_config(root, profile="simple", lang="en") + + +def plan_for(root: Path, **kwargs): + # A day of imaginary distance, so freshly written fixture files are past the + # ten-minute "just touched it" guard. + later = datetime.now().astimezone() + timedelta(days=1) + return build_plan( + root, + config=kwargs.pop("config", None) or simple_config(root), + home=kwargs.pop("home", root), + now=kwargs.pop("now", later), + guard_options=kwargs.pop("guard_options", GUARD_OFFLINE), + **kwargs + ) + + +def action_for(plan, filename, kind="move"): + return next( + action + for action in plan["actions"] + if action["filename"] == filename and action["kind"] == kind + ) + + +class CarlFileOrganizerTests(unittest.TestCase): + def test_classifier_is_small_and_predictable(self) -> None: + self.assertEqual(classify(Path("paper.PDF"))[0], "Documents") + self.assertEqual(classify(Path("photo.heic"))[0], "Images") + self.assertEqual(classify(Path("unknown.blob"))[0], "Other") + + def test_plan_is_shallow_and_ignores_boundaries(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "notes.txt").write_text("hello", encoding="utf-8") + (root / ".secret.txt").write_text("hidden", encoding="utf-8") + nested = root / "Already Organized" + nested.mkdir() + (nested / "inside.pdf").write_text("stay", encoding="utf-8") + + plan = plan_for(root) + + self.assertEqual(plan["summary"]["files"], 1) + self.assertEqual( + action_for(plan, "notes.txt")["destination"], + str(root / "Documents" / "notes.txt"), + ) + self.assertEqual(plan["summary"]["proposed_reclaimed_bytes"], 0) + # the folder travels whole, and nothing inside it is filed separately + self.assertEqual(action_for(plan, "Already Organized")["subject_kind"], "dir") + self.assertTrue((nested / "inside.pdf").exists()) + + def test_duplicate_detection_hashes_equal_size_candidates(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "one.txt").write_text("same", encoding="utf-8") + (root / "two.txt").write_text("same", encoding="utf-8") + (root / "other.txt").write_text("diff", encoding="utf-8") + + plan = plan_for(root, hash_duplicates=True) + + groups = [group for group in plan["groups"] if group["kind"] == "duplicate"] + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["evidence"], "sha256") + self.assertEqual(groups[0]["potential_bytes"], 4) + + def test_only_approved_action_moves(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = root / "first.pdf" + second = root / "second.jpg" + first.write_text("document", encoding="utf-8") + second.write_text("image", encoding="utf-8") + config = simple_config(root) + plan = plan_for(root, config=config) + chosen = action_for(plan, "first.pdf") + plan["approved_action_ids"] = [chosen["id"]] + + report = apply_approved_plan(plan, config=config, home=root) + + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue((root / "Documents" / "first.pdf").exists()) + self.assertTrue(second.exists()) + + def test_unknown_approval_id_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "first.pdf").write_text("document", encoding="utf-8") + plan = plan_for(root) + plan["approved_action_ids"] = ["invented"] + with self.assertRaises(PlanError): + apply_approved_plan(plan, home=root) + + def test_tampered_destination_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "first.pdf").write_text("document", encoding="utf-8") + config = simple_config(root) + plan = plan_for(root, config=config) + action = action_for(plan, "first.pdf") + plan["approved_action_ids"] = [action["id"]] + action["destination"] = str(root / "Other" / "renamed.pdf") + action["destination_portable"] = "$HOME/Other/renamed.pdf" + with self.assertRaises(PlanError): + apply_approved_plan(plan, config=config, home=root) + + def test_report_embeds_readable_plan_and_export(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "paper.pdf").write_text("document", encoding="utf-8") + plan = plan_for(root) + html = render(plan, mode="static") + self.assertIn("paper.pdf", html) + self.assertIn(json.dumps(plan["schema_version"]), html) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_classifier.py b/tests/test_cfo_classifier.py new file mode 100644 index 0000000..fe326ff --- /dev/null +++ b/tests/test_cfo_classifier.py @@ -0,0 +1,140 @@ +"""Rule matching, and above all the sensitive naming rules. + +The interesting cases are the near misses: a short word must never win by +substring, because that is exactly how ``r2`` once swallowed ``pixverse-r2.md``. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import unittest +from pathlib import Path + +from carl_file_organizer import classifier +from carl_file_organizer.config import resolve_config +from carl_file_organizer.classifier import ( + classify, + derivative_hint, + is_regenerable_name, + is_sensitive, + match_name_pattern, + match_pinned, + match_rule, +) + + +def tiered(): + return resolve_config(Path.cwd(), profile="tiered", lang="zh") + + +class ClassifierTests(unittest.TestCase): + def setUp(self) -> None: + self.config = tiered() + + # -- the original taxonomy stays untouched --------------------------- + def test_legacy_classify_still_answers(self) -> None: + self.assertEqual(classify(Path("paper.PDF"))[0], "Documents") + self.assertEqual(classify(Path("unknown.blob"))[0], "Other") + + # -- extension rules -------------------------------------------------- + def test_match_rule_reads_the_profile(self) -> None: + rule = match_rule(".pdf", self.config) + self.assertIsNotNone(rule) + self.assertEqual(rule.dest_key, "library.docs.pdf") + self.assertEqual(rule.min_age_hours, 48) + + def test_archive_rules_carry_the_longer_settling_period(self) -> None: + self.assertEqual(match_rule(".zip", self.config).min_age_hours, 168) + self.assertEqual(match_rule(".dmg", self.config).min_age_hours, 168) + + def test_compound_suffix_is_a_rule_of_its_own(self) -> None: + rule = match_rule(".tar.gz", self.config) + self.assertIsNotNone(rule) + self.assertEqual(rule.dest_key, "archive.zip.other") + + def test_unknown_suffix_has_no_rule(self) -> None: + self.assertIsNone(match_rule(".parquet", self.config)) + + # -- sensitive naming -------------------------------------------------- + def test_sensitive_exact_never_matches_a_substring(self) -> None: + self.assertIsNotNone(is_sensitive("key.txt", self.config)) + self.assertIsNone(is_sensitive("monkey.txt", self.config)) + self.assertIsNone(is_sensitive("pixverse-r2.md", self.config)) + self.assertIsNone(is_sensitive("keyboard-shortcuts.md", self.config)) + + def test_sensitive_layers_are_reported(self) -> None: + self.assertEqual(is_sensitive("id_rsa", self.config).where, "exact_names") + self.assertEqual(is_sensitive("key.txt", self.config).where, "exact_stems") + self.assertEqual(is_sensitive("server.pem", self.config).layer, "suffix") + hit = is_sensitive("my-password-list.md", self.config) + self.assertEqual(hit.layer, "marker") + self.assertEqual(hit.needle, "password") + + def test_dot_env_matches_as_a_suffix_and_as_a_whole_name(self) -> None: + self.assertIsNotNone(is_sensitive("prod.env", self.config)) + self.assertIsNotNone(is_sensitive(".env", self.config)) + + def test_chinese_markers_match_as_substrings(self) -> None: + self.assertIsNotNone(is_sensitive("公司营业执照扫描件.pdf", self.config)) + self.assertIsNotNone(is_sensitive("阿里云密钥备份.txt", self.config)) + + def test_a_short_ascii_marker_is_refused_at_load_time(self) -> None: + from carl_file_organizer.config import merge_profile + + with self.assertRaises(ValueError): + merge_profile(self.config.profile, {"sensitive": {"markers": ["r2"]}}) + + # -- derived copies, regenerable names, pinned ------------------------- + def test_derivative_hint(self) -> None: + self.assertEqual(derivative_hint("web-prototype-fix-20260830", self.config), "-fix-") + self.assertEqual(derivative_hint("repo-worktree-topic", self.config), "-worktree-") + self.assertIsNone(derivative_hint("fixture.pdf", self.config)) + + def test_is_regenerable_name(self) -> None: + self.assertTrue(is_regenerable_name("node_modules", self.config)) + self.assertTrue(is_regenerable_name("__pycache__", self.config)) + self.assertFalse(is_regenerable_name("src", self.config)) + + def test_match_pinned_takes_exact_names_and_globs(self) -> None: + self.assertEqual( + match_pinned("carl-file-organizer-approved-1.json", self.config), "carl-file-organizer-*.json" + ) + self.assertIsNone(match_pinned("notes.md", self.config)) + + def test_pinned_from_the_settings_file_is_exact(self) -> None: + config = resolve_config(Path.cwd(), profile="tiered", lang="zh") + config.profile["pinned"] = list(config.profile["pinned"]) + ["fable51-table"] + self.assertEqual(match_pinned("fable51-table", config), "fable51-table") + self.assertIsNone(match_pinned("fable51-table-copy", config)) + + # -- user written name rules ------------------------------------------ + def test_name_patterns_respect_suffix_filters(self) -> None: + config = tiered() + config.profile["name_patterns"] = [ + { + "id": "reports-sql", + "match": "contains", + "needle": "reports", + "suffixes": [".sql"], + "dest": "10_工作区/数据库与查询/SQL", + } + ] + self.assertIsNotNone(match_name_pattern("monthly-reports.sql", config)) + self.assertIsNone(match_name_pattern("monthly-reports.pdf", config)) + + def test_name_patterns_refuse_a_three_letter_substring(self) -> None: + config = tiered() + config.profile["name_patterns"] = [ + {"id": "tiny", "match": "contains", "needle": "abc", "dest": "00_收件箱/待判断"} + ] + self.assertIsNone(match_name_pattern("abcdef.md", config)) + + def test_archive_and_installer_suffix_lookups(self) -> None: + self.assertTrue(classifier.is_archive_suffix(".tar.gz", self.config)) + self.assertTrue(classifier.is_installer_suffix(".dmg", self.config)) + self.assertFalse(classifier.is_installer_suffix(".pdf", self.config)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_cli_flow.py b/tests/test_cfo_cli_flow.py new file mode 100644 index 0000000..072bb66 --- /dev/null +++ b/tests/test_cfo_cli_flow.py @@ -0,0 +1,351 @@ +"""scan -> report -> apply: the three commands a person actually types. + +Everything happens in a temporary folder with the whole-machine scan pointed at +a fake home directory, so the test never reads this machine and never writes +outside its own sandbox. ``apply`` runs with ``--dry-run`` only; a test that +moves real files is what ``test_cfo_executor`` is for. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import contextlib +import io +import json +import os +import tempfile +import unittest +from datetime import datetime, timedelta +from pathlib import Path + +from carl_file_organizer import flow +from carl_file_organizer.cli import main + +MANAGED_ZH = "00_下载目录管理" + + +def _run(argv): + """Run one subcommand and hand back ``(exit code, everything it printed)``.""" + + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = main(argv) + return code, out.getvalue() + err.getvalue() + + +class CombinedFlowTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.source = self.root / "downloads" + self.source.mkdir() + for name in ("quarterly-report.pdf", "screenshot.png", "notes.txt", "thing.weirdext"): + path = self.source / name + path.write_text("placeholder\n", encoding="utf-8") + # a fresh download is still settling and would produce no moves at + # all, which is a fine plan and a poor test fixture + old = (datetime.now() - timedelta(days=30)).timestamp() + os.utime(str(path), (old, old)) + self.fake_home = self.root / "fakehome" + (self.fake_home / "Library" / "Caches" / "pip").mkdir(parents=True) + (self.fake_home / "Library" / "Caches" / "pip" / "wheel.bin").write_bytes(b"0" * 4096) + self.managed = self.source / MANAGED_ZH + + # -- step 1 ------------------------------------------------------------- + + def test_scan_writes_both_halves_into_one_managed_directory(self) -> None: + code, text = _run( + [ + "scan", str(self.source), "--lang", "zh", "--allow-outside-home", + "--budget-seconds", "5", "--home", str(self.fake_home), + ] + ) + self.assertEqual(code, 0, text) + self.assertTrue((self.managed / "plan.json").is_file()) + self.assertTrue((self.managed / "storage-scan.json").is_file()) + # scan renders nothing; report is the one thing that writes a page + self.assertFalse((self.managed / "report.html").exists()) + self.assertIn("plan.json", text) + self.assertIn("storage-scan.json", text) + self.assertIn("analysis.json", text) + scan = json.loads((self.managed / "storage-scan.json").read_text(encoding="utf-8")) + self.assertIn("disks", scan) + + def test_no_storage_leaves_the_whole_machine_half_out(self) -> None: + code, text = _run(["scan", str(self.source), "--lang", "zh", "--allow-outside-home", "--no-storage"]) + self.assertEqual(code, 0, text) + self.assertTrue((self.managed / "plan.json").is_file()) + self.assertFalse((self.managed / "storage-scan.json").exists()) + + # -- step 2 ------------------------------------------------------------- + + def _scan_and_annotate(self) -> None: + code, text = _run(["scan", str(self.source), "--lang", "zh", "--allow-outside-home", "--no-storage"]) + self.assertEqual(code, 0, text) + plan = json.loads((self.managed / "plan.json").read_text(encoding="utf-8")) + (self.managed / "notes.json").write_text( + json.dumps( + { + "folder_line": "上周做提案时下载的一堆东西。", + "actions": { + action["id"]: { + "what": "一份提案用到的文件", + "why": "扩展名规则命中", + "if_removed": "只是搬家，内容不动", + } + for action in plan["actions"] + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (self.managed / "analysis.json").write_text( + json.dumps( + { + "schema": "carl-file-organizer/storage-analysis", + "schema_version": 1, + "lang": "zh", + "created_at": "2026-09-02T12:00:00+09:00", + "machine": {"hostname": "test-machine", "os": "macOS"}, + "disks": [ + { + "name": "Macintosh HD", + "mount": "/", + "total_bytes": 500 * 1024 ** 3, + "used_bytes": 400 * 1024 ** 3, + "free_bytes": 100 * 1024 ** 3, + "used_percent": 80.0, + "primary": True, + } + ], + "overview": { + "headline": "系统盘用了八成，先清缓存。", + "priority": [{"text": "pip 缓存可以直接清掉。"}], + "long_term": ["把老仓库搬到外置盘。"], + }, + "items": [ + { + "id": "st-pip", + "name": "pip cache", + "path_portable": "$HOME/Library/Caches/pip", + "size_bytes": 4096, + "kind": "cache", + "tier": "regenerable", + "color": "green", + "disk": "/", + "what": "pip 装包时留下的缓存", + "why": "删了下次装包重新下载", + "if_removed": "第一次装包慢一点", + "action": "trash", + "trash_paths": ["$HOME/Library/Caches/pip"], + "restore": "再装一次包就有了", + } + ], + "top5": [ + {"id": "st-pip", "name": "pip cache", "size_bytes": 4096, "kind": "cache", "color": "green"} + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + def test_report_renders_one_combined_page(self) -> None: + self._scan_and_annotate() + code, text = _run(["report", str(self.managed)]) + self.assertEqual(code, 0, text) + page = self.managed / "report.html" + self.assertTrue(page.is_file()) + html = page.read_text(encoding="utf-8") + self.assertIn('data-kind="combined"', html) + self.assertNotIn("/Users/", html) + # both halves and the after picture + self.assertIn("上周做提案时下载的一堆东西。", html) + self.assertIn("pip 装包时留下的缓存", html) + self.assertIn("pip 缓存可以直接清掉。", html) + self.assertEqual(html.count('id="cfo-preview"'), 1) + self.assertIn("carl-file-organizer-decisions.json", html) + + def test_report_takes_an_output_path_and_an_analysis_elsewhere(self) -> None: + self._scan_and_annotate() + moved = self.root / "elsewhere.json" + moved.write_text((self.managed / "analysis.json").read_text(encoding="utf-8"), encoding="utf-8") + (self.managed / "analysis.json").unlink() + out = self.root / "combined.html" + code, text = _run(["report", str(self.managed), "--analysis", str(moved), "-o", str(out)]) + self.assertEqual(code, 0, text) + self.assertIn('data-kind="combined"', out.read_text(encoding="utf-8")) + self.assertIn("pip 装包时留下的缓存", out.read_text(encoding="utf-8")) + + def test_report_serve_hands_both_halves_to_one_server(self) -> None: + self._scan_and_annotate() + from carl_file_organizer import server as server_module + + seen = {} + + def fake_serve(document, **kw): + seen["document"] = document + seen["kw"] = kw + return 0 + + original = server_module.serve_review + server_module.serve_review = fake_serve + self.addCleanup(setattr, server_module, "serve_review", original) + + code, text = _run(["report", str(self.managed), "--serve", "--no-open", "--port", "0"]) + self.assertEqual(code, 0, text) + self.assertIn("plan", seen["document"]) + self.assertIn("analysis", seen["document"]) + self.assertEqual(seen["kw"]["managed_dir"], self.managed) + self.assertIs(seen["kw"]["open_browser"], False) + # notes.json in the managed folder is picked up without being named + self.assertIn("actions", seen["kw"]["notes"]) + # the page is served, never written next to the plan + self.assertFalse((self.managed / "report.html").exists()) + + def test_report_without_either_half_says_so(self) -> None: + empty = self.root / "empty" + empty.mkdir() + code, text = _run(["report", str(empty)]) + self.assertEqual(code, 2) + self.assertIn("scan", text) + + # -- step 3 ------------------------------------------------------------- + + def _decisions(self) -> Path: + plan = json.loads((self.managed / "plan.json").read_text(encoding="utf-8")) + analysis = json.loads((self.managed / "analysis.json").read_text(encoding="utf-8")) + approvable = [a["id"] for a in plan["actions"] if a["kind"] == "move"][:2] + target = self.root / "carl-file-organizer-decisions.json" + target.write_text( + json.dumps( + { + "schema": flow.DECISIONS_SCHEMA, + "schema_version": 1, + "decided_at": "2026-09-02T12:00:00+09:00", + "decided_by": "html-static", + "plan": {"approved_action_ids": approvable, "overrides": [], "document": plan}, + "storage": { + "item_ids": ["st-pip"], + "actions": {"st-pip": "trash"}, + "document": analysis, + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + return target + + def test_apply_dry_run_walks_both_halves_and_touches_nothing(self) -> None: + self._scan_and_annotate() + decisions = self._decisions() + before = sorted(p.name for p in self.source.iterdir()) + code, text = _run( + ["apply", str(decisions), "--dry-run", "--lang", "zh", "--managed-dir", str(self.managed)] + ) + self.assertEqual(code, 0, text) + # one summary per half, the moves first and the cleanup second + moves_at = text.index("先搬动这一半") + cleanup_at = text.index("再清理这一半") + self.assertLess(moves_at, cleanup_at) + self.assertIn("dry-run", text) + self.assertIn("两段合计", text) + self.assertEqual(sorted(p.name for p in self.source.iterdir()), before) + + def test_an_older_approved_file_still_applies(self) -> None: + self._scan_and_annotate() + plan = json.loads((self.managed / "plan.json").read_text(encoding="utf-8")) + plan["approved_action_ids"] = [a["id"] for a in plan["actions"] if a["kind"] == "move"][:1] + plan["overrides"] = [] + approved = self.root / "carl-file-organizer-approved.json" + approved.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + code, text = _run(["apply", str(approved), "--dry-run"]) + self.assertEqual(code, 0, text) + self.assertIn("dry-run", text) + + def test_a_decisions_file_with_nothing_ticked_does_nothing(self) -> None: + self._scan_and_annotate() + empty = self.root / "empty-decisions.json" + empty.write_text( + json.dumps( + { + "schema": flow.DECISIONS_SCHEMA, + "plan": {"approved_action_ids": [], "overrides": []}, + "storage": {"item_ids": [], "actions": {}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + code, text = _run(["apply", str(empty), "--lang", "zh"]) + self.assertEqual(code, 0, text) + self.assertIn("一项都没勾", text) + + def test_a_missing_analysis_is_refused_before_anything_runs(self) -> None: + self._scan_and_annotate() + lonely = self.root / "no-analysis.json" + lonely.write_text( + json.dumps( + { + "schema": flow.DECISIONS_SCHEMA, + "plan": {"approved_action_ids": [], "overrides": []}, + "storage": {"item_ids": ["st-pip"], "actions": {"st-pip": "trash"}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + code, text = _run(["apply", str(lonely), "--lang", "zh"]) + self.assertEqual(code, 2) + self.assertIn("analysis.json", text) + + +class ManagedDirectoryTests(unittest.TestCase): + """The cleanup's paper trail joins the tidy-up half's folder, not a corner of its own.""" + + def setUp(self) -> None: + self.plan = json.loads( + (Path(cfo_path.FIXTURES_DIR) / "plan-v2-sample.json").read_text(encoding="utf-8") + ) + + def test_an_absolute_managed_dir_is_used_as_written(self) -> None: + found = flow._managed_of(self.plan) + self.assertIsNotNone(found) + self.assertEqual(found.name, "00_下载目录管理") + + def test_a_stripped_plan_still_gives_up_its_managed_dir(self) -> None: + import build_report + + stripped = build_report.sanitize(self.plan) + self.assertNotIn("managed_dir", stripped) + found = flow._managed_of(stripped) + self.assertIsNotNone(found) + self.assertEqual(found.name, "00_下载目录管理") + self.assertTrue(found.is_absolute()) + + def test_a_plan_without_either_field_means_the_old_location(self) -> None: + self.assertIsNone(flow._managed_of({})) + + +class CollectTests(unittest.TestCase): + """``report`` reads the Agent's analysis.json, never the raw scan.""" + + def test_a_raw_scan_alone_is_not_enough(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + managed = Path(tmp) + (managed / "storage-scan.json").write_text('{"disks": []}', encoding="utf-8") + self.assertEqual(flow.collect(managed), {}) + + def test_an_analysis_named_on_the_command_line_must_exist(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + managed = Path(tmp) + with self.assertRaises(flow.FlowError): + flow.collect(managed, managed / "nope.json") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_colors.py b/tests/test_cfo_colors.py new file mode 100644 index 0000000..9a5228f --- /dev/null +++ b/tests/test_cfo_colors.py @@ -0,0 +1,328 @@ +"""The three colours: one per action, one per group, one for the folder. + +The colour is what the review page turns into a button, so these tests care +about one property above all others: no colour is ever written by hand. It is +derived from the decision the rules already made, which means a rule change can +break a test here but a wording change never can. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import planner +from carl_file_organizer.config import resolve_config +from carl_file_organizer.planner import ( + COLORS, + GREEN, + MESS_THRESHOLDS, + RED, + YELLOW, + action_color, + build_plan, + color_totals, + mess_color, + mess_counts, + mess_score, +) + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + action_for, + actions_for, + age, + write, +) + + +def plan_for(root: Path, **kwargs): + config = kwargs.pop("config", None) or resolve_config(root, profile="tiered", lang="zh") + return build_plan( + root, + config=config, + now=kwargs.pop("now", NOW), + home=kwargs.pop("home", root), + guard_options=kwargs.pop("guard_options", GUARD_OFFLINE), + trash_backend=kwargs.pop("trash_backend", "finder"), + **kwargs + ) + + +class ActionColorTests(unittest.TestCase): + """The pure function, with no plan around it.""" + + def test_every_hold_is_red_whatever_held_it(self) -> None: + for tier in ("forbidden", "in_use", "referenced", "aging", "pinned"): + self.assertEqual(action_color({"kind": "hold", "tier": tier}), RED, tier) + + def test_a_settled_move_is_green_and_a_pending_one_is_yellow(self) -> None: + settled = {"kind": "move", "tier": "routine", "reroutable": False} + pending = {"kind": "move", "tier": "routine", "reroutable": True} + self.assertEqual(action_color(settled), GREEN) + self.assertEqual(action_color(pending), YELLOW) + + def test_a_sensitive_move_is_green_because_isolating_is_the_safe_move(self) -> None: + action = {"kind": "move", "tier": "sensitive", "reroutable": False} + self.assertEqual(action_color(action), GREEN) + + def test_a_move_inside_a_group_is_green_the_disposal_next_to_it_is_not(self) -> None: + self.assertEqual( + action_color({"kind": "move", "tier": "duplicate", "reroutable": False}), GREEN + ) + for kind in ("trash", "delete"): + self.assertEqual(action_color({"kind": kind, "tier": "duplicate"}), YELLOW, kind) + + def test_only_regenerable_disposal_is_green(self) -> None: + for kind in ("trash", "delete"): + self.assertEqual(action_color({"kind": kind, "tier": "regenerable"}), GREEN, kind) + self.assertEqual(action_color({"kind": kind, "tier": "cold"}), YELLOW, kind) + + +class ColorInAPlanTests(unittest.TestCase): + def test_a_filed_document_is_green_and_an_unknown_extension_is_yellow(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + plan = plan_for(root) + + self.assertEqual(action_for(plan, "notes.md")["color"], GREEN) + self.assertEqual(action_for(plan, "mystery.zzz")["color"], YELLOW) + + def test_a_settling_archive_is_red_while_it_waits(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "bundle.zip", "PK\x03\x04"), hours=0.2) + plan = plan_for(root) + + hold = action_for(plan, "bundle.zip", kind="hold") + self.assertEqual(hold["tier"], "aging") + self.assertEqual(hold["color"], RED) + self.assertFalse(hold["approvable"]) + + def test_no_red_action_is_ever_approvable(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "id_rsa"), hours=100) + age(write(root / "fresh.pdf"), hours=0.1) + plan = plan_for(root) + + reds = [a for a in plan["actions"] if a["color"] == RED] + self.assertTrue(reds) + for action in reds: + self.assertFalse(action["approvable"], action["id"]) + self.assertEqual(action["kind"], "hold") + + def test_a_pair_group_is_a_yellow_question(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "release.zip", "PK\x03\x04"), hours=100) + age(write(root / "release" / "readme.txt"), hours=100) + age(root / "release", hours=100) + plan = plan_for(root) + + self.assertTrue(plan["groups"]) + for group in plan["groups"]: + self.assertEqual(group["kind"], "pair", group["group_id"]) + self.assertEqual(group["color"], YELLOW, group["group_id"]) + # the moves inside the group stay green; only the disposals are yellow + moves = [a for a in plan["actions"] if a["kind"] == "move" and a["group_id"]] + self.assertTrue(moves) + for action in moves: + self.assertEqual(action["color"], GREEN, action["id"]) + + def test_a_regenerable_group_is_green_like_its_own_options(self) -> None: + """Build output rebuilds itself, so the question does not need a look.""" + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "web-prototype" / "index.html"), hours=100) + age(write(root / "web-prototype" / "node_modules" / "left-pad" / "index.js"), hours=100) + age(root / "web-prototype" / "node_modules" / "left-pad", hours=100) + age(root / "web-prototype" / "node_modules", hours=100) + age(root / "web-prototype", hours=100) + plan = plan_for(root) + + regen = [g for g in plan["groups"] if g["kind"] == "regenerable"] + self.assertTrue(regen) + for group in regen: + self.assertEqual(group["color"], GREEN, group["group_id"]) + member_ids = set() + for option in group["options"]: + member_ids.update(option["action_ids"]) + self.assertTrue(member_ids) + for action in plan["actions"]: + if action["id"] in member_ids: + self.assertEqual(action["color"], GREEN, action["id"]) + + def test_a_duplicate_head_keeps_its_colour_after_the_tier_is_rewritten(self) -> None: + """build_plan rewrites the tier of a duplicate's move after the fact.""" + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf", "same body\n"), hours=100) + age(write(root / "report copy.pdf", "same body\n"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + heads = [a for a in plan["actions"] if a["kind"] == "move" and a["group_id"]] + self.assertTrue(heads) + for head in heads: + self.assertEqual(head["tier"], "duplicate") + self.assertFalse(head["reroutable"]) + self.assertEqual(head["color"], GREEN, head["id"]) + for action in actions_for(plan, "report copy.pdf"): + if action["kind"] in ("trash", "delete"): + self.assertEqual(action["color"], YELLOW, action["id"]) + + def test_by_color_adds_up_to_every_action(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "fresh.pdf"), hours=0.1) + plan = plan_for(root) + + by_color = plan["summary"]["by_color"] + self.assertEqual(set(by_color), set(COLORS)) + self.assertEqual( + sum(slot["actions"] for slot in by_color.values()), len(plan["actions"]) + ) + for colour in COLORS: + wearing = [a for a in plan["actions"] if a["color"] == colour] + self.assertEqual(by_color[colour]["actions"], len(wearing), colour) + self.assertEqual( + by_color[colour]["bytes"], + sum(a["size_bytes"] for a in wearing), + colour, + ) + + def test_color_totals_ignores_an_action_with_no_colour(self) -> None: + totals = color_totals([{"color": "chartreuse", "size_bytes": 10}]) + self.assertEqual(sum(slot["actions"] for slot in totals.values()), 0) + + +class MessTests(unittest.TestCase): + def test_thresholds_are_the_ones_written_down(self) -> None: + self.assertEqual(MESS_THRESHOLDS[GREEN], {"loose": 3, "pending": 0}) + self.assertEqual(MESS_THRESHOLDS[YELLOW], {"loose": 15, "pending": 5}) + + def test_a_tidy_folder_is_green(self) -> None: + counts = {"loose": 3, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), GREEN) + + def test_one_thing_waiting_on_you_takes_green_away(self) -> None: + counts = {"loose": 2, "aging": 0, "pending": 1, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), YELLOW) + + def test_a_big_pile_with_few_questions_is_still_only_yellow(self) -> None: + counts = {"loose": 200, "aging": 0, "pending": 4, "groups": 0, "top_dirs": 0} + self.assertEqual(mess_color(counts), YELLOW) + + def test_many_items_and_many_questions_is_red(self) -> None: + counts = {"loose": 40, "aging": 3, "pending": 12, "groups": 4, "top_dirs": 9} + self.assertEqual(mess_color(counts), RED) + + def test_score_is_bounded_and_grows_with_the_mess(self) -> None: + tidy = {"loose": 1, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + awful = {"loose": 900, "aging": 90, "pending": 400, "groups": 60, "top_dirs": 80} + self.assertLess(mess_score(tidy), mess_score(awful)) + self.assertGreaterEqual(mess_score(tidy), 0) + self.assertLessEqual(mess_score(awful), 100) + + def test_the_bar_never_pegs_so_it_can_always_move(self) -> None: + """A merely untidy folder must not use up the whole bar.""" + + untidy = {"loose": 27, "aging": 2, "pending": 3, "groups": 4, "top_dirs": 3} + self.assertEqual(mess_color(untidy), YELLOW) + self.assertLess(mess_score(untidy), 90) + + landslide = {"loose": 400, "aging": 30, "pending": 90, "groups": 40, "top_dirs": 40} + self.assertEqual(mess_color(landslide), RED) + self.assertGreater(mess_score(landslide), mess_score(untidy)) + self.assertLess(mess_score(landslide), 100) + + def test_clearing_a_pile_visibly_moves_the_bar(self) -> None: + before = {"loose": 200, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + after = {"loose": 170, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertGreater(mess_score(before) - mess_score(after), 0) + + def test_loose_and_pending_are_not_counted_twice(self) -> None: + """They describe the same pile two ways; adding them would double it.""" + + counts = {"loose": 15, "aging": 0, "pending": 5, "groups": 0, "top_dirs": 0} + only_loose = dict(counts, pending=0) + self.assertEqual(mess_score(counts), mess_score(only_loose)) + + def test_score_and_colour_are_computed_separately(self) -> None: + """The bar length is not allowed to become the verdict, or the reverse.""" + + counts = {"loose": 200, "aging": 0, "pending": 0, "groups": 0, "top_dirs": 0} + self.assertGreater(mess_score(counts), 80) + self.assertEqual(mess_color(counts), YELLOW) + + def test_counts_come_from_the_scan_not_from_the_action_list(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "folder" / "inner.txt"), hours=100) + age(root / "folder", hours=100) + plan = plan_for(root) + + counts = plan["mess"]["counts"] + self.assertEqual(counts["loose"], plan["summary"]["entries"]) + self.assertEqual(counts["top_dirs"], plan["summary"]["dirs"]) + self.assertEqual(counts["groups"], len(plan["groups"])) + self.assertEqual( + counts["pending"], + len({a["subject_id"] for a in plan["actions"] if a["reroutable"]}), + ) + + def test_a_plan_says_out_loud_who_chose_the_folder_colour(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + + self.assertEqual(plan["mess"]["color_source"], "rule") + self.assertIn(plan["mess"]["color"], COLORS) + self.assertEqual(set(plan["mess"]["reason"]), {"zh", "en"}) + self.assertIn(str(plan["mess"]["counts"]["loose"]), plan["mess"]["reason"]["zh"]) + + def test_an_empty_folder_is_green(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan = plan_for(root) + + self.assertEqual(plan["mess"]["color"], GREEN) + self.assertEqual(plan["mess"]["score"], 0) + self.assertEqual(plan["mess"]["counts"]["loose"], 0) + + def test_mess_counts_reads_entry_objects_and_actions_together(self) -> None: + class FakeEntry(object): + def __init__(self, is_dir: bool) -> None: + self.is_dir = is_dir + + entries = [FakeEntry(False), FakeEntry(True), FakeEntry(True)] + actions = [ + {"subject_id": "a", "kind": "hold", "tier": "aging", "reroutable": False}, + {"subject_id": "b", "kind": "move", "tier": "routine", "reroutable": True}, + {"subject_id": "b", "kind": "trash", "tier": "cold", "reroutable": False}, + ] + counts = mess_counts(entries, actions, [{"group_id": "g"}]) + self.assertEqual( + counts, {"loose": 3, "aging": 1, "pending": 1, "groups": 1, "top_dirs": 2} + ) + + def test_the_module_only_knows_three_colours(self) -> None: + self.assertEqual(planner.COLORS, ("green", "yellow", "red")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_config.py b/tests/test_cfo_config.py new file mode 100644 index 0000000..e168246 --- /dev/null +++ b/tests/test_cfo_config.py @@ -0,0 +1,461 @@ +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import os +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest import mock + +from carl_file_organizer import config, paths + + +def _write_dotfile(root: Path, payload: dict) -> Path: + path = root / config.DOTFILE_NAME + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +class LanguageDetectionTests(unittest.TestCase): + def setUp(self) -> None: + patcher = mock.patch.object(config, "_apple_languages", return_value=None) + patcher.start() + self.addCleanup(patcher.stop) + self.profile = config.load_builtin_profile("tiered") + + def test_lang_precedence(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "00_收件箱").mkdir() + + self.assertEqual( + config.detect_lang( + root, + "en", + env={config.LANG_ENV: "zh"}, + dotfile={"lang": "zh"}, + profile=self.profile, + ), + ("en", "flag"), + ) + self.assertEqual( + config.detect_lang( + root, + None, + env={config.LANG_ENV: "en"}, + dotfile={"lang": "zh"}, + profile=self.profile, + ), + ("zh", "dotfile"), + ) + self.assertEqual( + config.detect_lang( + root, None, env={config.LANG_ENV: "en"}, dotfile={}, profile=self.profile + ), + ("en", "env"), + ) + self.assertEqual( + config.detect_lang(root, None, env={}, dotfile={}, profile=self.profile), + ("zh", "existing-dirs"), + ) + + with tempfile.TemporaryDirectory() as temporary: + bare = Path(temporary) + self.assertEqual( + config.detect_lang( + bare, None, env={"LANG": "zh_CN.UTF-8"}, dotfile={}, profile=self.profile + ), + ("zh", "posix-locale"), + ) + self.assertEqual( + config.detect_lang(bare, None, env={}, dotfile={}, profile=self.profile), + ("en", "default"), + ) + + def test_apple_languages_beats_posix_locale(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + bare = Path(temporary) + with mock.patch.object(config, "_apple_languages", return_value="zh-Hans-CN"): + self.assertEqual( + config.detect_lang( + bare, None, env={"LANG": "en_US.UTF-8"}, dotfile={}, profile=self.profile + ), + ("zh", "apple-languages"), + ) + + def test_existing_dirs_sniff_zh_en(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + zh_root = Path(temporary) / "zh" + en_root = Path(temporary) / "en" + both_root = Path(temporary) / "both" + for folder in (zh_root, en_root, both_root): + folder.mkdir() + (zh_root / "00_收件箱").mkdir() + (en_root / "00_Inbox").mkdir() + (both_root / "00_收件箱").mkdir() + (both_root / "00_Inbox").mkdir() + + self.assertEqual( + config.detect_lang(zh_root, None, env={}, dotfile={}, profile=self.profile), + ("zh", "existing-dirs"), + ) + self.assertEqual( + config.detect_lang(en_root, None, env={}, dotfile={}, profile=self.profile), + ("en", "existing-dirs"), + ) + with self.assertRaises(config.LanguageAmbiguous): + config.detect_lang(both_root, None, env={}, dotfile={}, profile=self.profile) + + # an explicit language always wins over the ambiguous layout + self.assertEqual( + config.detect_lang(both_root, "zh", env={}, dotfile={}, profile=self.profile), + ("zh", "flag"), + ) + + +class ProfileMergeTests(unittest.TestCase): + def test_dotfile_merge_union_and_override(self) -> None: + base = config.load_builtin_profile("tiered") + merged = config.merge_profile( + base, + { + "lang": "zh", + "large_bytes": 1024, + "aging": {"default_hours": 12}, + "names": {"library.docs": {"zh": "文档"}}, + "rules": {".numbers": {"dest": "library.docs.csv"}}, + "pinned": ["notes.md"], + "sensitive": {"exact_names": ["work-key.txt"], "markers": ["subscription"]}, + "forbidden": {"basenames": ["Sync"]}, + "regenerable": {"basenames": ["vendor"]}, + "guard": {"reference_files": ["$HOME/.config/mytool.conf"], "recent_minutes": 30}, + "name_patterns": [ + {"id": "reports", "match": "contains", "needle": "report", "dest": "20_知识库/报告"} + ], + }, + ) + + # union keeps the built-in entries + self.assertIn("carl-file-organizer-*.json", merged["pinned"]) + self.assertIn("notes.md", merged["pinned"]) + self.assertIn(".env", merged["sensitive"]["exact_names"]) + self.assertIn("work-key.txt", merged["sensitive"]["exact_names"]) + self.assertIn("credential", merged["sensitive"]["markers"]) + self.assertIn("subscription", merged["sensitive"]["markers"]) + self.assertIn(".git", merged["forbidden"]["basenames"]) + self.assertIn("Sync", merged["forbidden"]["basenames"]) + self.assertIn("node_modules", merged["regenerable"]["basenames"]) + self.assertIn("vendor", merged["regenerable"]["basenames"]) + self.assertIn("$HOME", merged["guard"]["reference_scan_roots"]) + self.assertIn("$HOME/.config/mytool.conf", merged["guard"]["reference_files"]) + self.assertEqual([p["id"] for p in merged["name_patterns"]], ["reports"]) + + # deep merge on names / rules + self.assertEqual(merged["names"]["library.docs"]["zh"], "文档") + self.assertEqual(merged["names"]["library.docs"]["en"], "Documents") + self.assertEqual(merged["rules"][".numbers"]["dest"], "library.docs.csv") + self.assertEqual(merged["rules"][".pdf"]["dest"], "library.docs.pdf") + + # plain overrides + self.assertEqual(merged["lang"], "zh") + self.assertEqual(merged["large_bytes"], 1024) + self.assertEqual(merged["aging"]["default_hours"], 12) + self.assertEqual(merged["aging"]["archive_hours"], 168) + self.assertEqual(merged["guard"]["recent_minutes"], 30) + self.assertEqual(merged["guard"]["lsof_timeout_s"], 5) + + # the built-in profile is never mutated + self.assertNotIn("notes.md", base["pinned"]) + + def test_short_ascii_marker_is_refused(self) -> None: + base = config.load_builtin_profile("tiered") + with self.assertRaises(ValueError) as caught: + config.merge_profile(base, {"sensitive": {"markers": ["r2"]}}) + self.assertIn("exact_stems", str(caught.exception)) + # a short CJK marker is fine, it cannot collide the way a two-letter word does + merged = config.merge_profile(base, {"sensitive": {"markers": ["证件"]}}) + self.assertIn("证件", merged["sensitive"]["markers"]) + + def test_save_dotfile_only_if_missing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cfg = config.resolve_config(root, lang="zh", profile="tiered") + written = config.save_dotfile(cfg, only_if_missing=True) + self.assertIsNotNone(written) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8")), + {"profile": "tiered", "lang": "zh"}, + ) + + cfg.lang = "en" + self.assertIsNone(config.save_dotfile(cfg, only_if_missing=True)) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8"))["lang"], "zh" + ) + + self.assertIsNotNone(config.save_dotfile(cfg, only_if_missing=False)) + self.assertEqual( + json.loads(Path(written).read_text(encoding="utf-8"))["lang"], "en" + ) + + def test_dotfile_is_read_back_by_resolve_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + _write_dotfile(root, {"profile": "simple", "lang": "en", "pinned": ["keepme.txt"]}) + cfg = config.resolve_config(root) + self.assertEqual(cfg.profile_name, "simple") + self.assertEqual(cfg.lang, "en") + self.assertEqual(cfg.lang_source, "dotfile") + self.assertEqual(cfg.profile_source, "builtin+dotfile") + self.assertIn("keepme.txt", cfg.profile["pinned"]) + self.assertEqual(len(cfg.profile_hash), 12) + + def test_recorded_language_stays_out_of_the_settings_digest(self) -> None: + base = config.load_builtin_profile("tiered") + zh = config.merge_profile(base, {"profile": "tiered", "lang": "zh"}) + en = config.merge_profile(base, {"profile": "tiered", "lang": "en"}) + self.assertEqual(zh["lang"], "zh") + self.assertEqual(en["lang"], "en") + self.assertEqual(config.profile_hash(zh), config.profile_hash(en)) + self.assertEqual(config.profile_hash(zh), config.profile_hash(base)) + + def test_an_edited_rule_changes_the_settings_digest(self) -> None: + base = config.load_builtin_profile("tiered") + edited = config.merge_profile(base, {"pinned": ["something-new"]}) + self.assertNotEqual(config.profile_hash(base), config.profile_hash(edited)) + + def test_broken_dotfile_reports_position(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / config.DOTFILE_NAME).write_text('{"lang": ', encoding="utf-8") + with self.assertRaises(ValueError) as caught: + config.load_dotfile(root) + self.assertIn("line", str(caught.exception)) + + +class NameResolutionTests(unittest.TestCase): + def test_dir_name_three_levels(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + zh = config.resolve_config(root, lang="zh", profile="tiered") + self.assertEqual(config.dir_name(zh, "library"), "20_知识库") + self.assertEqual(config.dir_name(zh, "library.docs"), "20_知识库/文档资料") + self.assertEqual(config.dir_name(zh, "library.docs.pdf"), "20_知识库/文档资料/PDF") + self.assertEqual( + config.dest_dir(zh, "library.docs.pdf"), root / "20_知识库/文档资料/PDF" + ) + self.assertEqual(config.managed_dir(zh), root / "00_下载目录管理") + + en = config.resolve_config(root, lang="en", profile="tiered") + self.assertEqual(config.dir_name(en, "library.docs.pdf"), "20_Library/Documents/PDF") + self.assertEqual(config.managed_dir(en), root / "00_File_Organizer") + + with self.assertRaises(ValueError): + config.dir_name(zh, "library.docs.pdf.extra") + with self.assertRaises(KeyError): + config.dir_name(zh, "library.nope") + + def test_partition_prefixes(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + cfg = config.resolve_config(Path(temporary), lang="zh", profile="tiered") + self.assertEqual( + config.partition_prefixes(cfg), ("00_", "10_", "20_", "60_", "90_") + ) + names = config.partition_dir_names(cfg) + self.assertIn("00_收件箱", names) + self.assertIn("00_Inbox", names) + self.assertIn("00_下载目录管理", names) + + +class PathGuardTests(unittest.TestCase): + def test_refuse_root(self) -> None: + home = Path.home() + with self.assertRaises(ValueError): + paths.refuse_root(Path("/")) + with self.assertRaises(ValueError): + paths.refuse_root(home) + if (home / "Library").is_dir(): + with self.assertRaises(ValueError): + paths.refuse_root(home / "Library") + + with tempfile.TemporaryDirectory() as temporary: + outside = Path(temporary) + # The home is injected rather than borrowed from the machine, + # because a temporary directory is not reliably outside the real + # one: on Windows it sits under the account's own AppData. Being + # outside the home directory is the thing under test, so it gets + # arranged instead of assumed. + fake_home = outside / "home" + (fake_home / "Downloads").mkdir(parents=True) + elsewhere = outside / "elsewhere" + elsewhere.mkdir() + + with self.assertRaises(ValueError): + paths.refuse_root(elsewhere, home=fake_home) + self.assertEqual( + paths.refuse_root(elsewhere, allow_outside_home=True, home=fake_home), + paths.realpath(elsewhere), + ) + + linked = outside / "link" + try: + os.symlink(str(elsewhere), str(linked), target_is_directory=True) + except OSError: + # Windows refuses this without the symlink privilege; the rule + # it would prove is checked on the platforms that can make one. + pass + else: + with self.assertRaises(ValueError): + paths.refuse_root(linked, allow_outside_home=True, home=fake_home) + + # a folder inside a fake home is accepted + self.assertEqual( + paths.refuse_root(fake_home / "Downloads", home=fake_home), + paths.realpath(fake_home / "Downloads"), + ) + with self.assertRaises(ValueError): + paths.refuse_root(fake_home, home=fake_home) + + def test_portable_round_trip_and_safe_names(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = home / "Downloads" / "report.pdf" + self.assertEqual(paths.portable(target, home=home), "$HOME/Downloads/report.pdf") + self.assertEqual( + paths.expand_portable("$HOME/Downloads/report.pdf", home=home), target + ) + self.assertEqual(paths.safe_dir_name("$CODE_HOME"), "CODE_HOME") + self.assertEqual(paths.safe_dir_name('"'), "quote-dir") + self.assertEqual(paths.safe_dir_name(" "), "unnamed") + self.assertEqual(paths.safe_dir_name("a/b"), "a-b") + + def test_conflict_name_keeps_the_suffix_last(self) -> None: + stamp = datetime(2026, 9, 2, 12, 0, 0) + self.assertEqual( + paths.conflict_name("quarterly-report.pdf", stamp), + "quarterly-report-2026-09-02_12-00-00.pdf", + ) + self.assertEqual( + paths.conflict_name("archive.tar.gz", stamp), + "archive-2026-09-02_12-00-00.tar.gz", + ) + self.assertEqual( + paths.conflict_name("Report.PDF", stamp), "Report-2026-09-02_12-00-00.PDF" + ) + self.assertEqual(paths.conflict_name("README", stamp), "README-2026-09-02_12-00-00") + self.assertEqual( + paths.conflict_name("my.project", stamp, is_dir=True), + "my.project-2026-09-02_12-00-00", + ) + # safe_dir_name eats a leading dot before the stamp is added; hidden + # entries never reach the planner, so this only pins the existing rule. + self.assertEqual(paths.conflict_name(".bashrc", stamp), "bashrc-2026-09-02_12-00-00") + + +class PlanPrivacyTests(unittest.TestCase): + """strip_absolute drops the four absolute fields; hydrate_absolute rebuilds them. + + Both homes below are invented rather than real ``//`` + paths, because this repository refuses to carry one machine's absolute paths + at all: the same rule this feature exists to enforce on report.html. + """ + + HOME = "/opt/gn-test/example" + OTHER = "/opt/gn-test/someone-else" + + def _plan(self) -> dict: + home = self.HOME + return { + "schema_version": 2, + "source_root": home + "/Downloads", + "source_root_portable": "$HOME/Downloads", + "managed_dir": home + "/Downloads/00_File_Organizer", + "managed_dir_portable": "$HOME/Downloads/00_File_Organizer", + "actions": [ + { + "id": "a1", + "kind": "move", + "source": home + "/Downloads/report.pdf", + "source_portable": "$HOME/Downloads/report.pdf", + "destination": home + "/Downloads/PDF/report.pdf", + "destination_portable": "$HOME/Downloads/PDF/report.pdf", + "guard": { + "referenced_in": [ + { + "file": home + "/.zshrc", + "line": 82, + "form": home + "/Downloads/report.pdf", + } + ], + "incoming_symlinks": [home + "/.local/bin/report"], + }, + }, + { + "id": "a2", + "kind": "trash", + "source": home + "/Downloads/old.dmg", + "source_portable": "$HOME/Downloads/old.dmg", + "destination": None, + "destination_portable": None, + }, + ], + } + + def test_strip_removes_every_absolute_path_including_the_reference_form(self) -> None: + stripped = paths.strip_absolute(self._plan()) + text = json.dumps(stripped, ensure_ascii=False) + + for field in ("source_root", "managed_dir"): + self.assertNotIn(field, stripped) + for action in stripped["actions"]: + self.assertNotIn("source", action) + self.assertNotIn("destination", action) + self.assertEqual(stripped["source_root_portable"], "$HOME/Downloads") + + guard = stripped["actions"][0]["guard"] + self.assertEqual(guard["referenced_in"][0]["file"], "$HOME/.zshrc") + self.assertEqual(guard["incoming_symlinks"], ["$HOME/.local/bin/report"]) + # form is no longer exempt: it gets the same $HOME rewrite as everything else + self.assertEqual( + guard["referenced_in"][0]["form"], "$HOME/Downloads/report.pdf" + ) + + self.assertNotIn(self.HOME, text) + + def test_strip_leaves_the_original_alone(self) -> None: + plan = self._plan() + paths.strip_absolute(plan) + self.assertEqual(plan["source_root"], self.HOME + "/Downloads") + self.assertEqual(plan["actions"][0]["source"], self.HOME + "/Downloads/report.pdf") + + def test_hydrate_rebuilds_against_the_running_home(self) -> None: + stripped = paths.strip_absolute(self._plan()) + back = paths.hydrate_absolute(stripped, Path(self.OTHER)) + + # Rebuilt paths are spelled the way the machine doing the applying + # spells them, which on Windows means backslashes, so the comparison + # is made on paths rather than on strings. + other = Path(self.OTHER) + self.assertEqual(Path(back["source_root"]), other / "Downloads") + self.assertEqual( + Path(back["managed_dir"]), other / "Downloads" / "00_File_Organizer" + ) + self.assertEqual( + Path(back["actions"][0]["source"]), other / "Downloads" / "report.pdf" + ) + self.assertEqual( + Path(back["actions"][0]["destination"]), + other / "Downloads" / "PDF" / "report.pdf", + ) + self.assertIsNone(back["actions"][1]["destination"]) + + def test_hydrate_is_a_no_op_on_a_full_plan(self) -> None: + plan = self._plan() + self.assertEqual(paths.hydrate_absolute(plan, Path(self.HOME)), plan) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_dispose.py b/tests/test_cfo_dispose.py new file mode 100644 index 0000000..f76cdfb --- /dev/null +++ b/tests/test_cfo_dispose.py @@ -0,0 +1,500 @@ +"""The gate in front of the whole-machine disposals. + +Every test here builds a fake home directory and a small analysis.json over it, +because the rules this module enforces are all about the relationship between a +path, the home directory and the colour the Agent gave it. Nothing touches the +real machine: the trash backend is always a stub, and ``lsof`` is never called. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import os +import sys +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from carl_file_organizer import dispose as dispose_module +from carl_file_organizer import trash as trash_module + +TZ = timezone(timedelta(hours=9)) +NOW = datetime(2026, 9, 2, 12, 0, 0, tzinfo=TZ) + +GREEN = "green" +YELLOW = "yellow" +RED = "red" + + +class Handles(object): + """What ``guard.open_handles`` hands back, without asking the machine.""" + + def __init__(self, handles=(), status="ok"): + self.handles = list(handles) + self.status = status + + +def free(_path): + return Handles() + + +def unknown(_path): + return Handles(status="unknown") + + +class Trashcan(object): + """A trash backend that really moves things, so "gone" means gone.""" + + def __init__(self, root, fail=False): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + self.fail = fail + self.calls = [] + self.before = [] + + def __call__(self, path, *, backend=None, lang="en", **kw): + self.calls.append(Path(path)) + self.before.append(self.snapshot()) + if self.fail: + raise trash_module.TrashError("the trash refused this one") + target = self.root / Path(path).name + os.rename(str(path), str(target)) + + def snapshot(self): + return None + + +def item(item_id, color, portable, trash_paths, **extra): + entry = { + "id": item_id, + "color": color, + "name": item_id, + "path_portable": portable, + "size_bytes": 4096, + "what": "a thing", + "trash_paths": list(trash_paths), + } + entry.update(extra) + return entry + + +def decisions_for(pairs): + return { + "schema": dispose_module.SCHEMA, + "schema_version": 1, + "item_ids": [i for i, _a in pairs], + "actions": {i: a for i, a in pairs}, + } + + +class DisposeCase(unittest.TestCase): + """A fake home with one green cache, one yellow download, one red library.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.home = Path(self.tmp.name) / "home" + self.home.mkdir() + self.trashcan = Trashcan(Path(self.tmp.name) / "trash") + self.managed = self.home / ".carl-file-organizer" / "storage" + + self.green_path = self.make_dir("Library/Caches/pip") + self.green_2 = self.make_dir("Library/Caches/uv") + self.yellow_path = self.make_file("Downloads/Toolbox.dmg") + self.red_path = self.make_dir("Pictures/Snapshots.photoslibrary") + + self.analysis = { + "schema": "carl-file-organizer/storage-analysis", + "lang": "en", + "items": [ + item("green-pip", GREEN, "$HOME/Library/Caches/pip", ["$HOME/Library/Caches/pip"]), + item("green-uv", GREEN, "$HOME/Library/Caches/uv", ["$HOME/Library/Caches/uv"]), + item("yellow-dmg", YELLOW, "$HOME/Downloads", ["$HOME/Downloads/Toolbox.dmg"]), + item("red-photos", RED, "$HOME/Pictures/Snapshots.photoslibrary", []), + ], + } + + # -- fixture helpers ---------------------------------------------------- + + def make_dir(self, relative, content="body\n"): + target = self.home / relative + target.mkdir(parents=True, exist_ok=True) + (target / "one.bin").write_text(content, encoding="utf-8") + return target + + def make_file(self, relative, content="body\n"): + target = self.home / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + def add(self, entry): + self.analysis["items"].append(entry) + + def apply(self, pairs, *, dry_run=False, allow_permanent_delete=False, trash_fn=None, handles=free): + return dispose_module.apply_decisions( + self.analysis, + decisions_for(pairs), + dry_run=dry_run, + allow_permanent_delete=allow_permanent_delete, + home=self.home, + managed_dir=self.managed, + now=NOW, + trash_backend=trash_module.GIO, + trash_fn=trash_fn or self.trashcan, + open_handles_fn=handles, + ) + + def status_of(self, report, item_id): + return [r["status"] for r in report.results if r["item_id"] == item_id] + + def detail_of(self, report, item_id): + return " ".join(r["detail"] for r in report.results if r["item_id"] == item_id) + + def manifests(self): + return sorted(self.managed.glob("storage-removals-*.tsv")) + + +# -------------------------------------------------------------------------- +# colour is the gate +# -------------------------------------------------------------------------- + + +class ColourGateTests(DisposeCase): + def test_a_red_item_is_refused_whatever_is_asked_for(self): + self.add(item("red-loaded", RED, "$HOME/Library/Caches/pip", ["$HOME/Library/Caches/pip"])) + for action in ("trash", "delete"): + report = self.apply([("red-loaded", action)], allow_permanent_delete=True) + self.assertEqual(self.status_of(report, "red-loaded"), ["refused"]) + self.assertTrue(self.green_path.is_dir()) + self.assertEqual(self.trashcan.calls, []) + + def test_a_yellow_item_may_be_trashed_but_never_deleted(self): + report = self.apply([("yellow-dmg", "delete")], allow_permanent_delete=True) + self.assertEqual(self.status_of(report, "yellow-dmg"), ["refused"]) + self.assertIn("trash", self.detail_of(report, "yellow-dmg")) + self.assertTrue(self.yellow_path.is_file()) + + report = self.apply([("yellow-dmg", "trash")]) + self.assertEqual(self.status_of(report, "yellow-dmg"), ["trashed"]) + self.assertFalse(self.yellow_path.exists()) + + def test_a_green_delete_without_the_flag_is_refused_and_the_files_stay(self): + report = self.apply([("green-pip", "delete")]) + self.assertEqual(self.status_of(report, "green-pip"), ["refused"]) + self.assertIn("allow-permanent-delete", self.detail_of(report, "green-pip")) + self.assertTrue(self.green_path.is_dir()) + self.assertTrue((self.green_path / "one.bin").is_file()) + + def test_a_green_delete_with_the_flag_really_deletes(self): + report = self.apply([("green-pip", "delete")], allow_permanent_delete=True) + self.assertEqual(self.status_of(report, "green-pip"), ["deleted"]) + self.assertFalse(self.green_path.exists()) + self.assertEqual(self.trashcan.calls, []) + + def test_an_unknown_id_and_an_unknown_action_are_refused_not_guessed(self): + report = self.apply([("no-such-item", "trash")]) + self.assertEqual(self.status_of(report, "no-such-item"), ["refused"]) + report = self.apply([("green-pip", "shred")]) + self.assertEqual(self.status_of(report, "green-pip"), ["refused"]) + self.assertTrue(self.green_path.is_dir()) + + def test_a_system_note_takes_no_action(self): + self.add( + item( + "note-winsxs", + GREEN, + "$HOME/Library/Caches/pip", + ["$HOME/Library/Caches/pip"], + scanned=False, + size_bytes=None, + ) + ) + report = self.apply([("note-winsxs", "trash")]) + self.assertEqual(self.status_of(report, "note-winsxs"), ["refused"]) + self.assertTrue(self.green_path.is_dir()) + + def test_an_item_with_no_whitelisted_path_has_nothing_to_act_on(self): + self.add(item("green-empty", GREEN, "$HOME/Library/Caches/pip", [])) + report = self.apply([("green-empty", "trash")]) + self.assertEqual(self.status_of(report, "green-empty"), ["refused"]) + self.assertEqual(self.trashcan.calls, []) + + +# -------------------------------------------------------------------------- +# the path rules +# -------------------------------------------------------------------------- + + +class PathRuleTests(DisposeCase): + def test_a_path_outside_the_home_directory_is_refused(self): + outside = Path(self.tmp.name) / "elsewhere" + outside.mkdir() + (outside / "one.bin").write_text("body\n", encoding="utf-8") + self.add(item("green-outside", GREEN, str(outside), [str(outside)])) + report = self.apply([("green-outside", "trash")]) + self.assertEqual(self.status_of(report, "green-outside"), ["refused"]) + self.assertTrue((outside / "one.bin").is_file()) + self.assertEqual(self.trashcan.calls, []) + + def test_a_path_that_climbs_back_out_is_refused(self): + outside = Path(self.tmp.name) / "elsewhere" + outside.mkdir() + self.add(item("green-climb", GREEN, "$HOME/x", ["$HOME/../elsewhere"])) + report = self.apply([("green-climb", "trash")]) + self.assertEqual(self.status_of(report, "green-climb"), ["refused"]) + self.assertTrue(outside.is_dir()) + + @unittest.skipUnless( + sys.platform != "win32", + "creating a symlink on Windows needs administrator rights", + ) + def test_a_symlink_segment_is_refused(self): + real = self.make_dir("Library/Caches/real-cache") + link = self.home / "Library" / "Caches" / "linked" + link.symlink_to(real, target_is_directory=True) + self.add(item("green-link", GREEN, "$HOME/Library/Caches/linked", ["$HOME/Library/Caches/linked"])) + report = self.apply([("green-link", "trash")]) + self.assertEqual(self.status_of(report, "green-link"), ["refused"]) + self.assertIn("symlink", self.detail_of(report, "green-link")) + self.assertTrue(real.is_dir()) + self.assertTrue(link.is_symlink()) + + def test_a_no_go_basename_is_refused_even_when_the_agent_called_it_green(self): + self.make_dir("projects/site/.git") + self.add(item("green-git", GREEN, "$HOME/projects/site/.git", ["$HOME/projects/site/.git"])) + report = self.apply([("green-git", "trash")]) + self.assertEqual(self.status_of(report, "green-git"), ["refused"]) + self.assertIn("forbidden", self.detail_of(report, "green-git")) + self.assertTrue((self.home / "projects" / "site" / ".git").is_dir()) + + def test_a_no_go_suffix_is_refused(self): + self.add( + item( + "green-photos", + GREEN, + "$HOME/Pictures/Snapshots.photoslibrary", + ["$HOME/Pictures/Snapshots.photoslibrary"], + ) + ) + report = self.apply([("green-photos", "trash")]) + self.assertEqual(self.status_of(report, "green-photos"), ["refused"]) + self.assertTrue(self.red_path.is_dir()) + + def test_the_home_directory_and_its_children_are_too_coarse(self): + for portable in ("$HOME", "$HOME/Downloads", "$HOME/Library"): + self.add(item("green-coarse", GREEN, portable, [portable])) + report = self.apply([("green-coarse", "trash")]) + self.assertEqual(self.status_of(report, "green-coarse"), ["refused"], portable) + self.assertIn("children", self.detail_of(report, "green-coarse")) + self.analysis["items"].pop() + self.assertTrue(self.yellow_path.is_file()) + self.assertTrue(self.green_path.is_dir()) + + def test_one_bad_path_condemns_the_whole_item(self): + self.add( + item( + "green-mixed", + GREEN, + "$HOME/Library/Caches", + ["$HOME/Library/Caches/pip", "$HOME/Library"], + ) + ) + report = self.apply([("green-mixed", "trash")]) + self.assertEqual(self.status_of(report, "green-mixed"), ["refused"]) + self.assertTrue(self.green_path.is_dir()) + self.assertEqual(self.trashcan.calls, []) + + def test_the_same_path_twice_in_one_batch_is_only_acted_on_once(self): + self.add(item("green-again", GREEN, "$HOME/Library/Caches/pip", ["$HOME/Library/Caches/pip"])) + report = self.apply([("green-pip", "trash"), ("green-again", "trash")]) + self.assertEqual(self.status_of(report, "green-pip"), ["trashed"]) + self.assertEqual(self.status_of(report, "green-again"), ["skipped"]) + self.assertEqual(len(self.trashcan.calls), 1) + + def test_a_path_that_is_already_gone_is_skipped_not_failed(self): + self.add(item("green-gone", GREEN, "$HOME/Library/Caches/vanished", ["$HOME/Library/Caches/vanished"])) + report = self.apply([("green-gone", "trash")]) + self.assertEqual(self.status_of(report, "green-gone"), ["skipped"]) + self.assertEqual(self.trashcan.calls, []) + + +# -------------------------------------------------------------------------- +# preflight +# -------------------------------------------------------------------------- + + +class PreflightTests(DisposeCase): + def test_an_open_file_is_refused(self): + def busy(_path): + return Handles([{"command": "Cursor", "pid": 4242}]) + + report = self.apply([("green-pip", "trash")], handles=busy) + self.assertEqual(self.status_of(report, "green-pip"), ["refused"]) + self.assertIn("Cursor", self.detail_of(report, "green-pip")) + self.assertTrue(self.green_path.is_dir()) + + def test_an_unknown_answer_lets_it_through_but_is_written_down(self): + report = self.apply([("green-pip", "trash")], handles=unknown) + self.assertEqual(self.status_of(report, "green-pip"), ["trashed"]) + record = [r for r in report.results if r["item_id"] == "green-pip"][0] + self.assertEqual(record["open_check"], "unknown") + audit = json.loads((self.managed / "audit.jsonl").read_text(encoding="utf-8").splitlines()[0]) + self.assertEqual(audit["open_check"], "unknown") + + +# -------------------------------------------------------------------------- +# the paper trail +# -------------------------------------------------------------------------- + + +class PaperTrailTests(DisposeCase): + def test_the_manifest_is_on_disk_before_the_first_path_is_touched(self): + seen = {} + + def watcher(path, *, backend=None, lang="en", **kw): + files = self.manifests() + seen["files"] = list(files) + seen["rows"] = dispose_module.read_manifest(files[0]) if files else [] + return self.trashcan(path, backend=backend, lang=lang) + + report = self.apply([("green-pip", "trash")], trash_fn=watcher) + self.assertEqual(self.status_of(report, "green-pip"), ["trashed"]) + self.assertEqual(len(seen["files"]), 1) + self.assertEqual([r["status"] for r in seen["rows"]], ["PLANNED"]) + self.assertEqual(seen["rows"][0]["path"], "$HOME/Library/Caches/pip") + self.assertTrue(int(seen["rows"][0]["size_kib_before"]) >= 1) + + final = dispose_module.read_manifest(self.manifests()[0]) + self.assertEqual([r["status"] for r in final], ["TRASHED"]) + self.assertEqual(final[0]["error"], "") + self.assertIn("Trash", final[0]["restore_method"]) + + def test_the_manifest_columns_are_the_storage_seven(self): + self.apply([("green-pip", "trash")]) + header = self.manifests()[0].read_text(encoding="utf-8").splitlines()[0] + self.assertEqual(header.split("\t"), list(dispose_module.MANIFEST_COLUMNS)) + + def test_each_path_gets_its_own_audit_line_naming_no_account(self): + self.apply([("green-pip", "trash"), ("yellow-dmg", "trash")]) + lines = (self.managed / "audit.jsonl").read_text(encoding="utf-8").splitlines() + self.assertEqual(len(lines), 2) + records = [json.loads(line) for line in lines] + self.assertEqual([r["item_id"] for r in records], ["green-pip", "yellow-dmg"]) + self.assertEqual([r["status"] for r in records], ["trashed", "trashed"]) + for record in records: + self.assertEqual(record["line"], "storage") + self.assertEqual(record["kind"], "trash") + self.assertTrue(record["source"].startswith("$HOME/")) + self.assertNotIn(str(self.home), json.dumps(record)) + + def test_a_dry_run_writes_nothing_and_moves_nothing(self): + report = self.apply([("green-pip", "trash"), ("yellow-dmg", "trash")], dry_run=True) + self.assertEqual([r["status"] for r in report.results], ["dry-run", "dry-run"]) + self.assertIsNone(report.manifest) + self.assertIsNone(report.audit) + self.assertFalse(self.managed.exists()) + self.assertEqual(self.trashcan.calls, []) + self.assertTrue(self.green_path.is_dir()) + self.assertTrue(self.yellow_path.is_file()) + + def test_a_refusal_still_reaches_the_manifest_with_its_sentence(self): + report = self.apply([("red-photos", "trash"), ("green-pip", "trash")]) + rows = dispose_module.read_manifest(self.manifests()[0]) + self.assertEqual([r["status"] for r in rows], ["REFUSED", "TRASHED"]) + self.assertTrue(rows[0]["error"]) + self.assertEqual(rows[1]["error"], "") + self.assertEqual(report.counts["total"], 2) + + +# -------------------------------------------------------------------------- +# what happens when the trash says no +# -------------------------------------------------------------------------- + + +class TrashFailureTests(DisposeCase): + def test_a_failed_trash_never_becomes_a_permanent_delete(self): + failing = Trashcan(Path(self.tmp.name) / "trash", fail=True) + report = self.apply([("green-pip", "trash")], trash_fn=failing, allow_permanent_delete=True) + self.assertEqual(self.status_of(report, "green-pip"), ["failed"]) + self.assertTrue(self.green_path.is_dir()) + self.assertTrue((self.green_path / "one.bin").is_file()) + rows = dispose_module.read_manifest(self.manifests()[0]) + self.assertEqual(rows[0]["status"], "FAILED") + self.assertIn("refused this one", rows[0]["error"]) + + def test_the_first_failure_stops_the_rest_of_the_batch(self): + failing = Trashcan(Path(self.tmp.name) / "trash", fail=True) + report = self.apply( + [("green-pip", "trash"), ("green-uv", "trash"), ("yellow-dmg", "trash")], + trash_fn=failing, + ) + self.assertEqual( + [r["status"] for r in report.results], ["failed", "skipped", "skipped"] + ) + self.assertTrue(report.stopped) + self.assertEqual(len(failing.calls), 1) + self.assertTrue(self.green_2.is_dir()) + self.assertTrue(self.yellow_path.is_file()) + + def test_no_trash_backend_means_no_disposal_at_all(self): + report = dispose_module.apply_decisions( + self.analysis, + decisions_for([("green-pip", "trash")]), + home=self.home, + managed_dir=self.managed, + now=NOW, + trash_backend=trash_module.NONE, + open_handles_fn=free, + ) + self.assertEqual(self.status_of(report, "green-pip"), ["refused"]) + self.assertTrue(self.green_path.is_dir()) + + +# -------------------------------------------------------------------------- +# the shape of the API +# -------------------------------------------------------------------------- + + +class ContractTests(DisposeCase): + def test_validate_returns_rows_without_touching_anything(self): + validated = dispose_module.validate_decisions( + self.analysis, + decisions_for([("green-pip", "trash"), ("red-photos", "trash")]), + home=self.home, + trash_backend=trash_module.GIO, + ) + self.assertEqual([r.status for r in validated.rows], ["planned", "refused"]) + self.assertEqual(len(validated.targets), 1) + self.assertEqual(validated.targets[0].path, self.green_path) + self.assertTrue(self.green_path.is_dir()) + + def test_a_decisions_file_that_is_not_one_is_refused_outright(self): + for bad in ( + {"schema": "something/else", "item_ids": ["green-pip"], "actions": {}}, + {"schema": dispose_module.SCHEMA, "item_ids": "green-pip", "actions": {}}, + {"schema": dispose_module.SCHEMA, "item_ids": [], "actions": {}}, + {"schema": dispose_module.SCHEMA, "item_ids": ["green-pip"], "actions": {"green-pip": 3}}, + ): + with self.assertRaises(dispose_module.DisposeError): + dispose_module.validate_decisions(self.analysis, bad, home=self.home) + + def test_results_carry_the_five_fields_the_page_reads(self): + report = self.apply([("green-pip", "trash")]) + record = report.results[0] + for key in ("item_id", "path", "status", "detail", "size_bytes"): + self.assertIn(key, record) + self.assertEqual(report.executed_ids, ["green-pip"]) + + def test_the_managed_folder_is_its_own_corner_of_the_home_directory(self): + self.assertEqual( + dispose_module.managed_dir(self.home), + self.home / ".carl-file-organizer" / "storage", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_executor.py b/tests/test_cfo_executor.py new file mode 100644 index 0000000..cb4c2bb --- /dev/null +++ b/tests/test_cfo_executor.py @@ -0,0 +1,949 @@ +"""Execution: what it refuses, what it skips, and what it puts back. + +Every plan in here is built from real temporary files with ``ids.subject_id`` +and ``ids.action_id`` computed on the spot, so the tests exercise the same +identity check the executor uses, without depending on the planner. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import contextlib +import io +import json +import os +import shutil +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest import mock + +from carl_file_organizer import config as config_module +from carl_file_organizer import executor, ids, manifest, paths +from carl_file_organizer.executor import ExecutionRefused, PlanError + +HOME = Path.home() +NOW = datetime(2026, 9, 2, 10, 20, 11).astimezone() + +NAMES = { + "inbox": "00_收件箱", + "inbox.pending": "00_收件箱/待判断", + "inbox.dup": "00_收件箱/重复待确认", + "library": "20_知识库", + "library.docs": "20_知识库/文档资料", + "library.docs.pdf": "20_知识库/文档资料/PDF", + "sensitive": "60_敏感信息", + "sensitive.pending": "60_敏感信息/待转移", + "work": "10_工作区", + "work.tools": "10_工作区/工具与导出", + "work.tools.scripts": "10_工作区/工具与导出/脚本", +} + + +def make_action( + path: Path, + kind: str, + *, + destination=None, + destination_key=None, + tier: str = "routine", + rule: str = "ext:.pdf", + approvable: bool = True, + reroutable: bool = False, + group_id=None, + size_bytes=None, + home: Path = HOME, +): + stat = path.stat() + subject_kind = "dir" if path.is_dir() else "file" + subject = ids.subject_id(path, subject_kind, stat.st_size, stat.st_mtime_ns) + destination_portable = ( + paths.portable(Path(destination), home) if destination is not None else None + ) + recorded_size = stat.st_size if size_bytes is None else size_bytes + return { + "id": ids.action_id(subject, kind, destination_portable), + "subject_id": subject, + "kind": kind, + "subject_kind": subject_kind, + "source": str(path), + "source_portable": paths.portable(path, home), + "filename": path.name, + "destination": str(destination) if destination is not None else None, + "destination_portable": destination_portable, + "destination_key": destination_key, + "size_bytes": recorded_size, + "modified_ns": stat.st_mtime_ns, + "age_hours": 100, + "rule": rule, + "reason": {"zh": "测试用的理由", "en": "reason for the test"}, + "detail": None, + "confidence": 0.98, + "tier": tier, + "group_id": group_id, + "approvable": approvable, + "reroutable": reroutable, + "default_selected": False, + "requires": ["allow-permanent-delete"] if kind == "delete" else [], + "reclaims_bytes": recorded_size if kind in ("trash", "delete") else 0, + "restore_method": None, + "tags": [], + "hints": [], + "hold_reason": None, + "guard": {"open_by": [], "referenced_in": [], "incoming_symlinks": [], "shape": []}, + "dir_stats": None, + } + + +def make_plan( + root: Path, + actions, + *, + approved=(), + overrides=(), + groups=(), + lang: str = "zh", + trash_backend: str = "finder", + home: Path = HOME, +): + return { + "schema_version": 2, + "generator": {"name": "carl-file-organizer", "version": "0.3.0"}, + "created_at": NOW.isoformat(), + "now": NOW.isoformat(), + "lang": lang, + "lang_source": "flag", + "profile": {"name": "tiered", "source": "builtin", "dotfile": None, "hash": ""}, + "source_root": str(root), + "source_root_portable": paths.portable(root, home), + "managed_dir": str(root / "00_下载目录管理"), + "managed_dir_portable": paths.portable(root / "00_下载目录管理", home), + "names": dict(NAMES), + "capabilities": { + "trash_backend": trash_backend, + "finder_tags": True, + "permanent_delete_offered": True, + }, + "scan_policy": {"depth": 1}, + "disk": {}, + "summary": {}, + "pinned": [], + "actions": list(actions), + "groups": list(groups), + "approved_action_ids": list(approved), + "overrides": list(overrides), + "approved_at": None, + "approved_by": "agent:test", + } + + +class Fixture: + """A temporary folder plus the config the executor would resolve for it.""" + + def __init__(self, stack: unittest.TestCase) -> None: + temporary = tempfile.TemporaryDirectory() + stack.addCleanup(temporary.cleanup) + self.root = paths.realpath(Path(temporary.name)) + self.config = config_module.resolve_config(self.root, lang="zh", profile="tiered") + self.managed = config_module.managed_dir(self.config) + + def file(self, name: str, content: str = "hello") -> Path: + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + def folder(self, name: str, entries=("a.txt", "b.txt")) -> Path: + target = self.root / name + target.mkdir(parents=True, exist_ok=True) + for entry in entries: + (target / entry).write_text(entry, encoding="utf-8") + return target + + def pdf_dest(self, name: str) -> Path: + return self.root / NAMES["library.docs.pdf"] / name + + def apply(self, plan, **kw): + options = {"config": self.config, "now": NOW, "open_handles_fn": lambda path: []} + options.update(kw) + return executor.apply_approved_plan(plan, **options) + + def audit_records(self): + return manifest.read_audit(manifest.audit_path(self.managed)) + + def manifest_file(self): + found = sorted(self.managed.glob("*.tsv")) + return found[0] if found else None + + +class ValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_only_approved_action_moves(self) -> None: + first = self.fixture.file("first.pdf", "document") + second = self.fixture.file("second.pdf", "image") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"), + destination_key="library.docs.pdf"), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf"), + destination_key="library.docs.pdf"), + ] + plan = make_plan(self.fixture.root, actions, approved=[actions[0]["id"]]) + + report = self.fixture.apply(plan) + + self.assertEqual(len(report.results), 1) + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(self.fixture.pdf_dest("first.pdf").exists()) + self.assertFalse(first.exists()) + self.assertTrue(second.exists()) + + def test_schema_version_one_is_refused(self) -> None: + plan = make_plan(self.fixture.root, []) + plan["schema_version"] = 1 + with self.assertRaises(PlanError): + executor.validate_plan(plan, allow_permanent_delete=False) + + def test_unknown_approval_id_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + actions = [make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"))] + plan = make_plan(self.fixture.root, actions, approved=["invented"]) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_tampered_destination_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + action = make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf"), + destination_key="library.docs.pdf") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + action["destination"] = str(self.fixture.root / "20_知识库" / "renamed.pdf") + action["destination_portable"] = paths.portable(Path(action["destination"]), HOME) + + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_destination_outside_the_root_is_rejected(self) -> None: + first = self.fixture.file("first.pdf") + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + action = make_action(first, "move", destination=outside / "first.pdf") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(first.exists()) + + def test_two_actions_on_the_same_subject_are_rejected(self) -> None: + target = self.fixture.file("Archive.zip") + move = make_action(target, "move", destination=self.fixture.root / NAMES["inbox.pending"] / "Archive.zip", + tier="duplicate", rule="pair") + drop = make_action(target, "trash", tier="duplicate", rule="pair") + plan = make_plan(self.fixture.root, [move, drop], approved=[move["id"], drop["id"]]) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("same item", str(caught.exception)) + self.assertTrue(target.exists()) + + def test_hold_id_is_rejected(self) -> None: + target = self.fixture.file("installer.dmg") + hold = make_action(target, "hold", tier="aging", rule="hold:aging", approvable=False) + plan = make_plan(self.fixture.root, [hold], approved=[hold["id"]]) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("not approvable", str(caught.exception)) + self.assertTrue(target.exists()) + + def test_root_that_is_a_symlink_is_refused(self) -> None: + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + link = self.fixture.root / "link-root" + link.symlink_to(outside, target_is_directory=True) + plan = make_plan(link, []) + with self.assertRaises(PlanError): + executor.validate_plan(plan, allow_permanent_delete=False) + + +class DeleteGateTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def _node_modules(self): + project = self.fixture.folder("web-prototype", entries=()) + modules = project / "node_modules" + modules.mkdir() + (modules / "left-pad.js").write_text("module", encoding="utf-8") + return modules + + def test_delete_requires_the_flag_and_refuses_the_whole_batch(self) -> None: + modules = self._node_modules() + other = self.fixture.file("report.pdf") + move = make_action(other, "move", destination=self.fixture.pdf_dest("report.pdf")) + drop = make_action(modules, "delete", tier="regenerable", rule="regenerable:node_modules") + plan = make_plan(self.fixture.root, [move, drop], approved=[move["id"], drop["id"]]) + + with self.assertRaises(ExecutionRefused): + self.fixture.apply(plan) + + self.assertTrue(modules.exists()) + self.assertTrue(other.exists(), "a refused batch must not move anything either") + self.assertFalse(self.fixture.managed.exists()) + + def test_delete_with_the_flag_removes_regenerable_output(self) -> None: + modules = self._node_modules() + drop = make_action(modules, "delete", tier="regenerable", rule="regenerable:node_modules") + plan = make_plan(self.fixture.root, [drop], approved=[drop["id"]]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual(report.results[0]["status"], "deleted") + self.assertFalse(modules.exists()) + + def test_delete_outside_the_regenerable_list_is_refused(self) -> None: + project = self.fixture.folder("web-prototype", entries=()) + source = project / "src" + source.mkdir() + (source / "index.js").write_text("real work", encoding="utf-8") + drop = make_action(source, "delete", tier="regenerable", rule="regenerable:src") + plan = make_plan(self.fixture.root, [drop], approved=[drop["id"]]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(source.exists()) + self.assertIn("src", report.results[0]["detail"]) + + def test_duplicate_delete_needs_a_surviving_copy(self) -> None: + first = self.fixture.file("guide.pdf", "same") + second = self.fixture.file("guide (1).pdf", "same") + keep = make_action(first, "move", destination=self.fixture.pdf_dest("guide.pdf"), + tier="duplicate", rule="dup:sha256", group_id="dup-1") + drop = make_action(second, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + group = { + "group_id": "dup-1", + "kind": "duplicate", + "members": [keep["subject_id"], drop["subject_id"]], + } + + plan = make_plan(self.fixture.root, [keep, drop], approved=[keep["id"], drop["id"]], + groups=[group]) + report = self.fixture.apply(plan, allow_permanent_delete=True) + self.assertEqual(report.status_of(drop["id"]), "deleted") + self.assertFalse(second.exists()) + self.assertTrue(self.fixture.pdf_dest("guide.pdf").exists()) + + def test_deleting_every_member_of_a_group_is_refused(self) -> None: + first = self.fixture.file("guide.pdf", "same") + second = self.fixture.file("guide (1).pdf", "same") + one = make_action(first, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + two = make_action(second, "delete", tier="duplicate", rule="dup:sha256", group_id="dup-1") + group = {"group_id": "dup-1", "kind": "duplicate", + "members": [one["subject_id"], two["subject_id"]]} + plan = make_plan(self.fixture.root, [one, two], approved=[one["id"], two["id"]], + groups=[group]) + + report = self.fixture.apply(plan, allow_permanent_delete=True) + + self.assertEqual({record["status"] for record in report.results}, {"refused"}) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + +class PreflightTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_changed_source_is_skipped(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + stat = target.stat() + os.utime(str(target), ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "skipped") + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.pdf_dest("report.pdf").exists()) + + def test_vanished_source_is_skipped(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + target.unlink() + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "skipped") + + def test_no_overwrite_at_apply_time(self) -> None: + target = self.fixture.file("report.pdf", "mine") + destination = self.fixture.pdf_dest("report.pdf") + action = make_action(target, "move", destination=destination) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("someone else", encoding="utf-8") + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "conflict") + self.assertEqual(target.read_text(encoding="utf-8"), "mine") + self.assertEqual(destination.read_text(encoding="utf-8"), "someone else") + + def test_symlink_escape_is_refused(self) -> None: + outside = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, str(outside), True) + real = outside / "secret.pdf" + real.write_text("outside the folder", encoding="utf-8") + link = self.fixture.root / "report.pdf" + link.symlink_to(real) + + action = make_action(link, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(real.exists()) + self.assertTrue(link.is_symlink()) + + def test_forbidden_zone_is_refused(self) -> None: + repository = self.fixture.folder(".git", entries=("HEAD",)) + action = make_action(repository, "move", + destination=self.fixture.root / NAMES["inbox.pending"] / ".git") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "refused") + self.assertTrue(repository.exists()) + + def test_an_open_file_is_skipped(self) -> None: + target = self.fixture.file("session.log") + action = make_action(target, "move", + destination=self.fixture.root / NAMES["inbox.pending"] / "session.log") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply( + plan, open_handles_fn=lambda path: [{"pid": "123", "command": "tail", "name": str(path)}] + ) + self.assertEqual(report.results[0]["status"], "skipped") + self.assertTrue(target.exists()) + + def test_spotlight_reading_a_file_does_not_count_as_in_use(self) -> None: + target = self.fixture.file("report.pdf") + + class Result: + returncode = 0 + stdout = "p501\ncmdworker_shared\nn{0}\n".format(target) + stderr = "" + + with mock.patch.object(executor.shutil, "which", lambda name: "/usr/sbin/lsof"), \ + mock.patch.object(executor.subprocess, "run", lambda *a, **kw: Result()): + self.assertEqual(executor.open_handles(target), []) + + class Real: + returncode = 0 + stdout = "p777\nctail\nn{0}\n".format(target) + stderr = "" + + with mock.patch.object(executor.shutil, "which", lambda name: "/usr/sbin/lsof"), \ + mock.patch.object(executor.subprocess, "run", lambda *a, **kw: Real()): + found = executor.open_handles(target) + self.assertEqual(found, [{"pid": "777", "command": "tail", "name": str(target)}]) + + def test_no_lsof_means_no_objection(self) -> None: + target = self.fixture.file("report.pdf") + with mock.patch.object(executor.shutil, "which", lambda name: None): + self.assertEqual(executor.open_handles(target), []) + + def test_a_folder_keeps_its_name_and_moves_whole(self) -> None: + folder = self.fixture.folder("web-prototype") + destination = self.fixture.root / NAMES["inbox.pending"] / "web-prototype" + action = make_action(folder, "move", destination=destination, rule="dir:pending", + reroutable=True, destination_key="inbox.pending") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue((destination / "a.txt").exists()) + + def test_a_folder_whose_recorded_size_was_walked_still_matches(self) -> None: + folder = self.fixture.folder("web-prototype") + destination = self.fixture.root / NAMES["inbox.pending"] / "web-prototype" + action = make_action(folder, "move", destination=destination, rule="dir:pending") + # A planner may record the walked size rather than the inode size. + walked = sum(item.stat().st_size for item in folder.iterdir()) + action["size_bytes"] = walked + action["subject_id"] = ids.subject_id(folder, "dir", walked, folder.stat().st_mtime_ns) + action["id"] = ids.action_id( + action["subject_id"], "move", paths.portable(destination, HOME) + ) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan) + self.assertEqual(report.results[0]["status"], "moved") + + +class OverrideTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def _pending(self): + target = self.fixture.file("dataset.parquet") + action = make_action( + target, + "move", + destination=self.fixture.root / NAMES["inbox.pending"] / "dataset.parquet", + destination_key="inbox.pending", + rule="unknown-ext", + reroutable=True, + ) + return target, action + + def test_override_reroutes_through_the_names_table(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "work.tools.scripts"}], + ) + + report = self.fixture.apply(plan) + + moved_to = self.fixture.root / NAMES["work.tools.scripts"] / "dataset.parquet" + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(moved_to.exists()) + self.assertFalse(target.exists()) + + def test_override_to_an_unknown_key_is_rejected(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "somewhere.else"}], + ) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(target.exists()) + + def test_override_cannot_carry_a_literal_path(self) -> None: + target, action = self._pending() + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "../../etc"}], + ) + with self.assertRaises(PlanError): + self.fixture.apply(plan) + self.assertTrue(target.exists()) + + def test_override_on_a_rule_bound_action_is_rejected(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf", rule="ext:.pdf") + plan = make_plan( + self.fixture.root, + [action], + approved=[action["id"]], + overrides=[{"action_id": action["id"], "destination_key": "inbox.pending"}], + ) + with self.assertRaises(PlanError) as caught: + self.fixture.apply(plan) + self.assertIn("cannot be rerouted", str(caught.exception)) + self.assertTrue(target.exists()) + + +class VerificationTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_verify_move_checks_size_and_both_ends(self) -> None: + source = self.fixture.file("a.txt", "hello") + destination = self.fixture.file("b.txt", "hello!") + + # A source still sitting there means the move never happened. + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 5}) + self.assertFalse(ok) + self.assertIn("source still there", detail) + + source.unlink() + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 5}) + self.assertFalse(ok) + self.assertIn("size mismatch", detail) + + ok, detail = executor.verify_move(source, destination, {"kind": "file", "size": 6}) + self.assertTrue(ok) + + ok, detail = executor.verify_move(source, self.fixture.root / "nope.txt", + {"kind": "file", "size": 6}) + self.assertFalse(ok) + + def test_verify_move_counts_folder_entries(self) -> None: + folder = self.fixture.folder("stuff", entries=("a.txt", "b.txt")) + moved = self.fixture.root / "moved" + shutil.move(str(folder), str(moved)) + self.assertEqual( + executor.verify_move(folder, moved, {"kind": "dir", "entries": 2})[0], True + ) + self.assertEqual( + executor.verify_move(folder, moved, {"kind": "dir", "entries": 5})[0], False + ) + + def test_a_move_that_does_not_verify_goes_back_and_stops_the_batch(self) -> None: + first = self.fixture.file("first.pdf", "one") + second = self.fixture.file("second.pdf", "two") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf")), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf")), + ] + plan = make_plan(self.fixture.root, actions, + approved=[actions[0]["id"], actions[1]["id"]]) + + def wrong_shape(path): + return {"kind": "file", "size": 999999} + + with mock.patch.object(executor, "_shape", wrong_shape): + report = self.fixture.apply(plan) + + self.assertEqual(report.results[0]["status"], "failed") + self.assertIn("原路移回", report.results[0]["detail"]) + self.assertEqual(report.results[1]["status"], "skipped") + self.assertTrue(report.stopped) + + self.assertEqual(first.read_text(encoding="utf-8"), "one") + self.assertFalse(self.fixture.pdf_dest("first.pdf").exists()) + self.assertTrue(second.exists()) + self.assertFalse(self.fixture.pdf_dest("second.pdf").exists()) + + def test_continue_on_error_keeps_going(self) -> None: + first = self.fixture.file("first.pdf", "one") + second = self.fixture.file("second.pdf", "two") + actions = [ + make_action(first, "move", destination=self.fixture.pdf_dest("first.pdf")), + make_action(second, "move", destination=self.fixture.pdf_dest("second.pdf")), + ] + plan = make_plan(self.fixture.root, actions, + approved=[actions[0]["id"], actions[1]["id"]]) + + with mock.patch.object(executor, "_shape", lambda path: {"kind": "file", "size": 999999}): + report = self.fixture.apply(plan, continue_on_error=True) + + self.assertEqual([record["status"] for record in report.results], ["failed", "failed"]) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + def test_manifest_is_written_before_execution(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + seen = {} + + real_move = shutil.move + + def exploding_move(source, destination, *args, **kw): + found = sorted(self.fixture.managed.glob("*.tsv")) + seen["manifest"] = found[0].read_text(encoding="utf-8") if found else "" + raise OSError("disk went away") + + with mock.patch.object(executor.shutil, "move", exploding_move): + report = self.fixture.apply(plan) + + self.assertIn("PLANNED", seen["manifest"]) + self.assertIn("report.pdf", seen["manifest"]) + self.assertEqual(report.results[0]["status"], "failed") + self.assertTrue(target.exists()) + + records = self.fixture.audit_records() + self.assertEqual(records[-1]["status"], "failed") + self.assertIn("disk went away", records[-1]["detail"]) + self.assertIn("FAILED", self.fixture.manifest_file().read_text(encoding="utf-8")) + self.assertIs(shutil.move, real_move) + + +class PaperTrailTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + def test_dry_run_writes_nothing_at_all(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + report = self.fixture.apply(plan, dry_run=True) + + self.assertEqual(report.results[0]["status"], "dry-run") + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.managed.exists()) + self.assertIsNone(report.manifest) + self.assertIsNone(report.audit) + + def test_a_full_run_writes_manifest_audit_and_log(self) -> None: + report_pdf = self.fixture.file("report.pdf", "a document") + secret = self.fixture.file("api-key-backup.txt", "never read") + occupied = self.fixture.file("guide.pdf", "mine") + taken = self.fixture.pdf_dest("guide.pdf") + taken.parent.mkdir(parents=True, exist_ok=True) + taken.write_text("already here", encoding="utf-8") + + actions = [ + make_action(report_pdf, "move", destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf"), + make_action(secret, "move", + destination=self.fixture.root / NAMES["sensitive.pending"] / "api-key-backup.txt", + destination_key="sensitive.pending", tier="sensitive", + rule="sensitive:marker"), + make_action(occupied, "move", destination=taken, + destination_key="library.docs.pdf"), + ] + plan = make_plan(self.fixture.root, actions, + approved=[action["id"] for action in actions]) + + result = self.fixture.apply(plan, recheck_fn=lambda: 0) + + self.assertEqual( + [record["status"] for record in result.results], ["moved", "moved", "conflict"] + ) + + rows = manifest.read_manifest(result.manifest) + self.assertEqual(result.manifest.name, "downloads-moves-2026-09-02_10-20-11.tsv".replace( + "downloads", manifest.slug(self.fixture.root.name))) + self.assertEqual([row["status"] for row in rows], ["MOVED", "MOVED", "CONFLICT"]) + self.assertTrue(rows[0]["restore_method"].startswith('mv "')) + self.assertEqual(rows[0]["kind"], "move") + self.assertEqual(rows[0]["action_id"], actions[0]["id"]) + + records = self.fixture.audit_records() + self.assertEqual(len(records), 3) + self.assertEqual(records[0]["run_id"], "2026-09-02_10-20-11") + self.assertEqual(records[0]["trash_backend"], "finder") + + log = result.review_log.read_text(encoding="utf-8") + self.assertIn("## 2026-09-02", log) + self.assertIn("未读取内容", log) + self.assertIn("未删除任一项", log) + self.assertIn("复查结果：dry-run actions=0。", log) + + def test_audit_paths_are_written_in_home_form(self) -> None: + home = self.fixture.root.parent + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf"), + home=home) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]], home=home) + + report = self.fixture.apply(plan, home=home) + + self.assertEqual(report.results[0]["status"], "moved") + self.assertTrue(report.results[0]["source"].startswith("$HOME/")) + self.assertTrue(report.results[0]["destination"].startswith("$HOME/")) + self.assertTrue(report.results[0]["manifest"].startswith("$HOME/")) + text = report.manifest.read_text(encoding="utf-8") + self.assertNotIn(str(self.fixture.root), text) + + def test_trash_uses_the_backend_and_never_deletes_on_failure(self) -> None: + target = self.fixture.file("Archive.zip", "zipped") + action = make_action(target, "trash", tier="duplicate", rule="pair") + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + + def refuse(path, **kw): + raise executor.trash_module.TrashError("Finder said no") + + report = self.fixture.apply(plan, trash_fn=refuse) + self.assertEqual(report.results[0]["status"], "failed") + self.assertIn("Finder said no", report.results[0]["detail"]) + self.assertTrue(target.exists()) + + sent = [] + + def accept(path, **kw): + sent.append((Path(path), kw.get("backend"))) + os.unlink(str(path)) + + plan["approved_action_ids"] = [action["id"]] + report = self.fixture.apply(plan, trash_fn=accept) + self.assertEqual(report.results[0]["status"], "trashed") + self.assertEqual(sent[0][1], "finder") + self.assertFalse(target.exists()) + + def test_empty_approval_list_does_nothing_quietly(self) -> None: + self.fixture.file("report.pdf") + plan = make_plan(self.fixture.root, [], approved=[]) + report = self.fixture.apply(plan) + self.assertEqual(report.results, []) + self.assertFalse(self.fixture.managed.exists()) + + +class CliTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture(self) + + @staticmethod + def _quiet(argv): + """Run a subcommand without spilling its report into the test output.""" + + from carl_file_organizer import cli + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + code = cli.main(argv) + return code, buffer.getvalue() + + def test_apply_subcommand_keeps_the_approval_file(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + code, output = self._quiet(["apply", str(approval)]) + + self.assertEqual(code, 0) + self.assertIn("moved", output) + self.assertTrue(self.fixture.pdf_dest("report.pdf").exists()) + kept = sorted(self.fixture.managed.glob("approved-*.json")) + self.assertEqual(len(kept), 1) + + def _apply_and_capture(self, plan_hash: str): + """Run ``apply --dry-run`` on a one-move plan and hand back stderr.""" + + from carl_file_organizer import cli + + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + plan["profile"]["hash"] = plan_hash + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = cli.main(["apply", str(approval), "--dry-run"]) + return code, err.getvalue() + + def test_writing_the_settings_file_does_not_look_like_drift(self) -> None: + # The first plan hashes the settings and then writes the settings file, + # which records the language: that is not drift and must not warn. + recorded = self.fixture.config.profile_hash + config_module.save_dotfile(self.fixture.config, only_if_missing=True) + + code, err = self._apply_and_capture(recorded) + + self.assertEqual(code, 0) + self.assertNotIn(executor.text("cli_profile_drift", "zh"), err) + + def test_edited_settings_still_warn_before_applying(self) -> None: + code, err = self._apply_and_capture("0123456789ab") + + self.assertEqual(code, 0) + self.assertIn(executor.text("cli_profile_drift", "zh"), err) + + def test_apply_dry_run_exits_zero_and_writes_nothing(self) -> None: + target = self.fixture.file("report.pdf") + action = make_action(target, "move", destination=self.fixture.pdf_dest("report.pdf")) + plan = make_plan(self.fixture.root, [action], approved=[action["id"]]) + approval = self.fixture.root / "carl-file-organizer-approved.json" + approval.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + code, output = self._quiet(["apply", str(approval), "--dry-run"]) + + self.assertEqual(code, 0) + self.assertIn("dry-run", output) + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.managed.exists()) + + +class PortableOnlyApprovalTests(unittest.TestCase): + """The review page exports $HOME paths only; apply has to cope with that. + + ``home`` is the fixture root itself, so ``portable`` actually produces + ``$HOME/...`` for a temporary folder that does not live under the real home. + """ + + def setUp(self) -> None: + self.fixture = Fixture(self) + self.home = self.fixture.root + + def _approval(self, *, kinds=("move",)): + target = self.fixture.file("report.pdf") + actions = [] + if "move" in kinds: + actions.append( + make_action( + target, + "move", + destination=self.fixture.pdf_dest("report.pdf"), + destination_key="library.docs.pdf", + home=self.home, + ) + ) + if "trash" in kinds: + actions.append(make_action(self.fixture.file("old.dmg"), "trash", home=self.home)) + plan = make_plan( + self.fixture.root, + actions, + approved=[action["id"] for action in actions], + home=self.home, + ) + return target, actions, paths.strip_absolute(plan) + + def test_the_exported_shape_really_has_no_absolute_path(self) -> None: + _target, _actions, approval = self._approval(kinds=("move", "trash")) + + self.assertNotIn("source_root", approval) + self.assertNotIn("managed_dir", approval) + self.assertNotIn(str(self.fixture.root), json.dumps(approval, ensure_ascii=False)) + for action in approval["actions"]: + self.assertNotIn("source", action) + self.assertNotIn("destination", action) + + def test_validate_accepts_it_and_rebuilds_the_real_paths(self) -> None: + target, actions, approval = self._approval() + + validated = executor.validate_plan( + approval, allow_permanent_delete=False, home=self.home + ) + + self.assertEqual(validated.root, paths.realpath(self.fixture.root)) + self.assertEqual(validated.home, self.home) + self.assertEqual(Path(validated.actions[0]["source"]), target) + self.assertEqual( + Path(validated.actions[0]["destination"]), self.fixture.pdf_dest("report.pdf") + ) + self.assertEqual(validated.actions[0]["id"], actions[0]["id"]) + + def test_dry_run_passes_preflight_and_touches_nothing(self) -> None: + target, actions, approval = self._approval() + + report = self.fixture.apply(approval, dry_run=True, home=self.home) + + self.assertEqual(report.status_of(actions[0]["id"]), "dry-run") + self.assertEqual(report.counts.get("failed", 0), 0) + self.assertEqual(report.counts.get("refused", 0), 0) + self.assertTrue(target.exists()) + self.assertFalse(self.fixture.pdf_dest("report.pdf").exists()) + + def test_a_real_apply_moves_the_file_and_audits_in_home_form(self) -> None: + target, actions, approval = self._approval() + + report = self.fixture.apply(approval, home=self.home) + + self.assertEqual(report.status_of(actions[0]["id"]), "moved") + self.assertFalse(target.exists()) + self.assertTrue(self.fixture.pdf_dest("report.pdf").exists()) + record = self.fixture.audit_records()[0] + self.assertEqual(record["source"], "$HOME/report.pdf") + self.assertEqual(record["destination"], "$HOME/20_知识库/文档资料/PDF/report.pdf") + + def test_a_tampered_destination_is_still_caught(self) -> None: + _target, _actions, approval = self._approval() + approval["actions"][0]["destination_portable"] = "$HOME/00_收件箱/report.pdf" + + with self.assertRaises(PlanError): + executor.validate_plan(approval, allow_permanent_delete=False, home=self.home) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_fixture_contract.py b/tests/test_cfo_fixture_contract.py new file mode 100644 index 0000000..443d869 --- /dev/null +++ b/tests/test_cfo_fixture_contract.py @@ -0,0 +1,284 @@ +"""The plan.json schema v2 contract, checked against the hand-written sample. + +Every work package reads and writes this shape, so the sample is the shared +truth: if a change breaks this file, it breaks the review page, the executor +and the agent skill at the same time. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import unittest +from pathlib import Path + +FIXTURE = Path(cfo_path.FIXTURES_DIR) / "plan-v2-sample.json" + +HOLD_TIERS = {"aging", "pinned", "forbidden", "in_use", "referenced"} +REROUTABLE_RULES = {"unknown-ext", "dir:pending", "dir:derivative"} +KINDS = {"move", "trash", "delete", "hold"} +COLORS = {"green", "yellow", "red"} +TIERS = { + "routine", + "sensitive", + "regenerable", + "duplicate", + "cold", + "aging", + "pinned", + "forbidden", + "in_use", + "referenced", +} + + +def _portable_values(node, trail=""): + """Yield every ``*_portable`` string in the document with its path.""" + + if isinstance(node, dict): + for key, value in node.items(): + where = "{0}.{1}".format(trail, key) + if isinstance(value, str) and key.endswith("_portable"): + yield where, value + else: + for item in _portable_values(value, where): + yield item + elif isinstance(node, list): + for index, value in enumerate(node): + for item in _portable_values(value, "{0}[{1}]".format(trail, index)): + yield item + + +class PlanV2ContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.plan = json.loads(FIXTURE.read_text(encoding="utf-8")) + cls.actions = cls.plan["actions"] + cls.by_id = {action["id"]: action for action in cls.actions} + + def test_schema_version_and_top_level_shape(self) -> None: + self.assertEqual(self.plan["schema_version"], 2) + for key in ( + "generator", "created_at", "now", "lang", "lang_source", "profile", + "source_root", "source_root_portable", "managed_dir", "managed_dir_portable", + "names", "capabilities", "scan_policy", "disk", "mess", "summary", + "pinned", "actions", "groups", "notes", "approved_action_ids", + "overrides", "approved_at", "approved_by", + ): + self.assertIn(key, self.plan) + self.assertEqual(self.plan["approved_action_ids"], []) + self.assertEqual(self.plan["overrides"], []) + self.assertIsNone(self.plan["notes"]) + + def test_action_ids_are_unique(self) -> None: + self.assertEqual(len(self.by_id), len(self.actions)) + + def test_every_action_carries_the_frozen_fields(self) -> None: + required = ( + "id", "subject_id", "kind", "subject_kind", "source", "source_portable", + "filename", "destination", "destination_portable", "destination_key", + "size_bytes", "modified_ns", "age_hours", "rule", "reason", "detail", + "confidence", "tier", "group_id", "approvable", "reroutable", + "default_selected", "requires", "reclaims_bytes", "restore_method", + "tags", "hints", "hold_reason", "guard", "dir_stats", "color", + ) + for action in self.actions: + for key in required: + self.assertIn(key, action, action["id"]) + self.assertIn(action["kind"], KINDS) + self.assertIn(action["tier"], TIERS) + self.assertIn(action["color"], COLORS) + self.assertIn(action["subject_kind"], {"file", "dir"}) + self.assertEqual(set(action["reason"]), {"zh", "en"}) + self.assertFalse(action["default_selected"]) + for slot in ("open_by", "referenced_in", "incoming_symlinks", "shape"): + self.assertIn(slot, action["guard"]) + + def test_group_options_map_onto_real_actions(self) -> None: + group_ids = set() + for group in self.plan["groups"]: + group_ids.add(group["group_id"]) + self.assertIn(group["kind"], {"pair", "duplicate", "regenerable"}) + self.assertTrue(group["options"], group["group_id"]) + keys = [option["key"] for option in group["options"]] + self.assertIn(group["default_choice"], keys) + for option in group["options"]: + self.assertTrue(option["action_ids"]) + for action_id in option["action_ids"]: + self.assertIn(action_id, self.by_id, option["key"]) + self.assertEqual(self.by_id[action_id]["group_id"], group["group_id"]) + self.assertTrue(self.by_id[action_id]["approvable"]) + delete_ids = [ + aid for aid in option["action_ids"] if self.by_id[aid]["kind"] == "delete" + ] + if delete_ids: + self.assertIn("allow-permanent-delete", option["requires"]) + member_subjects = set(group["members"]) + for member in member_subjects: + self.assertIn(member, group["member_labels"]) + for action in self.actions: + if action["group_id"] is not None: + self.assertIn(action["group_id"], group_ids) + + def test_group_kinds_present(self) -> None: + kinds = {group["kind"] for group in self.plan["groups"]} + self.assertEqual(kinds, {"pair", "duplicate", "regenerable"}) + pair = next(g for g in self.plan["groups"] if g["kind"] == "pair") + option_keys = {option["key"] for option in pair["options"]} + self.assertIn("keep_all", option_keys) + self.assertEqual( + sum(1 for key in option_keys if key.startswith("keep:")), 3 + ) + duplicate = next(g for g in self.plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(duplicate["evidence"], "sha256") + self.assertIsNotNone(duplicate["sha256"]) + regenerable = next(g for g in self.plan["groups"] if g["kind"] == "regenerable") + self.assertEqual( + {option["key"] for option in regenerable["options"]}, {"trash_all", "delete_all"} + ) + + def test_holds_are_never_approvable(self) -> None: + holds = [action for action in self.actions if action["kind"] == "hold"] + self.assertGreaterEqual(len(holds), 4) + for action in holds: + self.assertFalse(action["approvable"], action["id"]) + self.assertFalse(action["reroutable"], action["id"]) + self.assertIsNone(action["destination"]) + self.assertIsNone(action["destination_portable"]) + self.assertIsNotNone(action["hold_reason"]) + self.assertIn(action["tier"], HOLD_TIERS) + tiers = {action["tier"] for action in holds} + self.assertTrue({"aging", "forbidden", "in_use", "referenced"} <= tiers) + + def test_guard_evidence_on_in_use_and_referenced(self) -> None: + in_use = next(a for a in self.actions if a["tier"] == "in_use") + self.assertTrue(in_use["guard"]["open_by"]) + self.assertIn("pid", in_use["guard"]["open_by"][0]) + referenced = next(a for a in self.actions if a["tier"] == "referenced") + self.assertTrue(referenced["guard"]["referenced_in"]) + first = referenced["guard"]["referenced_in"][0] + self.assertIn("file", first) + self.assertIn("line", first) + + def test_reroutable_actions_stay_in_the_allowed_rule_set(self) -> None: + reroutable = [action for action in self.actions if action["reroutable"]] + self.assertTrue(reroutable) + self.assertTrue(any(action["rule"] == "unknown-ext" for action in reroutable)) + for action in reroutable: + self.assertIn(action["rule"], REROUTABLE_RULES, action["id"]) + self.assertEqual(action["kind"], "move") + self.assertTrue(action["approvable"]) + self.assertIn(action["destination_key"], self.plan["names"]) + + def test_destination_keys_resolve_through_names(self) -> None: + for action in self.actions: + if action["destination_key"] is None: + self.assertIsNone(action["destination"]) + continue + self.assertIn(action["destination_key"], self.plan["names"]) + relative = self.plan["names"][action["destination_key"]] + self.assertIn(relative, action["destination_portable"]) + + def test_delete_actions_require_the_flag(self) -> None: + deletes = [action for action in self.actions if action["kind"] == "delete"] + self.assertTrue(deletes) + for action in deletes: + self.assertIn("allow-permanent-delete", action["requires"]) + self.assertIn(action["tier"], {"regenerable", "duplicate", "cold"}) + self.assertGreater(action["reclaims_bytes"], 0) + + def test_moves_reclaim_nothing_and_carry_a_restore_line(self) -> None: + for action in self.actions: + if action["kind"] == "move": + self.assertEqual(action["reclaims_bytes"], 0) + self.assertIn("mv ", action["restore_method"]["zh"]) + self.assertIn(action["filename"], action["destination_portable"]) + + def test_portable_paths_never_leak_a_user_name(self) -> None: + seen = 0 + for where, value in _portable_values(self.plan): + seen += 1 + self.assertTrue(value.startswith("$HOME/"), "{0}: {1}".format(where, value)) + self.assertGreater(seen, 10) + self.assertEqual(self.plan["source_root_portable"], "$HOME/Downloads") + + def test_summary_matches_the_actions(self) -> None: + summary = self.plan["summary"] + by_kind = {} + by_tier = {} + for action in self.actions: + by_kind[action["kind"]] = by_kind.get(action["kind"], 0) + 1 + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + self.assertEqual(summary["by_kind"], by_kind) + self.assertEqual(summary["by_tier"], by_tier) + self.assertEqual(summary["proposed_reclaimed_bytes"], 0) + self.assertEqual(summary["entries"], summary["files"] + summary["dirs"]) + + def test_every_colour_follows_from_the_decision(self) -> None: + """The buttons follow this field, so it may never be an opinion.""" + + for action in self.actions: + if action["kind"] == "hold": + self.assertEqual(action["color"], "red", action["id"]) + elif action["kind"] == "move": + expected = "yellow" if action["reroutable"] else "green" + self.assertEqual(action["color"], expected, action["id"]) + elif action["tier"] == "regenerable": + self.assertEqual(action["color"], "green", action["id"]) + else: + self.assertEqual(action["color"], "yellow", action["id"]) + colours = {action["color"] for action in self.actions} + self.assertEqual(colours, COLORS) + + def test_no_red_action_can_ever_be_approved(self) -> None: + for action in self.actions: + if action["color"] == "red": + self.assertFalse(action["approvable"], action["id"]) + + def test_a_group_is_yellow_unless_it_is_only_build_output(self) -> None: + self.assertTrue(self.plan["groups"]) + kinds = {group["kind"] for group in self.plan["groups"]} + self.assertIn("regenerable", kinds) + for group in self.plan["groups"]: + expected = "green" if group["kind"] == "regenerable" else "yellow" + self.assertEqual(group["color"], expected, group["group_id"]) + + def test_mess_block_describes_the_whole_folder(self) -> None: + mess = self.plan["mess"] + self.assertIn(mess["color"], COLORS) + self.assertEqual(mess["color_source"], "rule") + self.assertTrue(0 <= mess["score"] <= 100) + self.assertEqual( + set(mess["counts"]), {"loose", "aging", "pending", "groups", "top_dirs"} + ) + for key, value in mess["counts"].items(): + self.assertIsInstance(value, int, key) + self.assertGreaterEqual(value, 0, key) + self.assertEqual(set(mess["reason"]), {"zh", "en"}) + self.assertEqual(mess["counts"]["groups"], len(self.plan["groups"])) + self.assertEqual(mess["counts"]["loose"], self.plan["summary"]["entries"]) + self.assertEqual(mess["counts"]["top_dirs"], self.plan["summary"]["dirs"]) + + def test_by_color_counts_every_action_exactly_once(self) -> None: + by_color = self.plan["summary"]["by_color"] + self.assertEqual(set(by_color), COLORS) + self.assertEqual( + sum(slot["actions"] for slot in by_color.values()), len(self.actions) + ) + for colour in COLORS: + wearing = [a for a in self.actions if a["color"] == colour] + self.assertEqual(by_color[colour]["actions"], len(wearing), colour) + self.assertEqual( + by_color[colour]["bytes"], sum(a["size_bytes"] for a in wearing), colour + ) + + def test_sensitive_action_says_contents_were_never_read(self) -> None: + sensitive = next(a for a in self.actions if a["tier"] == "sensitive") + self.assertTrue(sensitive["rule"].startswith("sensitive:")) + self.assertIn("未读取内容", sensitive["reason"]["zh"]) + self.assertIn("60_敏感信息/待转移", sensitive["destination_portable"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_grouping.py b/tests/test_cfo_grouping.py new file mode 100644 index 0000000..4646107 --- /dev/null +++ b/tests/test_cfo_grouping.py @@ -0,0 +1,201 @@ +"""Pairing, duplicates and regenerable build output.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import grouping +from carl_file_organizer.config import resolve_config +from carl_file_organizer.grouping import ( + archive_stem, + copy_base, + dir_stem, + file_sha256, + find_duplicates, + find_regenerable, + pair_archives, +) +from carl_file_organizer.scanner import scan_top_level + +from cfo_helpers import NOW, write # noqa: E402 - the path shim has to run first + + +def scan(root: Path, profile: str = "tiered"): + config = resolve_config(root, profile=profile, lang="zh") + entries, _notes = scan_top_level(root, config, now=NOW, read_finder_tags=False) + return config, entries + + +class NameArithmeticTests(unittest.TestCase): + def test_copy_base(self) -> None: + self.assertEqual(copy_base("foo (1).pdf"), ("foo.pdf", True)) + self.assertEqual(copy_base("foo_1.pdf"), ("foo.pdf", True)) + self.assertEqual(copy_base("Archive 2"), ("Archive", True)) + self.assertEqual(copy_base("foo.pdf"), ("foo.pdf", False)) + + def test_archive_stem_only_answers_for_archives(self) -> None: + self.assertEqual(archive_stem("Archive.zip"), "archive") + self.assertEqual(archive_stem("Archive (1).zip"), "archive") + self.assertEqual(archive_stem("bundle.tar.gz"), "bundle") + self.assertEqual(archive_stem("xxx-v2.zip"), "xxx-v2") + self.assertIsNone(archive_stem("notes.md")) + + def test_dir_stem(self) -> None: + self.assertEqual(dir_stem("Archive (1)"), "archive") + self.assertEqual(dir_stem("Archive 2"), "archive") + self.assertEqual(dir_stem("xxx-v2"), "xxx-v2") + + +class PairTests(unittest.TestCase): + def test_pair_archive_variants(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "z") + write(root / "Archive (1).zip", "z") + (root / "Archive").mkdir() + write(root / "Archive" / "inside.txt") + (root / "Archive 2").mkdir() + write(root / "Archive 2" / "inside.txt") + write(root / "xxx-v2.zip", "z") + (root / "xxx-v2").mkdir() + write(root / "xxx-v2" / "inside.txt") + (root / "xxx").mkdir() + write(root / "xxx" / "inside.txt") + + config, entries = scan(root) + groups = pair_archives(entries, config) + by_stem = {group.stem: group for group in groups} + + self.assertIn("archive", by_stem) + self.assertEqual( + sorted(member.name for member in by_stem["archive"].members), + ["Archive", "Archive (1).zip", "Archive 2", "Archive.zip"], + ) + self.assertIn("xxx-v2", by_stem) + self.assertEqual( + sorted(member.name for member in by_stem["xxx-v2"].members), + ["xxx-v2", "xxx-v2.zip"], + ) + self.assertNotIn("xxx", by_stem) + + def test_a_lone_archive_is_not_a_pair(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "solo.zip", "z") + config, entries = scan(root) + self.assertEqual(pair_archives(entries, config), []) + + +class DuplicateTests(unittest.TestCase): + def test_identical_content_groups_by_sha256(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "identical bytes\n") + write(root / "a (1).pdf", "identical bytes\n") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=True) + + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0].evidence, "sha256") + self.assertIsNotNone(groups[0].sha256) + self.assertEqual([m.name for m in groups[0].members], ["a (1).pdf", "a.pdf"]) + + def test_same_name_shape_but_different_bytes_is_only_a_name_pattern(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "export.png", "AAAA") + write(root / "export_1.png", "BBBB") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=True) + + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0].evidence, "name-pattern") + self.assertIsNone(groups[0].sha256) + + def test_without_hashing_only_names_are_compared(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "identical bytes\n") + write(root / "a (1).pdf", "identical bytes\n") + _config, entries = scan(root) + + groups = find_duplicates(entries, hash_duplicates=False) + self.assertEqual([group.evidence for group in groups], ["name-pattern"]) + + def test_excluded_subjects_never_form_a_duplicate_group(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "a.pdf", "same\n") + write(root / "a (1).pdf", "same\n") + _config, entries = scan(root) + everything = [entry.subject_id for entry in entries] + self.assertEqual( + find_duplicates(entries, hash_duplicates=True, exclude=everything), [] + ) + + def test_file_sha256_gives_up_above_the_cap(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "big.bin", "0123456789") + self.assertIsNone(file_sha256(target, max_bytes=4)) + self.assertIsNotNone(file_sha256(target, max_bytes=4096)) + + +class RegenerableTests(unittest.TestCase): + def test_scan_respects_depth_and_never_enters_git(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "package.json", "{}") + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + write(root / "proj" / ".git" / "objects" / "node_modules" / "keep", "x") + write(root / "proj" / "a" / "b" / "c" / "node_modules" / "deep.js", "x") + write(root / "proj" / "a" / "b" / "node_modules" / "ok.js", "x") + + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + relatives = sorted(hit.relative for hit in hits) + + self.assertIn("proj/node_modules", relatives) + self.assertIn("proj/a/b/node_modules", relatives) + self.assertNotIn("proj/a/b/c/node_modules", relatives) + self.assertFalse([r for r in relatives if ".git" in r]) + + def test_hit_carries_a_bounded_size_and_a_subject_id(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "a.js", "0123456789") + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0].dir_stats.files, 1) + self.assertEqual(hits[0].dir_stats.bytes, 10) + self.assertFalse(hits[0].dir_stats.truncated) + self.assertEqual(len(hits[0].subject_id), 16) + + def test_a_matched_folder_is_not_descended_into(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "pkg" / "node_modules" / "x.js", "x") + config, entries = scan(root) + hits = find_regenerable(entries, config, max_depth=3, now=NOW) + self.assertEqual([hit.relative for hit in hits], ["proj/node_modules"]) + + def test_forbidden_folders_are_left_alone(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "vault" / ".obsidian" / "config", "{}") + write(root / "vault" / "node_modules" / "x.js", "x") + config, entries = scan(root) + self.assertEqual(find_regenerable(entries, config, max_depth=3, now=NOW), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_guard.py b/tests/test_cfo_guard.py new file mode 100644 index 0000000..3fb5859 --- /dev/null +++ b/tests/test_cfo_guard.py @@ -0,0 +1,315 @@ +"""The safety net, with every outside dependency injected. + +Nothing in this file may consult the real machine: no lsof, no launch agents, +no crontab, no home directory scan. Every check gets its inputs handed to it. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import guard +from carl_file_organizer.config import resolve_config +from carl_file_organizer.guard import ( + config_references, + dir_shape, + file_shape, + incoming_symlinks, + open_handles, + path_forms, + recently_modified, +) +from carl_file_organizer.scanner import make_entry + +from cfo_helpers import NOW, age, fake_lsof, write # noqa: E402 - the path shim runs first + + +def tiered(root: Path): + return resolve_config(root, profile="tiered", lang="zh") + + +class OpenHandleTests(unittest.TestCase): + def test_a_held_file_reports_the_process(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "session.log", "line\n") + with open(str(target), "r", encoding="utf-8"): + result = open_handles( + target, 5.0, lsof_runner=fake_lsof({"session.log": "python3,48213"}) + ) + self.assertTrue(result) + self.assertEqual(result.status, "ok") + self.assertEqual(result.handles[0]["command"], "python3") + self.assertEqual(result.handles[0]["pid"], 48213) + + def test_an_untouched_file_reports_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "quiet.txt") + result = open_handles(target, 5.0, lsof_runner=fake_lsof({})) + self.assertFalse(result) + self.assertEqual(result.status, "ok") + + def test_a_timeout_is_unknown_rather_than_a_crash(self) -> None: + import subprocess + + def runner(argv, timeout_s): + raise subprocess.TimeoutExpired(argv, timeout_s) + + result = open_handles(Path("/nonexistent"), 0.1, lsof_runner=runner) + self.assertEqual(result.status, "unknown") + self.assertFalse(result.handles) + + def test_a_broken_runner_is_unknown_too(self) -> None: + def runner(argv, timeout_s): + raise OSError("no lsof here") + + self.assertEqual(open_handles(Path("/tmp"), 0.1, lsof_runner=runner).status, "unknown") + + def test_directories_are_asked_with_plus_d(self) -> None: + seen = {} + + class _Result(object): + stdout = "" + returncode = 1 + + def runner(argv, timeout_s): + seen["argv"] = list(argv) + return _Result() + + with tempfile.TemporaryDirectory() as temporary: + open_handles(Path(temporary), 1.0, lsof_runner=runner, is_dir=True) + self.assertIn("+d", seen["argv"]) + + def test_spotlight_and_friends_are_not_counted_as_users(self) -> None: + # A file that just landed in Downloads is open in mdworker_shared within + # seconds. Counting that would hold back every fresh download. + self.assertTrue(guard.is_observer("mdworker_shared")) + self.assertTrue(guard.is_observer("quicklookd")) + self.assertFalse(guard.is_observer("tail")) + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "photo.png") + result = open_handles( + target, 5.0, lsof_runner=fake_lsof({"photo.png": "mdworker_shared,900"}) + ) + self.assertFalse(result.handles) + self.assertEqual(result.status, "ok") + + def test_finder_preview_daemons_are_observers_too(self) -> None: + # lsof cuts its COMMAND column off, so the QuickLook thumbnail agent + # usually arrives under a truncated name. Both spellings, and the rest + # of the Finder preview and iCloud crowd, only look at the file. + for command in ( + "com.apple.quicklook.ThumbnailsAgent", + "com.apple.quicklook.ThumbnailsA", + "QuickLookUIService", + "Finder", + "mds_stores", + "suggestd", + "photoanalysisd", + "cloudd", + "bird", + ): + self.assertTrue(guard.is_observer(command), command) + self.assertFalse(guard.is_observer("Preview")) + + def test_a_truncated_quicklook_handle_is_not_a_user(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + target = write(Path(temporary) / "poster.png") + result = open_handles( + target, + 5.0, + lsof_runner=fake_lsof( + {"poster.png": "com.apple.quicklook.ThumbnailsA,71204"} + ), + ) + self.assertFalse(result.handles) + self.assertEqual(result.status, "ok") + + def test_batched_probe_attributes_each_hit_to_its_own_path(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + busy = write(root / "busy.log") + quiet = write(root / "quiet.log") + table = guard.probe_open_handles( + [busy, quiet], [], 1.0, lsof_runner=fake_lsof({"busy.log": "tail,777"}) + ) + self.assertTrue(table[str(busy)].handles) + self.assertFalse(table[str(quiet)].handles) + + +class ReferenceTests(unittest.TestCase): + def test_the_three_spellings_are_all_generated(self) -> None: + home = Path("/home/example") + target = home / "Downloads" / "tool.sh" + forms = path_forms(target, home) + + # The absolute path is spelled the way this platform spells it, and on + # Windows the forward-slash spelling comes along too: a config file + # there carries whichever one its author typed. + expected = [str(target)] + if str(target) != target.as_posix(): + expected.append(target.as_posix()) + expected += ["~/Downloads/tool.sh", "$HOME/Downloads/tool.sh"] + + self.assertEqual(forms, expected) + self.assertIn("/home/example/Downloads/tool.sh", forms) + + def test_a_launch_agent_reference_reports_the_line_number(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh", "#!/bin/sh\n") + plist = write( + home / "fake-LaunchAgents" / "com.example.backup.plist", + "\nhello\n" + "$HOME/Downloads/backup-tool.sh\n\n", + ) + config = tiered(home / "Downloads") + + found = config_references(target, config, home, reference_files=[plist]) + + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["line"], 3) + self.assertEqual(found[0]["form"], "$HOME/Downloads/backup-tool.sh") + self.assertTrue(found[0]["file"].startswith("$HOME/")) + + def test_a_tilde_spelling_in_a_shell_rc_also_counts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh") + rc = write(home / "fake-zshrc", "\n\nalias b='~/Downloads/backup-tool.sh'\n") + config = tiered(home / "Downloads") + + found = config_references(target, config, home, reference_files=[rc]) + + self.assertEqual(found[0]["line"], 3) + self.assertEqual(found[0]["form"], "~/Downloads/backup-tool.sh") + + def test_an_unrelated_file_matches_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + rc = write(home / "fake-zshrc", "export PATH=$HOME/bin:$PATH\n") + config = tiered(home / "Downloads") + self.assertEqual(config_references(target, config, home, reference_files=[rc]), []) + + def test_a_missing_reference_file_is_skipped_quietly(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + config = tiered(home / "Downloads") + self.assertEqual( + config_references( + target, config, home, reference_files=[home / "nowhere.plist"] + ), + [], + ) + + +class SymlinkTests(unittest.TestCase): + def test_a_symlink_in_a_scan_root_points_back_at_the_entry(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "backup-tool.sh") + binaries = home / "bin" + binaries.mkdir(parents=True) + (binaries / "backup-tool").symlink_to(target) + config = tiered(home / "Downloads") + + found = incoming_symlinks(target, config, home, scan_roots=[binaries]) + + self.assertEqual(found, ["$HOME/bin/backup-tool"]) + + def test_a_symlink_into_a_folder_counts_for_the_folder(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + inside = write(home / "Downloads" / "proj" / "run.sh") + binaries = home / "bin" + binaries.mkdir(parents=True) + (binaries / "run").symlink_to(inside) + config = tiered(home / "Downloads") + + found = incoming_symlinks( + home / "Downloads" / "proj", config, home, scan_roots=[binaries] + ) + self.assertEqual(found, ["$HOME/bin/run"]) + + def test_no_scan_roots_means_no_findings(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + target = write(home / "Downloads" / "notes.md") + config = tiered(home / "Downloads") + self.assertEqual(incoming_symlinks(target, config, home, scan_roots=[]), []) + + +class ShapeTests(unittest.TestCase): + def test_a_git_file_is_a_worktree_and_a_git_folder_is_a_repo(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + worktree = root / "topic-worktree" + write(worktree / ".git", "gitdir: /elsewhere/.git/worktrees/topic\n") + repo = root / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + + self.assertIn("git-worktree", dir_shape(worktree).labels) + self.assertIn("git-dir", dir_shape(repo, count_uncommitted=False).labels) + + def test_uncommitted_count_comes_from_the_injected_runner(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + repo = Path(temporary) / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + + class _Result(object): + returncode = 0 + stdout = " M a.py\n?? b.py\n" + + shape = dir_shape(repo, git_status_runner=lambda path, timeout: _Result()) + self.assertEqual(shape.uncommitted, 2) + + def test_virtualenv_dotenv_and_node_project_are_flagged(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + write(project / "package.json", "{}") + write(project / ".env", "A=1\n") + (project / ".venv").mkdir(parents=True) + + labels = dir_shape(project).labels + self.assertIn("venv", labels) + self.assertIn("dotenv", labels) + self.assertIn("node-project", labels) + + def test_file_shape_sees_the_shebang(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + script = write(Path(temporary) / "run.sh", "#!/bin/sh\necho hi\n") + self.assertIn("shebang", file_shape(script)) + + # Windows has no execute bit: os.chmod there only moves the read-only flag, + # and os.access(X_OK) answers from the file extension. The label is a + # POSIX fact, so the test that pins it is a POSIX test. + @unittest.skipIf(os.name == "nt", "Windows 没有执行位") + def test_file_shape_sees_the_executable_bit(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + script = write(Path(temporary) / "run.sh", "#!/bin/sh\necho hi\n") + os.chmod(str(script), 0o755) + self.assertIn("executable", file_shape(script)) + + +class RecentTests(unittest.TestCase): + def test_ten_minute_window(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + fresh = age(write(Path(temporary) / "fresh.md"), hours=0.05) + settled = age(write(Path(temporary) / "settled.md"), hours=5) + fresh_entry = make_entry(fresh, now=NOW, read_finder_tags=False) + settled_entry = make_entry(settled, now=NOW, read_finder_tags=False) + + self.assertTrue(recently_modified(fresh_entry, NOW, 10)) + self.assertFalse(recently_modified(settled_entry, NOW, 10)) + self.assertFalse(recently_modified(fresh_entry, NOW, 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_manifest.py b/tests/test_cfo_manifest.py new file mode 100644 index 0000000..5dcb1bb --- /dev/null +++ b/tests/test_cfo_manifest.py @@ -0,0 +1,234 @@ +"""Manifests, tag lists, the audit log and the human review log.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import json +import tempfile +import unittest +from datetime import date, datetime +from pathlib import Path + +from carl_file_organizer import manifest + + +#: The exact header the reference script has been writing since August. +LEGACY_HEADER = ( + "moved_at", + "status", + "original_path", + "target_path", + "size_bytes", + "reason", + "restore_method", +) + +STAMP = datetime(2026, 9, 2, 10, 20, 11) + + +def _row(**kw): + row = { + "moved_at": "2026-09-02T10:20:11+09:00", + "status": "MOVED", + "original_path": "$HOME/Downloads/report.pdf", + "target_path": "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf", + "size_bytes": 2411008, + "reason": "文档按类型归档", + "restore_method": 'mv "$HOME/Downloads/20_知识库/文档资料/PDF/report.pdf" "$HOME/Downloads/"', + "kind": "move", + "action_id": "9c1f0a7b2d3e4f55", + } + row.update(kw) + return row + + +class ManifestColumnTests(unittest.TestCase): + def test_manifest_columns_prefix_compatible(self) -> None: + self.assertEqual(manifest.MANIFEST_COLUMNS[:7], LEGACY_HEADER) + self.assertEqual(manifest.MANIFEST_COLUMNS[7:], ("kind", "action_id")) + + def test_manifest_path_names_match_the_reference_layout(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + path = manifest.manifest_path(managed, "Downloads", "moves", STAMP) + self.assertEqual(path.name, "downloads-moves-2026-09-02_10-20-11.tsv") + tags = manifest.tag_list_path(managed, "Downloads", STAMP) + self.assertEqual(tags.name, "downloads-move-tags-2026-09-02_10-20-11.txt") + self.assertEqual(manifest.audit_path(managed).name, "audit.jsonl") + + def test_unknown_series_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(ValueError): + manifest.manifest_path(Path(temporary), "Downloads", "whatever", STAMP) + + def test_write_and_read_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "downloads-moves-2026-09-02_10-20-11.tsv" + manifest.write_manifest(path, [_row(), _row(status="PLANNED")]) + text = path.read_text(encoding="utf-8") + self.assertTrue(text.startswith("\t".join(manifest.MANIFEST_COLUMNS))) + rows = manifest.read_manifest(path) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]["kind"], "move") + self.assertEqual(rows[1]["status"], "PLANNED") + self.assertFalse(list(Path(temporary).glob("*.tmp"))) + + def test_tabs_and_newlines_cannot_break_a_row(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.manifest_path(Path(temporary), "Downloads", "moves", STAMP) + manifest.write_manifest(path, [_row(reason="a\tb\nc")]) + rows = manifest.read_manifest(path) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["reason"], "a b c") + + def test_seven_column_manifest_still_reads(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "downloads-moves-2026-08-30_15-06-43.tsv" + path.write_text( + "\t".join(LEGACY_HEADER) + + "\n" + + "\t".join( + [ + "2026-08-30T15:06:43+09:00", + "MOVED", + "$HOME/Downloads/api-credential.txt", + "$HOME/Downloads/60_敏感信息/待转移/api-credential.txt", + "98", + "敏感命名文件直接移入待转移，不读取内容", + 'mv "$HOME/Downloads/60_敏感信息/待转移/api-credential.txt" "$HOME/Downloads/"', + ] + ) + + "\n", + encoding="utf-8", + ) + rows = manifest.read_manifest(path) + self.assertEqual(rows[0]["status"], "MOVED") + self.assertEqual(rows[0].get("kind", ""), "") + + +class TagListTests(unittest.TestCase): + def test_tag_list_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.tag_list_path(Path(temporary), "Downloads", STAMP) + manifest.write_tag_list(path, ["$HOME/Downloads/a.txt", "$HOME/Downloads/b.txt"]) + self.assertEqual( + manifest.read_tag_list(path), + ["$HOME/Downloads/a.txt", "$HOME/Downloads/b.txt"], + ) + + def test_missing_tag_list_reads_as_empty(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + self.assertEqual(manifest.read_tag_list(Path(temporary) / "nope.txt"), []) + + +class AuditTests(unittest.TestCase): + def test_append_audit_is_one_json_object_per_line(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.audit_path(Path(temporary)) + manifest.append_audit([{"action_id": "a", "status": "moved"}], path) + manifest.append_audit([{"action_id": "b", "status": "skipped"}], path) + lines = path.read_text(encoding="utf-8").strip().split("\n") + self.assertEqual(len(lines), 2) + self.assertEqual(json.loads(lines[1])["action_id"], "b") + records = manifest.read_audit(path) + self.assertEqual([item["status"] for item in records], ["moved", "skipped"]) + + def test_a_broken_line_does_not_stop_the_read(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.audit_path(Path(temporary)) + path.write_text('{"status": "moved"}\nnot json\n\n', encoding="utf-8") + self.assertEqual(len(manifest.read_audit(path)), 1) + + +class ReviewLogTests(unittest.TestCase): + def test_review_log_format(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + first = manifest.append_review_log( + managed, "zh", date(2026, 9, 2), ["移动了 3 项。"] + ) + self.assertEqual(first.name, "人工调整日志.md") + text = first.read_text(encoding="utf-8") + self.assertTrue(text.startswith("# 人工调整日志")) + self.assertIn("## 2026-09-02", text) + self.assertIn("- 移动了 3 项。", text) + + manifest.append_review_log( + managed, "zh", date(2026, 9, 2), ["复查结果：dry-run actions=0。"] + ) + text = first.read_text(encoding="utf-8") + self.assertEqual(text.count("## 2026-09-02"), 1) + self.assertIn("- 复查结果：dry-run actions=0。", text) + self.assertLess( + text.index("- 移动了 3 项。"), text.index("- 复查结果：dry-run actions=0。") + ) + + manifest.append_review_log(managed, "zh", date(2026, 9, 3), ["第二天。"]) + text = first.read_text(encoding="utf-8") + self.assertIn("## 2026-09-03", text) + self.assertLess(text.index("## 2026-09-02"), text.index("## 2026-09-03")) + # A later day must not swallow the earlier day's bullets. + self.assertLess(text.index("- 复查结果"), text.index("## 2026-09-03")) + + def test_english_log_has_its_own_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.append_review_log( + Path(temporary), "en", date(2026, 9, 2), ["Moved 3 items."] + ) + self.assertEqual(path.name, "review-log.md") + self.assertIn("# Review log", path.read_text(encoding="utf-8")) + + def test_no_bullets_writes_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = manifest.append_review_log(Path(temporary), "zh", date(2026, 9, 2), []) + self.assertFalse(path.exists()) + + +class LatestRunTests(unittest.TestCase): + def _manifest(self, managed: Path, series: str, stamp: str, rows=1, status="MOVED"): + path = managed / "downloads-{0}-{1}.tsv".format(series, stamp) + manifest.write_manifest( + path, + [ + _row(status=status, moved_at="{0}+09:00".format(stamp.replace("_", "T"))) + for _ in range(rows) + ], + ) + return path + + def test_status_reads_only_manifests(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) / "00_下载目录管理" + managed.mkdir() + # A very fresh folder with no manifest means nobody ever tidied it. + (managed / "plan.json").write_text("{}", encoding="utf-8") + (managed / "notes.md").write_text("hello", encoding="utf-8") + self.assertIsNone(manifest.latest_run(managed)) + + self._manifest(managed, "moves", "2026-08-30_15-06-43", rows=2) + self._manifest(managed, "moves", "2026-09-01_03-11-13", rows=3) + self._manifest(managed, "undo", "2026-09-02_04-00-00", rows=9) + + latest = manifest.latest_run(managed) + self.assertIsNotNone(latest) + self.assertEqual(latest.path.name, "downloads-moves-2026-09-01_03-11-13.tsv") + self.assertEqual(latest.series, "moves") + self.assertEqual(latest.rows, 3) + self.assertEqual(latest.done, 3) + + def test_conflict_moves_is_its_own_series(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + managed = Path(temporary) + self._manifest(managed, "conflict-moves", "2026-09-01_03-12-04") + latest = manifest.latest_run(managed) + self.assertEqual(latest.series, "conflict-moves") + + def test_missing_folder_is_not_an_error(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + self.assertIsNone(manifest.latest_run(Path(temporary) / "nope")) + self.assertEqual(manifest.list_runs(Path(temporary) / "nope"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_notes.py b/tests/test_cfo_notes.py new file mode 100644 index 0000000..e45e1ad --- /dev/null +++ b/tests/test_cfo_notes.py @@ -0,0 +1,287 @@ +"""The agent explanation layer, and the promise that it decides nothing. + +Every test here is really the same test asked a different way: a notes file is +prose, and prose cannot approve, move, delete, reroute or recolour anything. +The one thing it may change is the folder's own colour, and even then the rule's +answer stays in the file next to it. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import notes as notes_module +from carl_file_organizer.cli import main +from carl_file_organizer.config import resolve_config +from carl_file_organizer.notes import NotesError, merge_notes, validate_notes +from carl_file_organizer.planner import build_plan + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + age, + write, +) + + +def sample_plan(root: Path): + config = resolve_config(root, profile="tiered", lang="zh") + return build_plan( + root, + config=config, + now=NOW, + home=root, + guard_options=GUARD_OFFLINE, + trash_backend="finder", + ) + + +def populated(root: Path) -> None: + age(write(root / "notes.md"), hours=100) + age(write(root / "mystery.zzz"), hours=100) + age(write(root / "bundle.zip", "PK\x03\x04"), hours=100) + age(write(root / "bundle" / "inner.txt"), hours=100) + age(root / "bundle", hours=100) + + +class MergeTests(unittest.TestCase): + def setUp(self) -> None: + self._temporary = tempfile.TemporaryDirectory() + self.root = Path(self._temporary.name) + populated(self.root) + self.plan = sample_plan(self.root) + self.action_id = self.plan["actions"][0]["id"] + + def tearDown(self) -> None: + self._temporary.cleanup() + + def test_notes_land_in_their_own_block(self) -> None: + merged = merge_notes( + self.plan, + { + "folder_line": "这个目录基本是上周的下载", + "actions": { + self.action_id: { + "what": "上周的会议纪要", + "why": "扩展名规则命中", + "if_removed": "在备份里还有一份", + } + }, + }, + ) + block = merged["notes"] + self.assertEqual(block["schema"], notes_module.NOTES_SCHEMA) + self.assertEqual(block["folder_line"], "这个目录基本是上周的下载") + self.assertEqual(block["actions"][self.action_id]["what"], "上周的会议纪要") + self.assertEqual(block["warnings"], []) + + def test_the_original_plan_is_untouched(self) -> None: + before = copy.deepcopy(self.plan) + merge_notes(self.plan, {"folder_line": "一句话"}) + self.assertEqual(self.plan, before) + + def test_no_action_field_is_ever_changed(self) -> None: + before = copy.deepcopy(self.plan["actions"]) + merged = merge_notes( + self.plan, + { + "actions": { + self.action_id: { + "what": "x", + "why": "y", + "if_removed": "z", + "color": "green", + "approvable": True, + "destination": "/somewhere/else", + "kind": "delete", + } + } + }, + ) + self.assertEqual(merged["actions"], before) + self.assertEqual( + set(merged["notes"]["actions"][self.action_id]), {"what", "why", "if_removed"} + ) + self.assertTrue( + any("unknown field" in line for line in merged["notes"]["warnings"]) + ) + + def test_an_unknown_id_only_warns(self) -> None: + merged = merge_notes( + self.plan, {"actions": {"0000000000000000": {"what": "谁知道呢"}}} + ) + self.assertEqual(merged["notes"]["actions"], {}) + self.assertTrue( + any("no such id" in line for line in merged["notes"]["warnings"]) + ) + + def test_a_group_note_needs_a_real_group(self) -> None: + group_id = self.plan["groups"][0]["group_id"] + merged = merge_notes( + self.plan, + {"groups": {group_id: {"what": "压缩包和它解出来的目录"}, "nope": {"what": "x"}}}, + ) + self.assertIn(group_id, merged["notes"]["groups"]) + self.assertNotIn("nope", merged["notes"]["groups"]) + + def test_the_folder_colour_can_be_overridden_and_says_so(self) -> None: + rule_colour = self.plan["mess"]["color"] + merged = merge_notes( + self.plan, {"mess": {"color": "red", "line": "我看过了，这里比数字显示的更乱"}} + ) + self.assertEqual(merged["mess"]["color"], "red") + self.assertEqual(merged["mess"]["color_source"], "notes") + self.assertEqual(merged["mess"]["rule_color"], rule_colour) + self.assertEqual(merged["mess"]["line"], "我看过了，这里比数字显示的更乱") + + def test_without_an_override_the_rule_keeps_the_credit(self) -> None: + merged = merge_notes(self.plan, {"folder_line": "一句话"}) + self.assertEqual(merged["mess"]["color_source"], "rule") + self.assertEqual(merged["mess"]["color"], merged["mess"]["rule_color"]) + + def test_an_invented_colour_is_refused_not_stored(self) -> None: + merged = merge_notes(self.plan, {"mess": {"color": "purple"}}) + self.assertEqual(merged["mess"]["color"], self.plan["mess"]["color"]) + self.assertTrue(any("mess.color" in line for line in merged["notes"]["warnings"])) + + def test_merging_twice_replaces_rather_than_piles_up(self) -> None: + once = merge_notes(self.plan, {"folder_line": "第一版"}) + twice = merge_notes(once, {"folder_line": "第二版"}) + self.assertEqual(twice["notes"]["folder_line"], "第二版") + self.assertEqual(twice["mess"]["rule_color"], self.plan["mess"]["color"]) + + def test_empty_and_non_string_sentences_are_dropped(self) -> None: + merged = merge_notes( + self.plan, + {"actions": {self.action_id: {"what": " ", "why": 7, "if_removed": "真的"}}}, + ) + self.assertEqual( + merged["notes"]["actions"][self.action_id], {"if_removed": "真的"} + ) + + def test_a_very_long_sentence_is_trimmed_not_discarded(self) -> None: + merged = merge_notes( + self.plan, {"actions": {self.action_id: {"what": "字" * 5000}}} + ) + self.assertEqual( + len(merged["notes"]["actions"][self.action_id]["what"]), notes_module.MAX_TEXT + ) + + def test_an_unknown_top_level_field_warns(self) -> None: + _cleaned, warnings = validate_notes(self.plan, {"nonsense": 1}) + self.assertTrue(any("unknown top level" in line for line in warnings)) + + def test_a_notes_file_that_is_not_an_object_is_refused(self) -> None: + with self.assertRaises(NotesError): + merge_notes(self.plan, ["not", "an", "object"]) + + +class LoadTests(unittest.TestCase): + def test_broken_json_names_the_file(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "notes.json" + path.write_text("{not json", encoding="utf-8") + with self.assertRaises(NotesError) as caught: + notes_module.load_notes(path) + self.assertIn("notes.json", str(caught.exception)) + + def test_a_missing_file_is_a_notes_error_not_an_oserror(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(NotesError): + notes_module.load_notes(Path(temporary) / "absent.json") + + def test_a_json_list_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "notes.json" + path.write_text("[]", encoding="utf-8") + with self.assertRaises(NotesError): + notes_module.load_notes(path) + + +class BuildCommandTests(unittest.TestCase): + def test_build_writes_the_notes_into_the_plan_on_disk(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text( + json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8" + ) + notes_path = root / "notes.json" + notes_path.write_text( + json.dumps( + { + "folder_line": "上周的下载都在这儿", + "mess": {"color": "yellow", "line": "看着乱，其实好收拾"}, + "actions": {plan["actions"][0]["id"]: {"what": "会议纪要"}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + code = main(["build", str(plan_path), "--notes", str(notes_path)]) + self.assertEqual(code, 0) + + written = json.loads(plan_path.read_text(encoding="utf-8")) + self.assertEqual(written["notes"]["folder_line"], "上周的下载都在这儿") + self.assertEqual(written["mess"]["color"], "yellow") + self.assertEqual(written["mess"]["color_source"], "notes") + + def test_build_can_write_somewhere_else_and_leave_the_plan_alone(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + notes_path = root / "notes.json" + notes_path.write_text('{"folder_line": "一句话"}', encoding="utf-8") + out_path = root / "out" / "annotated.json" + + self.assertEqual( + main( + [ + "build", + str(plan_path), + "--notes", + str(notes_path), + "--output", + str(out_path), + ] + ), + 0, + ) + self.assertIsNone(json.loads(plan_path.read_text(encoding="utf-8"))["notes"]) + self.assertEqual( + json.loads(out_path.read_text(encoding="utf-8"))["notes"]["folder_line"], + "一句话", + ) + + def test_build_without_notes_is_a_no_op_rewrite(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + populated(root) + plan = sample_plan(root) + plan_path = root / "plan.json" + plan_path.write_text(json.dumps(plan, ensure_ascii=False), encoding="utf-8") + + self.assertEqual(main(["build", str(plan_path)]), 0) + self.assertIsNone(json.loads(plan_path.read_text(encoding="utf-8"))["notes"]) + + def test_build_on_a_broken_plan_exits_two_without_a_traceback(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = Path(temporary) / "plan.json" + plan_path.write_text("not json at all", encoding="utf-8") + self.assertEqual(main(["build", str(plan_path)]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_opener.py b/tests/test_cfo_opener.py new file mode 100644 index 0000000..a5cb735 --- /dev/null +++ b/tests/test_cfo_opener.py @@ -0,0 +1,108 @@ +"""Showing a path to a person, on three platforms, from one of them. + +The command shape is asserted rather than the effect, because the effect is a +window opening on somebody's screen. What matters and what can be checked here +is that no shell is involved and that the path arrives as one argument. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from carl_file_organizer import opener + + +class Launcher(object): + """Stands in for subprocess.Popen and records the call.""" + + def __init__(self, error: bool = False) -> None: + self.calls = [] + self.kwargs = [] + self.error = error + + def __call__(self, argv, **kwargs): + self.calls.append(list(argv)) + self.kwargs.append(dict(kwargs)) + if self.error: + raise OSError("no such file manager") + return object() + + +class CommandShapeTests(unittest.TestCase): + def test_macos_selects_the_item_itself(self) -> None: + argv = opener.reveal_command(Path("/tmp/report.pdf"), platform="darwin") + self.assertEqual(argv[:2], ["open", "-R"]) + self.assertEqual(argv[2], os.path.abspath("/tmp/report.pdf")) + + def test_windows_passes_one_argument_with_the_comma_attached(self) -> None: + argv = opener.reveal_command(Path("C:/Users/someone/report.pdf"), platform="win32") + self.assertEqual(argv[0], "explorer") + self.assertEqual(len(argv), 2) + self.assertTrue(argv[1].startswith("/select,")) + self.assertNotIn(" ", argv[1][:8]) + self.assertIn("report.pdf", argv[1]) + + def test_linux_opens_the_parent_because_xdg_open_cannot_select(self) -> None: + with mock.patch.object(opener.shutil, "which", return_value="/usr/bin/xdg-open"): + argv = opener.reveal_command(Path("/tmp/deep/report.pdf"), platform="linux") + self.assertEqual(argv, ["/usr/bin/xdg-open", os.path.abspath("/tmp/deep")]) + + def test_linux_without_xdg_open_has_no_command_at_all(self) -> None: + with mock.patch.object(opener.shutil, "which", return_value=None): + self.assertIsNone(opener.reveal_command(Path("/tmp/x.pdf"), platform="linux")) + self.assertFalse(opener.supported(platform="linux")) + + def test_a_hostile_filename_stays_one_argument(self) -> None: + name = "$(rm -rf ~) & echo pwned.pdf" + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / name + for platform in ("darwin", "win32"): + argv = opener.reveal_command(target, platform=platform) + self.assertTrue(any(name in part for part in argv), platform) + self.assertEqual(len(argv), 3 if platform == "darwin" else 2, platform) + + +class RevealTests(unittest.TestCase): + def test_reveal_never_goes_through_a_shell(self) -> None: + launcher = Launcher() + self.assertTrue( + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + ) + self.assertIsInstance(launcher.calls[0], list) + self.assertFalse(launcher.kwargs[0].get("shell", False)) + self.assertNotIn("shell", launcher.kwargs[0]) + + def test_reveal_does_not_hold_on_to_the_file_manager_output(self) -> None: + launcher = Launcher() + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + self.assertIn("stdout", launcher.kwargs[0]) + self.assertIn("stderr", launcher.kwargs[0]) + + def test_a_missing_file_manager_is_false_not_an_exception(self) -> None: + launcher = Launcher(error=True) + self.assertFalse( + opener.reveal(Path("/tmp/x.pdf"), platform="darwin", runner=launcher) + ) + + def test_no_command_means_no_launch_attempt(self) -> None: + launcher = Launcher() + with mock.patch.object(opener.shutil, "which", return_value=None): + self.assertFalse( + opener.reveal(Path("/tmp/x.pdf"), platform="linux", runner=launcher) + ) + self.assertEqual(launcher.calls, []) + + @unittest.skipUnless(sys.platform == "darwin", "the macOS default only exists on macOS") + def test_this_machine_can_show_a_path(self) -> None: + self.assertTrue(opener.supported()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_planner.py b/tests/test_cfo_planner.py new file mode 100644 index 0000000..dc365d2 --- /dev/null +++ b/tests/test_cfo_planner.py @@ -0,0 +1,669 @@ +"""The decision chain end to end, with the clock and the safety net injected.""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import tempfile +import unittest +from datetime import timedelta +from pathlib import Path + +from carl_file_organizer import paths +from carl_file_organizer.config import resolve_config +from carl_file_organizer.planner import SCHEMA_VERSION, build_plan, plan_paths, write_plan + +from cfo_helpers import ( # noqa: E402 - the path shim has to run first + GUARD_OFFLINE, + NOW, + action_for, + actions_for, + age, + fake_lsof, + guard_with, + write, +) + + +def plan_for(root: Path, *, lang: str = "zh", profile: str = "tiered", **kwargs): + """Build a plan whose home is the temp root, so portable paths stay $HOME/.""" + + config = kwargs.pop("config", None) or resolve_config(root, profile=profile, lang=lang) + options = kwargs.pop("guard_options", GUARD_OFFLINE) + return build_plan( + root, + config=config, + now=kwargs.pop("now", NOW), + home=kwargs.pop("home", root), + guard_options=options, + trash_backend=kwargs.pop("trash_backend", "finder"), + **kwargs + ) + + +class EnvelopeTests(unittest.TestCase): + def test_schema_and_top_level_shape(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md", "# hi\n"), hours=100) + plan = plan_for(root) + + self.assertEqual(plan["schema_version"], SCHEMA_VERSION) + for key in ( + "generator", "created_at", "now", "lang", "lang_source", "profile", + "source_root", "source_root_portable", "managed_dir", "managed_dir_portable", + "names", "capabilities", "scan_policy", "disk", "summary", "pinned", + "actions", "groups", "approved_action_ids", "overrides", + "approved_at", "approved_by", + ): + self.assertIn(key, plan) + self.assertEqual(plan["approved_action_ids"], []) + self.assertEqual(plan["overrides"], []) + self.assertEqual(plan["now"], NOW.isoformat()) + + def test_every_portable_path_starts_at_home(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "photo.png"), hours=100) + plan = plan_for(root) + + seen = 0 + stack = [plan] + while stack: + node = stack.pop() + if isinstance(node, dict): + for key, value in node.items(): + if isinstance(value, str) and key.endswith("_portable"): + seen += 1 + self.assertTrue(value.startswith("$HOME/"), "{0}={1}".format(key, value)) + else: + stack.append(value) + elif isinstance(node, list): + stack.extend(node) + self.assertGreater(seen, 3) + + def test_absolute_paths_appear_only_where_they_are_allowed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + allowed = {"source", "destination", "source_root", "managed_dir"} + for action in plan["actions"]: + for key, value in action.items(): + if key in allowed or not isinstance(value, str): + continue + self.assertNotIn(str(root), value, key) + + def test_capabilities_are_reported(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + plan = plan_for(root) + self.assertIn( + plan["capabilities"]["trash_backend"], {"finder", "gio", "none"} + ) + self.assertIn("finder_tags", plan["capabilities"]) + self.assertIn("permanent_delete_offered", plan["capabilities"]) + + def test_summary_counts_match_the_actions(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + age(write(root / "fresh.md"), hours=1) + (root / "folder").mkdir() + age(root / "folder", hours=100) + plan = plan_for(root) + + by_kind = {} + by_tier = {} + for action in plan["actions"]: + by_kind[action["kind"]] = by_kind.get(action["kind"], 0) + 1 + by_tier[action["tier"]] = by_tier.get(action["tier"], 0) + 1 + self.assertEqual(plan["summary"]["by_kind"], by_kind) + self.assertEqual(plan["summary"]["by_tier"], by_tier) + self.assertEqual( + plan["summary"]["entries"], + plan["summary"]["files"] + plan["summary"]["dirs"], + ) + self.assertEqual(plan["summary"]["proposed_reclaimed_bytes"], 0) + + def test_plan_paths_and_write_plan(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + config = resolve_config(root, profile="tiered", lang="zh") + where = plan_paths(config) + self.assertEqual(where.managed.name, "00_下载目录管理") + plan = plan_for(root, config=config) + write_plan(plan, where.plan) + self.assertTrue(where.plan.is_file()) + + +class SettlingTests(unittest.TestCase): + def test_forty_seven_hours_holds_and_forty_nine_moves(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = write(root / "report.pdf", "%PDF\n") + + age(target, hours=47) + plan = plan_for(root) + hold = action_for(plan, "report.pdf", "hold") + self.assertEqual(hold["rule"], "hold:aging") + self.assertEqual(hold["tier"], "aging") + self.assertFalse(hold["approvable"]) + self.assertIsNone(hold["destination"]) + self.assertIn("ready_at", hold["hold_reason"]) + ready = hold["hold_reason"]["ready_at"] + self.assertTrue(ready > NOW.isoformat()) + + age(target, hours=49) + plan = plan_for(root) + move = action_for(plan, "report.pdf", "move") + self.assertEqual(move["rule"], "ext:.pdf") + self.assertEqual(move["tier"], "routine") + self.assertTrue(move["approvable"]) + self.assertIn("20_知识库/文档资料/PDF", move["destination_portable"]) + + def test_archives_wait_a_full_week(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "tool.zip", "PK"), hours=100) + plan = plan_for(root) + self.assertEqual(action_for(plan, "tool.zip", "hold")["rule"], "hold:aging") + + def test_ready_at_is_the_modification_time_plus_the_window(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf"), hours=10) + plan = plan_for(root) + hold = action_for(plan, "report.pdf", "hold") + expected = (NOW - timedelta(hours=10) + timedelta(hours=48)).isoformat() + self.assertEqual(hold["hold_reason"]["ready_at"][:16], expected[:16]) + + +class PriorityTests(unittest.TestCase): + def test_pinned_beats_sensitive(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "api-password-notes.md"), hours=100) + config = resolve_config(root, profile="tiered", lang="zh") + config.profile["pinned"] = list(config.profile["pinned"]) + [ + "api-password-notes.md" + ] + plan = plan_for(root, config=config) + + found = actions_for(plan, "api-password-notes.md") + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["rule"], "hold:pinned") + self.assertEqual(found[0]["tier"], "pinned") + + def test_forbidden_beats_sensitive(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + vault = root / "secret-vault.photoslibrary" + vault.mkdir() + age(vault, hours=100) + plan = plan_for(root) + + found = actions_for(plan, "secret-vault.photoslibrary") + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["rule"], "hold:forbidden") + self.assertIn("forbidden:suffix", found[0]["hold_reason"]["label"]) + + def test_a_sensitive_folder_moves_whole_and_ignores_the_settling_period(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + folder = root / "api-credential-dump" + write(folder / "inside.txt") + age(folder, hours=1) + plan = plan_for(root) + + move = action_for(plan, "api-credential-dump", "move") + self.assertEqual(move["tier"], "sensitive") + self.assertEqual(move["subject_kind"], "dir") + self.assertTrue(move["rule"].startswith("sensitive:")) + self.assertIn("60_敏感信息/待转移/api-credential-dump", move["destination_portable"]) + self.assertIn("未读取内容", move["reason"]["zh"]) + + def test_sensitive_beats_the_extension_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "server.pem"), hours=1) + plan = plan_for(root) + move = action_for(plan, "server.pem", "move") + self.assertEqual(move["rule"], "sensitive:suffix") + + def test_pairing_beats_the_plain_folder_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "PK") + write(root / "Archive" / "inside.txt") + for item in (root / "Archive.zip", root / "Archive"): + age(item, hours=300) + plan = plan_for(root) + + folder = action_for(plan, "Archive", "move") + self.assertEqual(folder["rule"], "pair") + self.assertIn("00_收件箱/待判断/Archive/Archive", folder["destination_portable"]) + self.assertIsNotNone(folder["group_id"]) + + +class FolderTests(unittest.TestCase): + def test_a_folder_moves_whole_and_keeps_its_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "foo" / "inner.txt") + age(root / "foo", hours=100) + plan = plan_for(root) + + move = action_for(plan, "foo", "move") + self.assertEqual(move["rule"], "dir:pending") + self.assertTrue(move["reroutable"]) + self.assertTrue(move["destination_portable"].endswith("00_收件箱/待判断/foo")) + self.assertIsNotNone(move["dir_stats"]) + + def test_a_scratch_copy_only_gets_a_hint(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "project-worktree-topic" / "inner.txt") + age(root / "project-worktree-topic", hours=100) + plan = plan_for(root) + + move = action_for(plan, "project-worktree-topic", "move") + self.assertEqual(move["rule"], "dir:derivative") + self.assertIn("derivative-copy:-worktree-", move["hints"]) + + def test_unknown_extensions_wait_in_pending_and_can_be_rerouted(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "dataset.parquet"), hours=100) + plan = plan_for(root) + + move = action_for(plan, "dataset.parquet", "move") + self.assertEqual(move["rule"], "unknown-ext") + self.assertTrue(move["reroutable"]) + self.assertEqual(move["confidence"], 0.55) + self.assertIn(move["destination_key"], plan["names"]) + + +class BoundaryTests(unittest.TestCase): + def test_partitions_hidden_files_and_symlinks_are_skipped(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "notes.md"), hours=100) + write(root / ".hidden.md") + for name in ("00_收件箱", "20_知识库", "00_下载目录管理"): + (root / name).mkdir() + (root / "link.md").symlink_to(root / "notes.md") + write(root / "carl-file-organizer-approved-1.json", "{}") + + plan = plan_for(root) + names = {action["filename"] for action in plan["actions"]} + + self.assertEqual(names, {"notes.md"}) + self.assertEqual(plan["scan_policy"]["skipped"]["partitions"], 3) + self.assertEqual(plan["scan_policy"]["skipped"]["symlinks"], 1) + self.assertEqual(plan["scan_policy"]["skipped"]["artifacts"], 1) + + def test_a_taken_destination_is_redirected_and_never_overwritten(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf", "new\n"), hours=100) + write(root / "20_知识库" / "文档资料" / "PDF" / "report.pdf", "already here\n") + + plan = plan_for(root) + move = action_for(plan, "report.pdf", "move") + + self.assertEqual(move["rule"], "conflict") + self.assertEqual(move["destination_key"], "inbox.dup") + self.assertEqual( + move["destination_portable"], + "$HOME/00_收件箱/重复待确认/report-2026-09-02_12-00-00.pdf", + ) + self.assertFalse(move["reroutable"]) + self.assertEqual( + (root / "20_知识库" / "文档资料" / "PDF" / "report.pdf").read_text(), "already here\n" + ) + + def test_a_redirected_copy_keeps_a_double_suffix_and_a_folder_keeps_none(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "release.tar.gz", "new\n"), hours=400) + write(root / "90_归档" / "压缩包" / "其他压缩包" / "release.tar.gz", "already here\n") + folder = root / "web-prototype" + age(write(folder / "index.html", "

    hi

    \n"), hours=100) + age(folder, hours=100) + write(root / "00_收件箱" / "待判断" / "web-prototype" / "keep.txt", "taken\n") + + plan = plan_for(root) + archive = action_for(plan, "release.tar.gz", "move") + self.assertEqual( + archive["destination_portable"], + "$HOME/00_收件箱/重复待确认/release-2026-09-02_12-00-00.tar.gz", + ) + + moved_dir = action_for(plan, "web-prototype", "move") + self.assertEqual( + moved_dir["destination_portable"], + "$HOME/00_收件箱/重复待确认/web-prototype-2026-09-02_12-00-00", + ) + + +class LanguageTests(unittest.TestCase): + def test_english_names_are_used_end_to_end(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "report.pdf"), hours=100) + age(write(root / "dataset.parquet"), hours=100) + plan = plan_for(root, lang="en") + + self.assertEqual(plan["lang"], "en") + self.assertIn( + "20_Library/Documents/PDF", action_for(plan, "report.pdf")["destination_portable"] + ) + self.assertIn( + "00_Inbox/Pending", action_for(plan, "dataset.parquet")["destination_portable"] + ) + self.assertEqual(plan["names"]["library.docs.pdf"], "20_Library/Documents/PDF") + + +class GroupTests(unittest.TestCase): + def test_pair_group_options_all_point_at_real_approvable_actions(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "Archive.zip", "PK") + write(root / "Archive" / "inside.txt") + for item in (root / "Archive.zip", root / "Archive"): + age(item, hours=300) + plan = plan_for(root) + + group = next(g for g in plan["groups"] if g["kind"] == "pair") + by_id = {action["id"]: action for action in plan["actions"]} + keys = [option["key"] for option in group["options"]] + + self.assertIn("keep_all", keys) + self.assertIn(group["default_choice"], keys) + self.assertTrue(group["default_choice"].startswith("keep:")) + for option in group["options"]: + self.assertTrue(option["action_ids"]) + for action_id in option["action_ids"]: + self.assertIn(action_id, by_id) + self.assertTrue(by_id[action_id]["approvable"]) + self.assertEqual(by_id[action_id]["group_id"], group["group_id"]) + if any(by_id[a]["kind"] == "delete" for a in option["action_ids"]): + self.assertIn("allow-permanent-delete", option["requires"]) + + def test_identical_files_form_a_sha256_group_that_defaults_to_keeping_one(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "guide.pdf", "same bytes\n"), hours=100) + age(write(root / "guide (1).pdf", "same bytes\n"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + group = next(g for g in plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(group["evidence"], "sha256") + self.assertIsNotNone(group["sha256"]) + self.assertTrue(group["default_choice"].startswith("keep:")) + self.assertGreater(group["potential_bytes"], 0) + + def test_lookalike_names_default_to_keeping_everything(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "export.png", "AAAA"), hours=100) + age(write(root / "export_1.png", "BBBB"), hours=100) + plan = plan_for(root, hash_duplicates=True) + + group = next(g for g in plan["groups"] if g["kind"] == "duplicate") + self.assertEqual(group["evidence"], "name-pattern") + self.assertEqual(group["default_choice"], "keep_all") + self.assertEqual(group["potential_bytes"], 0) + self.assertFalse( + [ + action + for action in plan["actions"] + if action["group_id"] == group["group_id"] and action["kind"] == "delete" + ] + ) + + def test_regenerable_output_offers_trash_and_delete_only(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "package.json", "{}") + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root) + + group = next(g for g in plan["groups"] if g["kind"] == "regenerable") + self.assertEqual( + {option["key"] for option in group["options"]}, {"trash_all", "delete_all"} + ) + self.assertEqual(group["default_choice"], "trash_all") + self.assertEqual(group["member_labels"][group["members"][0]], "proj/node_modules") + kinds = { + action["kind"] + for action in plan["actions"] + if action["group_id"] == group["group_id"] + } + self.assertEqual(kinds, {"trash", "delete"}) + + def test_permanent_delete_can_be_turned_off_entirely(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root, offer_permanent_delete=False) + + self.assertFalse([a for a in plan["actions"] if a["kind"] == "delete"]) + self.assertFalse(plan["capabilities"]["permanent_delete_offered"]) + + def test_without_a_trash_backend_nothing_offers_the_trash(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "proj" / "node_modules" / "left-pad" / "index.js", "x") + age(root / "proj", hours=100) + plan = plan_for(root, trash_backend="none") + + self.assertFalse([a for a in plan["actions"] if a["kind"] == "trash"]) + group = next(g for g in plan["groups"] if g["kind"] == "regenerable") + self.assertEqual([option["key"] for option in group["options"]], ["delete_all"]) + + +class GuardChainTests(unittest.TestCase): + def test_an_open_file_is_held_with_the_process_named(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = age(write(root / "session.log", "line\n"), hours=100) + with open(str(target), "r", encoding="utf-8"): + plan = plan_for( + root, + guard_options=guard_with( + lsof_enabled=True, + lsof_runner=fake_lsof({"session.log": "python3,48213"}), + ), + ) + + hold = action_for(plan, "session.log", "hold") + self.assertEqual(hold["rule"], "hold:in_use") + self.assertEqual(hold["tier"], "in_use") + self.assertFalse(hold["approvable"]) + self.assertEqual(hold["guard"]["open_by"][0]["pid"], 48213) + self.assertIn("python3", hold["reason"]["zh"]) + + def test_a_file_the_finder_is_previewing_is_not_held(self) -> None: + # Clicking a file in a Finder window hands it to the QuickLook thumbnail + # agent, and lsof reports that agent under a truncated name. Reading it + # as a user would hold back everything the user happens to be looking at. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "poster.png", "png\n"), hours=100) + plan = plan_for( + root, + guard_options=guard_with( + lsof_enabled=True, + lsof_runner=fake_lsof( + {"poster.png": "com.apple.quicklook.ThumbnailsA,71204"} + ), + ), + ) + + kinds = {a["kind"] for a in plan["actions"] if a["filename"] == "poster.png"} + self.assertNotIn("hold", kinds) + self.assertNotEqual(action_for(plan, "poster.png")["tier"], "in_use") + + def test_a_referenced_script_is_held_with_the_line_number(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + age(write(root / "backup-tool.sh", "#!/bin/sh\n"), hours=100) + plist = write( + home / "com.example.backup.plist", + "\n$HOME/Downloads/backup-tool.sh\n\n", + ) + plan = plan_for( + root, home=home, guard_options=guard_with(reference_files=[plist]) + ) + + hold = action_for(plan, "backup-tool.sh", "hold") + self.assertEqual(hold["rule"], "hold:referenced") + self.assertEqual(hold["tier"], "referenced") + self.assertEqual(hold["guard"]["referenced_in"][0]["line"], 2) + self.assertIn(":2", hold["reason"]["zh"]) + + def test_allow_referenced_downgrades_the_hold_to_a_warning(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + age(write(root / "backup-tool.sh", "#!/bin/sh\n"), hours=100) + plist = write( + home / "com.example.backup.plist", + "\n$HOME/Downloads/backup-tool.sh\n\n", + ) + plan = plan_for( + root, + home=home, + allow_referenced=True, + guard_options=guard_with(reference_files=[plist]), + ) + + move = action_for(plan, "backup-tool.sh", "move") + self.assertTrue(move["approvable"]) + self.assertIn("referenced-warning", move["hints"]) + self.assertTrue(move["guard"]["referenced_in"]) + self.assertIn("改引用", move["detail"]["zh"]) + self.assertTrue(plan["scan_policy"]["allow_referenced"]) + + def test_a_symlink_pointing_in_is_a_reference_too(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + root = home / "Downloads" + root.mkdir() + target = age(write(root / "backup-tool.sh"), hours=100) + binaries = home / "bin" + binaries.mkdir() + (binaries / "backup-tool").symlink_to(target) + + plan = plan_for(root, home=home, guard_options=guard_with(scan_roots=[binaries])) + hold = action_for(plan, "backup-tool.sh", "hold") + + self.assertEqual(hold["tier"], "referenced") + self.assertEqual(hold["guard"]["incoming_symlinks"], ["$HOME/bin/backup-tool"]) + + def test_a_git_file_means_a_worktree_and_a_worktree_is_never_moved(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + worktree = root / "topic-worktree-fix" + write(worktree / ".git", "gitdir: /elsewhere/.git/worktrees/topic\n") + age(worktree, hours=100) + + plan = plan_for(root) + hold = action_for(plan, "topic-worktree-fix", "hold") + + self.assertEqual(hold["rule"], "hold:referenced") + self.assertEqual(hold["tier"], "referenced") + self.assertIn("git-worktree", hold["guard"]["shape"]) + self.assertIn("worktree", hold["reason"]["en"]) + + def test_a_git_folder_only_gets_a_hint(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + repo = root / "repo" + write(repo / ".git" / "HEAD", "ref: refs/heads/main\n") + write(repo / "package.json", "{}") + age(repo, hours=100) + + config = resolve_config(root, profile="tiered", lang="zh") + + class _Status(object): + returncode = 0 + stdout = " M a.py\n" + + plan = plan_for( + root, + config=config, + guard_options=guard_with(git_status_runner=lambda path, timeout: _Status()), + ) + move = action_for(plan, "repo", "move") + + self.assertTrue(move["approvable"]) + self.assertIn("git-repo", move["hints"]) + self.assertIn("node-project", move["hints"]) + self.assertIn("git-dir", move["guard"]["shape"]) + self.assertEqual(move["guard"]["uncommitted"], 1) + + def test_something_touched_ten_minutes_ago_waits(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "dataset.parquet"), hours=0.05) + plan = plan_for(root) + + hold = action_for(plan, "dataset.parquet", "hold") + self.assertEqual(hold["rule"], "hold:aging") + self.assertEqual(hold["tier"], "aging") + self.assertIn("十分钟", hold["reason"]["zh"]) + + +class ColdTests(unittest.TestCase): + def test_a_large_file_gains_a_disposal_option(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "clip.mp4", "x" * 4096), hours=100) + plan = plan_for(root, large_bytes=1024) + + kinds = {action["kind"] for action in actions_for(plan, "clip.mp4")} + self.assertEqual(kinds, {"move", "trash", "delete"}) + trash = action_for(plan, "clip.mp4", "trash") + self.assertEqual(trash["tier"], "cold") + self.assertEqual(trash["rule"], "cold:large") + self.assertEqual(trash["reclaims_bytes"], 4096) + self.assertEqual(action_for(plan, "clip.mp4", "move")["reclaims_bytes"], 0) + + def test_a_settled_installer_gains_a_disposal_option(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + age(write(root / "tool.dmg", "x"), hours=400) + plan = plan_for(root) + + trash = action_for(plan, "tool.dmg", "trash") + self.assertEqual(trash["rule"], "cold:installer") + self.assertEqual(trash["tier"], "cold") + + +class SimpleProfileTests(unittest.TestCase): + def test_the_flat_profile_still_works(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root / "notes.txt", "hello") + plan = plan_for(root, profile="simple", lang="en") + + move = action_for(plan, "notes.txt", "move") + self.assertEqual(move["destination_key"], "documents") + self.assertTrue(move["destination_portable"].endswith("Documents/notes.txt")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_portability.py b/tests/test_cfo_portability.py new file mode 100644 index 0000000..b1921ca --- /dev/null +++ b/tests/test_cfo_portability.py @@ -0,0 +1,182 @@ +"""Reading a plan that was written on the other kind of machine. + +A plan.json and a report.html both travel: made on a Mac, opened on Windows, or +the reverse. So one token is stored, both tokens are understood, and the token +a person actually sees is decided at render time and nowhere else. +""" + +from __future__ import annotations + +import cfo_path # noqa: F401 - puts the skill's scripts dir on sys.path + +import os +import sys +import unittest +from pathlib import Path + +from carl_file_organizer import guard, paths, trash + + +class TokenTests(unittest.TestCase): + def test_only_one_token_is_ever_written(self) -> None: + home = Path("/home/someone") + self.assertEqual( + paths.portable(home / "Downloads" / "a.pdf", home), "$HOME/Downloads/a.pdf" + ) + self.assertTrue(paths.portable(home, home).startswith("$HOME")) + + def test_a_stored_path_always_uses_forward_slashes(self) -> None: + home = Path("/home/someone") + stored = paths.portable(home / "Downloads" / "deep" / "a.pdf", home) + self.assertNotIn("\\", stored) + + def test_both_tokens_expand_to_the_same_place(self) -> None: + home = Path("/home/someone") + expected = home / "Downloads" / "a.pdf" + for text in ( + "$HOME/Downloads/a.pdf", + "%USERPROFILE%/Downloads/a.pdf", + "%USERPROFILE%\\Downloads\\a.pdf", + ): + self.assertEqual(paths.expand_portable(text, home), expected, text) + + def test_a_bare_token_is_the_home_directory(self) -> None: + home = Path("/home/someone") + for text in ("$HOME", "%USERPROFILE%"): + self.assertEqual(paths.expand_portable(text, home), home, text) + + def test_something_that_is_not_a_token_is_left_alone(self) -> None: + home = Path("/home/someone") + self.assertEqual(paths.expand_portable("/etc/hosts", home), Path("/etc/hosts")) + + def test_split_home_token_reports_which_spelling_it_found(self) -> None: + self.assertEqual(paths.split_home_token("$HOME/a/b"), ("$HOME", "a/b")) + self.assertEqual( + paths.split_home_token("%USERPROFILE%\\a\\b"), ("%USERPROFILE%", "a/b") + ) + self.assertIsNone(paths.split_home_token("relative/a")) + self.assertIsNone(paths.split_home_token(None)) + + def test_display_swaps_the_token_only_for_windows_eyes(self) -> None: + self.assertEqual( + paths.display_portable("$HOME/Downloads/a.pdf", platform="darwin"), + "$HOME/Downloads/a.pdf", + ) + self.assertEqual( + paths.display_portable("$HOME/Downloads/a.pdf", platform="win32"), + "%USERPROFILE%\\Downloads\\a.pdf", + ) + self.assertEqual(paths.display_portable("$HOME", platform="win32"), "%USERPROFILE%") + + def test_display_leaves_a_path_it_does_not_recognise_untouched(self) -> None: + self.assertEqual( + paths.display_portable("/etc/hosts", platform="win32"), "/etc/hosts" + ) + + def test_home_token_follows_the_platform(self) -> None: + self.assertEqual(paths.home_token("win32"), "%USERPROFILE%") + self.assertEqual(paths.home_token("darwin"), "$HOME") + self.assertEqual(paths.home_token("linux"), "$HOME") + + +class PlanHomeTests(unittest.TestCase): + def test_a_windows_written_plan_still_gives_up_its_home(self) -> None: + plan = { + "source_root": "/home/someone/Downloads", + "source_root_portable": "%USERPROFILE%\\Downloads", + } + self.assertEqual(paths.plan_home(plan), Path("/home/someone")) + + def test_a_mac_written_plan_gives_up_its_home_too(self) -> None: + plan = { + "source_root": "/home/someone/Downloads", + "source_root_portable": "$HOME/Downloads", + } + self.assertEqual(paths.plan_home(plan), Path("/home/someone")) + + def test_a_stripped_plan_falls_back_to_this_machine(self) -> None: + self.assertEqual(paths.plan_home({}), Path.home()) + + +class WindowsBackendTests(unittest.TestCase): + """The Windows code paths, exercised from whatever machine is running this.""" + + def test_the_flag_word_always_keeps_undo(self) -> None: + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_ALLOWUNDO) + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_NOCONFIRMATION) + self.assertTrue(trash.WINDOWS_FLAGS & trash.FOF_SILENT) + + def test_the_path_argument_ends_in_two_nuls(self) -> None: + argument = trash.windows_path_argument(Path("C:/Users/someone/a.pdf")) + self.assertTrue(argument.endswith("\0\0")) + self.assertEqual(argument.count("\0"), 2) + self.assertTrue(argument.startswith(os.path.abspath("C:/Users/someone/a.pdf"))) + + def test_a_recycle_bin_call_that_returns_nonzero_is_a_failure(self) -> None: + with self.assertRaises(trash.TrashError) as caught: + trash.move_to_trash( + Path("nowhere.pdf"), + backend=trash.WINDOWS, + win_runner=lambda path: (124, False), + ) + self.assertIn("124", str(caught.exception)) + + def test_an_aborted_call_is_a_failure_too(self) -> None: + with self.assertRaises(trash.TrashError): + trash.move_to_trash( + Path("nowhere.pdf"), + backend=trash.WINDOWS, + win_runner=lambda path: (0, True), + ) + + def test_a_success_that_left_the_file_behind_is_still_a_failure(self) -> None: + here = Path(__file__) + with self.assertRaises(trash.TrashError) as caught: + trash.move_to_trash( + here, backend=trash.WINDOWS, win_runner=lambda path: (0, False), lang="zh" + ) + self.assertIn(here.name, str(caught.exception)) + + def test_a_failed_recycle_bin_call_never_becomes_a_delete(self) -> None: + here = Path(__file__) + for runner in ( + lambda path: (1, False), + lambda path: (0, True), + lambda path: (_ for _ in ()).throw(OSError("shell32 went away")), + ): + with self.assertRaises(trash.TrashError): + trash.move_to_trash(here, backend=trash.WINDOWS, win_runner=runner) + self.assertTrue(here.exists()) + + def test_an_unknown_backend_name_is_refused_outright(self) -> None: + with self.assertRaises(trash.TrashError): + trash.move_to_trash(Path("x"), backend="dumpster") + + @unittest.skipUnless(sys.platform == "win32", "shell32 only exists on Windows") + def test_this_windows_machine_can_reach_the_recycle_bin(self) -> None: + self.assertTrue(trash.windows_available()) + self.assertEqual(trash.detect_backend(), trash.WINDOWS) + + @unittest.skipIf(sys.platform == "win32", "the negative case is for everywhere else") + def test_windows_helpers_are_inert_off_windows(self) -> None: + self.assertFalse(trash.windows_available()) + + +class LsofTests(unittest.TestCase): + def test_windows_never_claims_nobody_has_it_open(self) -> None: + """Unknown is a real answer; a guessed "nobody" would let a file move.""" + + if sys.platform == "win32": + self.assertFalse(guard.lsof_possible()) + else: + self.assertEqual(guard.lsof_possible(), guard.shutil.which("lsof") is not None) + + @unittest.skipUnless(sys.platform == "win32", "only Windows lacks lsof by definition") + def test_open_handles_on_windows_is_unknown(self) -> None: + result = guard.open_handles(Path(__file__)) + self.assertEqual(result.status, "unknown") + self.assertFalse(result.handles) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cfo_preview.py b/tests/test_cfo_preview.py new file mode 100644 index 0000000..5a2c332 --- /dev/null +++ b/tests/test_cfo_preview.py @@ -0,0 +1,368 @@ +"""The after picture, and the promise that it is only a picture. + +The preview reads plan.json and draws where things land. It decides nothing, it +fetches nothing, and it never shows anybody's home directory. Every test here +is one of those three promises, or the arithmetic that keeps the drawing honest: +one node per subject, five partitions, hold stays at the root, and a graph too +big to read collapses into an aggregate instead of a hairball. +""" + +from __future__ import annotations + +import os +import sys + +# Runs under both invocations the repo uses: ``unittest discover -s tests`` +# (which puts tests/ on the path for us) and ``unittest tests.test_cfo_preview`` +# (which does not). +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import cfo_path # noqa: E402,F401 - puts the skill's scripts dir on sys.path + +import copy +import json +import re +import tempfile +import unittest +from pathlib import Path + +from carl_file_organizer import preview +from carl_file_organizer.preview import ( + MAX_NODES, + PREVIEW_TEXT, + build_graph, + render_preview_html, + render_preview_page, +) + +FIXTURES = Path(cfo_path.FIXTURES_DIR) +PLAN_PATH = FIXTURES / "plan-v2-sample.json" +ANALYSIS_PATH = FIXTURES / "storage-analysis-sample.json" + + +def load(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +def index(graph): + return {node["id"]: node for node in graph["nodes"]} + + +class GraphShapeTest(unittest.TestCase): + """The graph mirrors the plan: one node per subject, and the tree above it.""" + + def setUp(self): + self.plan = load(PLAN_PATH) + self.graph = build_graph(self.plan) + + def test_one_node_per_subject_move_beats_hold_and_the_rest(self): + subjects = { + action["subject_id"] + for action in self.plan["actions"] + if action["kind"] in ("move", "hold") + } + files = [node for node in self.graph["nodes"] if node["type"] == "file"] + self.assertEqual(len(files), len(subjects)) + self.assertEqual(len({node["id"] for node in files}), len(files)) + + # Archive.zip carries move + trash + delete; only the move survives. + archive = index(self.graph)["f:3c4d5e6f7a8b9c0d"] + self.assertTrue(archive["moving"]) + self.assertTrue(archive["to"].startswith("00_收件箱/待判断")) + + def test_five_partitions_labelled_by_their_real_directory_names(self): + partitions = [node for node in self.graph["nodes"] if node["type"] == "partition"] + self.assertEqual(len(partitions), 5) + self.assertEqual(self.graph["stats"]["partitions"], 5) + self.assertEqual( + [node["label"] for node in partitions], + [self.plan["names"][key] for key in preview.PARTITION_ORDER], + ) + + def test_second_level_folders_are_deduplicated(self): + folders = [node for node in self.graph["nodes"] if node["type"] == "folder"] + labels = [node["label"] for node in folders] + self.assertEqual(len(labels), len(set(labels)) if len(set(labels)) == len(labels) else len(labels)) + self.assertEqual(len({node["id"] for node in folders}), len(folders)) + # Three PDFs share one 文档资料 folder node. + self.assertEqual(sum(1 for node in folders if node["label"] == "文档资料"), 1) + self.assertIn("待判断", labels) + + def test_hold_files_stay_at_the_root(self): + nodes = index(self.graph) + root_label = nodes["root"]["label"] + holds = [ + action["subject_id"] + for action in self.plan["actions"] + if action["kind"] == "hold" + ] + self.assertTrue(holds) + for subject in holds: + node = nodes["f:" + subject] + self.assertEqual(node["to"], root_label) + self.assertFalse(node["moving"]) + self.assertEqual(node["color"], "red") + + def test_every_link_points_at_a_real_node_and_now_links_cover_every_file(self): + ids = set(index(self.graph)) + for link in self.graph["links"]: + self.assertIn(link["source"], ids) + self.assertIn(link["target"], ids) + self.assertIn(link["kind"], ("now", "after")) + files = [node for node in self.graph["nodes"] if node["type"] == "file"] + now_links = [link for link in self.graph["links"] if link["kind"] == "now"] + self.assertEqual(len(now_links), len(files)) + self.assertTrue(all(link["target"] == "root" for link in now_links)) + # Child -> parent, exactly once per node that has a parent. + after_parents = [link["source"] for link in self.graph["links"] if link["kind"] == "after"] + self.assertEqual(len(after_parents), len(set(after_parents))) + + def test_stats_add_up(self): + stats = self.graph["stats"] + self.assertEqual(stats["files"], stats["moving"] + stats["staying"]) + self.assertGreater(stats["moving"], 0) + self.assertGreater(stats["staying"], 0) + # node_modules is trashed rather than moved, so it only shows up as space. + self.assertGreaterEqual(stats["reclaim_bytes"], 198311744) + + def test_no_home_directory_ever_reaches_the_graph(self): + blob = json.dumps(self.graph, ensure_ascii=False) + self.assertNotIn("/Users/", blob) + self.assertNotIn("/home/", blob) + + +class MachineLayerTest(unittest.TestCase): + """With an analysis in hand, the cleanup map joins the same drawing.""" + + def setUp(self): + self.plan = load(PLAN_PATH) + self.analysis = load(ANALYSIS_PATH) + self.graph = build_graph(self.plan, self.analysis) + + def test_machine_root_carries_every_analysis_item(self): + nodes = index(self.graph) + self.assertEqual(nodes["machine"]["type"], "machine") + self.assertEqual(nodes["machine"]["label"], self.analysis["machine"]["hostname"]) + storage = [node for node in self.graph["nodes"] if node["type"] == "storage"] + self.assertEqual(len(storage), len(self.analysis["items"])) + self.assertEqual(self.graph["stats"]["storage_items"], len(storage)) + parents = { + link["target"] + for link in self.graph["links"] + if link["source"].startswith("s:") + } + self.assertEqual(parents, {"machine"}) + self.assertIn( + {"source": "root", "target": "machine", "kind": "after"}, self.graph["links"] + ) + + def test_green_items_are_flagged_as_pending_clean(self): + nodes = index(self.graph) + derived = nodes["s:st-derived-data"] + self.assertEqual(derived["color"], "green") + self.assertTrue(derived["pending_clean"]) + self.assertIn(PREVIEW_TEXT["zh"]["tip_clean"], derived["hint"]) + chrome = nodes["s:st-chrome-profile"] + self.assertEqual(chrome["color"], "yellow") + self.assertFalse(chrome["pending_clean"]) + + def test_analysis_green_bytes_join_the_reclaim_total(self): + without = build_graph(self.plan)["stats"]["reclaim_bytes"] + green = sum( + item["size_bytes"] for item in self.analysis["items"] if item["color"] == "green" + ) + self.assertEqual(self.graph["stats"]["reclaim_bytes"], without + green) + + def test_storage_paths_stay_portable(self): + blob = json.dumps(self.graph, ensure_ascii=False) + self.assertNotIn("/Users/", blob) + self.assertIn("$HOME", blob) + + +class NodeBudgetTest(unittest.TestCase): + """Past four hundred nodes the picture stops adding dots and starts counting.""" + + def synthetic(self, count): + plan = load(PLAN_PATH) + template = plan["actions"][0] + actions = [] + for i in range(count): + action = copy.deepcopy(template) + action["id"] = "gen%06d" % i + action["subject_id"] = "gen%06d" % i + action["filename"] = "file-%06d.pdf" % i + action["size_bytes"] = 1000 + i + action["destination_portable"] = ( + "$HOME/Downloads/20_知识库/文档资料/PDF/file-%06d.pdf" % i + ) + action["destination"] = None + actions.append(action) + plan["actions"] = actions + return plan + + def test_under_the_cap_nothing_is_aggregated(self): + graph = build_graph(self.synthetic(50)) + self.assertEqual(graph["stats"]["dropped"], 0) + self.assertNotIn("agg:rest", index(graph)) + + def test_over_the_cap_the_tail_becomes_one_node(self): + graph = build_graph(self.synthetic(900)) + self.assertLessEqual(len(graph["nodes"]), MAX_NODES) + aggregate = index(graph)["agg:rest"] + self.assertGreater(aggregate["aggregate"], 0) + self.assertEqual( + aggregate["label"], PREVIEW_TEXT["zh"]["rest"].format(k=aggregate["aggregate"]) + ) + kept = [node for node in graph["nodes"] if node["type"] == "file" and node["id"] != "agg:rest"] + self.assertEqual(len(kept) + aggregate["aggregate"], 900) + # The survivors are the big ones. + self.assertEqual( + min(node["size_bytes"] for node in kept), + max( + node["size_bytes"] for node in kept + ) - (len(kept) - 1), + ) + + def test_the_cap_also_covers_the_machine_layer(self): + graph = build_graph(self.synthetic(900), load(ANALYSIS_PATH)) + self.assertLessEqual(len(graph["nodes"]), MAX_NODES) + + +class FragmentTest(unittest.TestCase): + """A fragment you can paste anywhere: no network, no leaks, no surprises.""" + + def setUp(self): + self.graph = build_graph(load(PLAN_PATH), load(ANALYSIS_PATH)) + self.fragment = render_preview_html(self.graph, lang="zh") + + def test_it_is_one_self_contained_section(self): + self.assertTrue(self.fragment.startswith("
    ")) + self.assertIn("", self.fragment) + self.assertIn("