From 136feeac962a08c3ac4194b4b1d0ef9f06130072 Mon Sep 17 00:00:00 2001 From: yw0nam Date: Fri, 10 Jul 2026 10:50:57 +0900 Subject: [PATCH 1/9] chore: remove cron/report/promote dead files and pyproject entry points Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/cron-wrapup/SKILL.md | 324 --------- .../templates/cron-wrapup-summary.md | 82 --- .../reference/optional-global-digest.md | 40 -- .claude/skills/memory-report/SKILL.md | 401 ----------- .../reference/templates/checklist.md | 19 - .../reference/templates/concept.md | 18 - .../reference/templates/decision.md | 17 - .../reference/templates/entity.md | 18 - .../reference/templates/improvement.md | 24 - .../reference/templates/question.md | 18 - .../reference/templates/summaries/monthly.md | 49 -- .../reference/templates/summaries/weekly.md | 66 -- .claude/skills/usage-report-setup/SKILL.md | 117 --- .claude/skills/wiki-approval/SKILL.md | 202 ------ DESIGN.md | 129 ---- PRODUCT.md | 76 -- .../hermes-monitoring-guide.md | 106 --- .../hermes-schema-reference.md | 169 ----- .../opencode-monitoring-guide.md | 84 --- .../opencode-schema-reference.md | 215 ------ docs/db_informations/report-pipeline.md | 109 --- docs/raw/ops_cron.md | 41 -- docs/workflows.md | 110 --- pyproject.toml | 5 - scripts/cron/_db.sh | 72 -- scripts/cron/kb-claude-code-daily-report.sh | 24 - scripts/cron/kb-cron-wrapup.sh | 45 -- scripts/cron/kb-hermes-daily-report.sh | 24 - scripts/cron/kb-ingest-papers.sh | 24 - scripts/cron/kb-memory-daily.sh | 24 - scripts/cron/kb-memory-monthly.sh | 25 - scripts/cron/kb-memory-weekly.sh | 25 - scripts/cron/kb-opencode-daily-report.sh | 24 - scripts/cron/kb-wiki-promote.sh | 44 -- scripts/cron/kb-wiki-ttl-sweep.sh | 22 - src/kb/cli/_payloads.py | 93 --- src/kb/cli/_submit.py | 48 -- src/kb/cli/claude_code_daily_report.py | 673 ------------------ src/kb/cli/db_ttl_sweep.py | 22 - src/kb/cli/hermes_daily_report.py | 226 ------ src/kb/cli/opencode_daily_report.py | 265 ------- src/kb/cli/submit_cron_run.py | 41 -- src/kb/cli/usage_reports/__init__.py | 0 src/kb/cli/usage_reports/collect.py | 640 ----------------- src/kb/cli/usage_reports/render.py | 94 --- test/test_cli_consumers.py | 190 ----- test/test_payloads.py | 151 ---- test/test_usage_daily_report.py | 215 ------ 48 files changed, 5450 deletions(-) delete mode 100644 .claude/skills/cron-wrapup/SKILL.md delete mode 100644 .claude/skills/cron-wrapup/reference/templates/cron-wrapup-summary.md delete mode 100644 .claude/skills/knowledgebase-initialize/reference/optional-global-digest.md delete mode 100644 .claude/skills/memory-report/SKILL.md delete mode 100644 .claude/skills/memory-report/reference/templates/checklist.md delete mode 100644 .claude/skills/memory-report/reference/templates/concept.md delete mode 100644 .claude/skills/memory-report/reference/templates/decision.md delete mode 100644 .claude/skills/memory-report/reference/templates/entity.md delete mode 100644 .claude/skills/memory-report/reference/templates/improvement.md delete mode 100644 .claude/skills/memory-report/reference/templates/question.md delete mode 100644 .claude/skills/memory-report/reference/templates/summaries/monthly.md delete mode 100644 .claude/skills/memory-report/reference/templates/summaries/weekly.md delete mode 100644 .claude/skills/usage-report-setup/SKILL.md delete mode 100644 .claude/skills/wiki-approval/SKILL.md delete mode 100644 DESIGN.md delete mode 100644 PRODUCT.md delete mode 100644 docs/db_informations/hermes-monitoring-guide.md delete mode 100644 docs/db_informations/hermes-schema-reference.md delete mode 100644 docs/db_informations/opencode-monitoring-guide.md delete mode 100644 docs/db_informations/opencode-schema-reference.md delete mode 100644 docs/db_informations/report-pipeline.md delete mode 100644 docs/raw/ops_cron.md delete mode 100644 docs/workflows.md delete mode 100644 scripts/cron/_db.sh delete mode 100755 scripts/cron/kb-claude-code-daily-report.sh delete mode 100755 scripts/cron/kb-cron-wrapup.sh delete mode 100755 scripts/cron/kb-hermes-daily-report.sh delete mode 100755 scripts/cron/kb-ingest-papers.sh delete mode 100755 scripts/cron/kb-memory-daily.sh delete mode 100755 scripts/cron/kb-memory-monthly.sh delete mode 100755 scripts/cron/kb-memory-weekly.sh delete mode 100755 scripts/cron/kb-opencode-daily-report.sh delete mode 100755 scripts/cron/kb-wiki-promote.sh delete mode 100755 scripts/cron/kb-wiki-ttl-sweep.sh delete mode 100644 src/kb/cli/_payloads.py delete mode 100644 src/kb/cli/_submit.py delete mode 100644 src/kb/cli/claude_code_daily_report.py delete mode 100644 src/kb/cli/db_ttl_sweep.py delete mode 100644 src/kb/cli/hermes_daily_report.py delete mode 100644 src/kb/cli/opencode_daily_report.py delete mode 100644 src/kb/cli/submit_cron_run.py delete mode 100644 src/kb/cli/usage_reports/__init__.py delete mode 100644 src/kb/cli/usage_reports/collect.py delete mode 100644 src/kb/cli/usage_reports/render.py delete mode 100644 test/test_cli_consumers.py delete mode 100644 test/test_payloads.py delete mode 100644 test/test_usage_daily_report.py diff --git a/.claude/skills/cron-wrapup/SKILL.md b/.claude/skills/cron-wrapup/SKILL.md deleted file mode 100644 index 28cd748..0000000 --- a/.claude/skills/cron-wrapup/SKILL.md +++ /dev/null @@ -1,324 +0,0 @@ ---- -name: cron-wrapup -description: Use when running the nightly KB cron wrap-up — aggregating the previous day's usage reports, memory page, wiki-promote / TTL outcomes, and `data/raw/ops/cron/` per-run log exit states into a single durable Slack-digest-stable wiki summary plus run handoff. ---- - -# cron-wrapup - -## DB-Canonical Override - -DB rows are the source of truth. Write the wrap-up summary and run handoff -through the kb-mcp MCP tools (`upsert_page`, `create_handoff`), and write the -operation log through the kb-mcp `create_operation_log` tool. Markdown under -`data/` is generated export. If any older instruction below says to lint as a -write gate or commit `data/`, ignore that instruction and use -the kb-mcp tools instead. - -## Overview - -One skill, one job: at 05:00 KST, after the night's pipeline has finished, produce a deterministic operational snapshot of what the KB cron jobs did and what the user needs to notice. The output is the single source of truth for the 09:00 Slack digest — Slack must never read memory pages directly. - -Two artefacts per run: - -1. **Wiki summary** — `data/wiki/summaries/{YYYY}/{MM}/{TARGET}-cron-wrapup.md` (`type: summary`, `subtype: cron-daily`). Body uses fixed H2 section names (`Status`, `Jobs`, `Insights`, `Action Items`, `Anomalies`, `Counters`, `Links`) so downstream parsers stay stable as prose evolves. -2. **Run handoff** — `data/handoffs/{YYYY}/{MM}/cron-wrapup/{TARGET}_{role}_handoff_NN.md` via the `handoff-document` skill. - -This skill is the **operational command layer**. Content synthesis (what happened in the world) lives in `memory-report`'s daily memory page — do not duplicate it here. Wrap-up describes what happened in the KB *pipeline* and extracts the small set of user-facing insights and follow-up actions created by that pipeline. - -**Bundled reference (self-contained — do not consult docs/ at runtime):** - -- `reference/templates/cron-wrapup-summary.md` — wiki summary skeleton with seven canonical H2 sections pre-filled with placeholder rows. Copy this verbatim and fill cells. - -For handoff authoring, import `handoff-document`. This skill does not duplicate the handoff schema. - -If anything below contradicts the lint code (`src/kb/lint/wiki.py`), the lint code wins — file an issue and update this skill. - -## When to Use - -- Daily 05:00 KST cron fires via `scripts/cron/kb-cron-wrapup.sh`. -- Manual invocation: "Run the KB cron wrap-up for {YYYY-MM-DD}". - -**Do NOT use** for: - -- Content synthesis — use `memory-report` (daily memory page). -- Wiki promotion — use `wiki-approval` (separate cron). -- Raw ingestion — manual or external scripts only; never the wrap-up. -- Re-validating the whole KB history or using lint as analysis input. The wrap-up runs the required lint gate only after writing its own summary/handoff. - -## Target - -`TARGET = YYYY-MM-DD` (yesterday in KST). The cron wrapper supplies this: - -```bash -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -``` - -If the user manually invokes "Run the KB cron wrap-up for 2026-05-20", `TARGET = 2026-05-20`. Never default to "today" — the night's pipeline writes for yesterday. - -## Repo Layout & read model - -Postgres is the source of truth; `data/` is a human-readable export. Read wiki, handoff, metrics, and cron-run inputs from the DB with `psql` (schema + recipes in `docs/db_informations/state-db-schema-reference.md`) — never open `data/**/*.md` during normal operation. Per-run cron evidence lives in `cron_runs.log_body`; archived files under `data/raw/ops/cron/` are export/debug copies only, not the source of truth. - -```bash -TARGET="$1"; psql_kb() { psql "${DATABASE_URL/+psycopg/}" -tAc "$1"; } -# Usage + memory summary bodies for the target (summary slugs are date-prefixed) -psql_kb "SELECT slug, body_md FROM pages WHERE type='summary' AND slug LIKE '$TARGET-%';" -# Canonical usage numbers when present (see docs/db_informations/state-db-schema-reference.md) -psql_kb "SELECT report_type, session_count, token_total, cost_usd, tool_error_count FROM metrics WHERE report_date='$TARGET';" -# Run handoffs from the night's pipeline -psql_kb "SELECT task_slug, handoff_id, status FROM handoffs WHERE status='ready';" -# Per-job outcomes when present (status/exit_code/timestamps) -psql_kb "SELECT job_name, status, exit_code, started_at, finished_at FROM cron_runs WHERE target='$TARGET';" -``` - -The export tree below shows where the same data also lands as human-readable markdown: - -``` -data/ -├── wiki/summaries/{YYYY}/{MM}/ # generated export of the summary pages (read via psql, above) -│ ├── {TARGET}-opencode-usage.md # DB-backed (kb-opencode-daily-report writes via kb-mcp / service layer) -│ ├── {TARGET}-claude-code-usage.md # DB-backed -│ ├── {TARGET}-hermes-usage.md # DB-backed -│ └── {TARGET}-memory.md # DB-backed (kb-memory-daily writes via kb-mcp / service layer) -├── handoffs/{YYYY}/{MM}/ -│ ├── wiki-daily-build/{TARGET}_*_handoff_*.md # run handoff for memory-daily -│ └── wiki-promote/... # if wiki-promote ran (folder per wiki-approval) -├── raw/ops/cron/{YYYY}/{MM}/ # per-run cron log files (one file per job per target) -│ ├── {TARGET}_kb-opencode-daily-report.log -│ ├── {TARGET}_kb-claude-code-daily-report.log -│ ├── {TARGET}_kb-hermes-daily-report.log -│ ├── {TARGET}_kb-memory-daily.log -│ ├── {TARGET}_kb-wiki-ttl-sweep.log -│ ├── {TARGET}_kb-wiki-promote.log -│ ├── {TARGET}_kb-memory-weekly.log # exists on Monday (after weekly run for prior ISO week) -│ └── {TARGET}_kb-memory-monthly.log # exists on day 1 (after monthly run for prior month) -│ └── {TARGET}_kb-cron-wrapup.log # committed by the shell wrapper AFTER session exits -└── log.md -``` - -> **Note:** `{TARGET}_kb-cron-wrapup.log` does not exist inside `data/` while the session is running. -> The shell wrapper records the cron run via the `kb-submit-cron-run` CLI after the session exits (not your job). -> Do not read or stage it during the session. - -`sources:` frontmatter paths are **relative to `data/`** (e.g. `wiki/summaries/...`, NOT `data/wiki/summaries/...`). - -Every wrapper uses `TARGET_DATE` (= yesterday in KST) as the filename prefix, even when its workflow target is a week or month — so the daily wrap-up's glob `{TARGET}_*.log` matches every wrapper that ran in last night's pipeline. The actual weekly/monthly period stays in the log body and the produced wiki page. - -Each run writes its own immutable log file, so the wrap-up reads exactly one file per job per target — no time-window grep required. - -## Wiki Summary Schema (the only thing lint ERRORs you for) - -```yaml ---- -type: summary -subtype: cron-daily -date: "YYYY-MM-DD" # the TARGET -created: "YYYY-MM-DD" # the run date (today in KST) -updated: "YYYY-MM-DD" -sources: - - wiki/summaries/YYYY/MM/{TARGET}-memory.md - - wiki/summaries/YYYY/MM/{TARGET}-opencode-usage.md - - wiki/summaries/YYYY/MM/{TARGET}-claude-code-usage.md - - wiki/summaries/YYYY/MM/{TARGET}-hermes-usage.md - - handoffs/YYYY/MM/wiki-daily-build/{TARGET}_*_handoff_*.md - # plus handoffs/YYYY/MM/wiki-promote/... if wiki-promote produced output that day -tags: [cron-wrapup, ops-daily] ---- -``` - -Required: `type, created, updated, sources, tags, date`. **No `review_status`** — summaries are exempt from approval. `subtype: cron-daily` is currently free-form (lint does not enforce subtype values) but use it consistently. - -## Body Contract (Slack-digest-stable — DO NOT rename sections) - -The seven H2 sections below are the parsing contract for the 09:00 Slack digest. Section names, order, and column shapes must remain stable. Authors may add prose under each section but must not rename or reorder them. - -```markdown -# Cron Wrap-up — {TARGET} - -## Status - -OK | DEGRADED | FAILED - -(One-word verdict on the first line, optional one-sentence elaboration on the second.) - -## Jobs - -| Job | Schedule | Exit | Output | Notes | -| --- | --- | --- | --- | --- | -| kb-opencode-daily-report | 10 3 * * * | 0 | wiki/summaries/2026/05/2026-05-20-opencode-usage.md | 1 session, 1.05 USD | -| kb-claude-code-daily-report| 20 3 * * * | 0 | wiki/summaries/2026/05/2026-05-20-claude-code-usage.md| 10/14 sessions, 69.77 USD | -| kb-hermes-daily-report | 15 3 * * * | 0 | wiki/summaries/2026/05/2026-05-20-hermes-usage.md | 3 sessions, zombie 0 | -| kb-memory-daily | 30 3 * * * | 0 | wiki/summaries/2026/05/2026-05-20-memory.md | 2 new pages | -| kb-wiki-ttl-sweep | 30 0 * * * | 0 | (log only) | 0 pages expired | -| kb-wiki-promote | 0 4 * * * | 0 | handoffs/2026/05/wiki-promote/... | 1 promoted | - -Exactly 5 columns. Missing jobs (e.g. weekly on non-Monday) → omit the row, do not invent. - -## Insights - -- none - -(Extract 1-5 user-facing signals from the daily memory page, usage reports, and run handoffs. Prefer durable or decision-relevant signals: new improvement pages, recurring error trends, cost spikes, completed work that needs review, newly blocked work, or promotion candidates. Use the literal line `- none` only when the KB produced no meaningful new signal.) - -## Action Items - -| Priority | Owner | Item | Source | -| --- | --- | --- | --- | -| high | user | Review frontend-review-console PR #20 CI and merge readiness | wiki/summaries/2026/05/2026-05-20-memory.md | - -(Extract concrete follow-ups from daily memory `Open Items`, `Next Run Notes`, run handoff `Next handoff instructions`, and unresolved improvement pages. Use owner `user`, `agent`, or `unknown`. If empty, use one row with `none | none | none | none`.) - -## Anomalies - -- none - -(Or bullet list of issues: non-zero exits, missing inputs, lint failures, lock skips, late starts > 10 min. Use the literal line `- none` when empty — never delete the section.) - -## Counters - -| Metric | Value | -| --- | --- | -| pages_created | 2 | -| pages_promoted | 1 | -| pages_ttl_swept | 0 | -| insights_count | 5 | -| action_items_count | 3 | -| lint_errors | 0 | -| lint_warnings | 12 | -| total_usd | 70.82 | -| sessions_total | 14 | - -Keep snake_case keys so the digest can match on them. `insights_count` and `action_items_count` make an empty-looking status easy to detect even when jobs all succeeded. - -## Links - -- memory: `wiki/summaries/2026/05/2026-05-20-memory.md` -- opencode: `wiki/summaries/2026/05/2026-05-20-opencode-usage.md` -- claude: `wiki/summaries/2026/05/2026-05-20-claude-code-usage.md` -- hermes: `wiki/summaries/2026/05/2026-05-20-hermes-usage.md` -- daily-handoff: `handoffs/2026/05/wiki-daily-build/2026-05-20_opencode_handoff_04.md` -- promote-handoff: `handoffs/2026/05/wiki-promote/...` (omit line if absent) -``` - -## Status Verdict — deterministic rules - -Compute in this order; the first match wins: - -| Verdict | Condition | -|---|---| -| `FAILED` | Any cron wrapper exited non-zero **OR** any *required* input is missing (memory page, all three usage reports). | -| `DEGRADED` | Any non-blocking anomaly: lint warnings appearing today that weren't there yesterday, lock skip, optional input missing (promote handoff on a day promote was supposed to run), or a per-run log file file-mtime far past its scheduled cron time. | -| `OK` | All expected logs show clean exit; Anomalies list is `- none`. | - -Late-start detection: per-run logs do not contain wrapper-side start timestamps, so use the log file's mtime (or first content line's timestamp if the underlying tool emits one) versus the scheduled cron time. Treat slips > 10 min as DEGRADED only when the signal is clearly available; otherwise omit the anomaly rather than guess. - -## Inputs the wrap-up reads — and what to extract from each - -| Input (read via `psql`; logs are files) | Extract | -|---|---| -| `pages` slug `{T}-opencode-usage` (numbers also in `metrics` report_type=opencode) | sessions, total USD | -| `pages` slug `{T}-claude-code-usage` (numbers also in `metrics`) | sessions, USD, tool_error_rate, hot files count | -| `pages` slug `{T}-hermes-usage` | sessions, zombie count | -| `pages` slug `{T}-memory` | key events, promotion candidates, open items, next-run notes; **do not copy the whole narrative** | -| `handoffs` task_slug=`wiki-daily-build`, status=`ready` | new pages count from §6 Outputs, risks, next handoff instructions | -| `handoffs` task_slug=`wiki-promote`, status=`ready` | promoted count, rejected count, expired count | -| `cron_runs` rows for target `T` | per-job status, exit code, timestamps, `log_body` diagnostics such as `ERROR:`, lock skips, and TTL sweep counts | - -Read each input at most once. Never use lint as a discovery source, never re-render existing summaries. The wrap-up MUST NOT read `data/raw/` or other `data/` exports for discovery; use `cron_runs.log_body` instead. Archived log files are debug copies only if DB submission itself is broken. The wrap-up may extract concise insights/actions from the daily memory and handoffs, but must not duplicate the memory page's full content narrative. - -## DB Write Order - -1. Write the wrap-up summary through the kb-mcp `upsert_page` tool (pass `type: summary`, structured `frontmatter`, `body_md` without the `---` fence, `slug`, and `export_path`). -2. Write the handoff through the kb-mcp `create_handoff` tool (structured `frontmatter` + `body_md`). -3. Write the operation note through the kb-mcp `create_operation_log` tool. -4. Confirm every tool result reports `export.status == success`. If a result carries an `error`/`code` (`missing_args`, `not_found`, `conflict`, `lint_failed`, `export_failed`), fix the cause and retry that write. - -## Common Lint Failures (preemptive fixes) - -| ERROR pattern | Cause | Fix before lint | -|---|---|---| -| `source file not found: wiki/summaries/...` | Cited a path that wasn't actually produced (e.g. promote handoff on day promote didn't run) | Drop the source line or change the path | -| `source file not found: data/wiki/...` | `sources:` paths must be `data/`-relative, not `data/data/` | Drop the `data/` prefix | - -## Handoff (defer to `handoff-document`) - -Every run writes one handoff. Use the `handoff-document` skill for filename grammar, frontmatter, body sections, and lint. - -This skill specifies only: - -- **Task folder**: `data/handoffs/{YYYY}/{MM}/cron-wrapup/` (one folder for the task; per-role NN keeps incrementing across dates, same shape as `wiki-daily-build/`). -- **Subject**: `{TARGET}` (the date being wrapped up). -- **Role**: whoever the cron wrapper invokes (currently `opencode`). -- **Content focus**: link to the produced wiki summary, list anomalies, set `status: ready` and a clear `## 9. Next handoff instructions` block for the next day or for the 09:00 digest. - -## Log Format (write through the kb-mcp `create_operation_log` tool) - -```markdown - -## YYYY-MM-DD (cron wrap-up — target: {TARGET}) - -- **fill**: {TARGET} cron wrap-up summary - - 소스: wiki/summaries/Y/M/{TARGET}-{opencode,claude-code,hermes}-usage.md, wiki/summaries/Y/M/{TARGET}-memory.md, raw/ops/cron/Y/M/{TARGET}_*.log - - 출력: wiki/summaries/Y/M/{TARGET}-cron-wrapup.md (신규) -- **handoff**: handoffs/Y/M/cron-wrapup/{TARGET}_{role}_handoff_NN.md -- **db_write**: summary + handoff + operation log exported successfully -``` - -## Workflow (10 steps) - -``` -1. Resolve TARGET = yesterday in KST (or accept explicit argument) -2. Query inputs via `psql`: usage + memory pages (`pages`), ready handoffs (`handoffs`), usage numbers (`metrics`), job outcomes (`cron_runs`); - then read per-run cron logs at `raw/ops/cron/{Y}/{M}/{TARGET}_*.log` (files). Note any MISSING required input → Status: FAILED later. -3. Extract per-input metrics (see Inputs table). Compute Counters. -4. Read every `raw/ops/cron/{Y}/{M}/{TARGET}_*.log` except `{TARGET}_kb-cron-wrapup.log` — that file does not exist during the session (the shell wrapper writes and commits it after the session exits, so it is never available to read here) - for non-zero exits, ERRORs, lock skips. Collect into Anomalies. -5. Compute Status verdict (FAILED > DEGRADED > OK, first match wins). -6. Fill `reference/templates/cron-wrapup-summary.md` and write it through the kb-mcp `upsert_page` tool (`type: summary`, structured `frontmatter` + `body_md`) with export path `wiki/summaries/{Y}/{M}/{TARGET}-cron-wrapup.md`. -7. Write run handoff via the kb-mcp `create_handoff` tool (subject = TARGET, role = invoker, status: ready). -8. Append the operation note via the kb-mcp `create_operation_log` tool. -9. Confirm every tool result reports `export.status == success` (retry any write that returns an `error`/`code`). -10. STOP. Do NOT commit or push `data/`; it is generated export. -``` - -## Data Persistence Policy - -DB rows are the durable checkpoint. Markdown, handoff files, `log.md`, and cron -logs under `data/` are generated export. Never stage, commit, or push from this workflow. - -## Idempotency - -- If `{TARGET}-cron-wrapup.md` already exists, **overwrite it** (the cron may rerun after a failure). Update `updated:` to today. -- For the handoff, use the next per-role `NN` in `handoffs/{Y}/{M}/cron-wrapup/`. Do not overwrite a prior handoff. - -## Failure path - -If a required input is missing (e.g. memory page never wrote): - -1. Still write the wrap-up. Set `Status: FAILED`. -2. List the missing input as a bullet under `## Anomalies`. -3. In the run handoff, set `status: ready` and put the failure cause in `§8. Risks / uncertainties` with a clear escalation note for §9 so the 09:00 digest surfaces it to Slack. -4. Exit non-zero from the wrapper so the cron log records the failure. - -## Workflow Discipline - -- **One target per run.** Never extend scope implicitly to "and also the day before". -- **Never edit raw files.** The wrap-up reads `data/raw/ops/cron/{Y}/{M}/{TARGET}_*.log` as read-only evidence, but must never modify their contents. Do not read other `data/raw/` subtrees. -- **Replay caveat.** A wrapper rerunning for an already-written TARGET will append to a persisted log entry. If you must replay a job for a previously-run TARGET, use the kb-mcp tools to manage the existing log entry before the wrapper runs. -- **Do not auto-promote** anything — wiki promotion is `wiki-approval`'s job. -- **Do not use lint as analysis input** — run only the required final gate: `kb-lint all` once. -- **Persist only through the kb-mcp tools.** Never commit outer repo files or generated `data/` export. -- **If blocked**, write the handoff with `status: ready` and the wrap-up with `Status: FAILED`, then exit non-zero. - -## Red Flags — STOP and re-check - -- About to read any cron log other than `raw/ops/cron/{Y}/{M}/{TARGET}_*.log` → STOP. The wrap-up summarizes exactly this target's per-run files; no time-window scan. (`{TARGET}_kb-cron-wrapup.log` does not exist during the session — the shell wrapper creates and commits it after the session exits; never read or stage it.) -- About to inspect `git status` or prepare a data commit → STOP. DB rows are the source of truth. -- About to re-render the full memory page into the wrap-up → STOP. Extract only concise insights and action items. -- About to rename or reorder one of the seven H2 sections → STOP. The Slack digest will break. -- About to create a wiki page by editing `data/wiki` directly → STOP. Write through the kb-mcp `upsert_page` tool. -- About to put `data/wiki/...` in `sources:` → drop the `data/` prefix -- About to mark Status: OK while Anomalies list is non-empty → re-read the Status table; non-empty anomaly = DEGRADED at minimum -- About to overwrite a prior cron-wrapup handoff with the same NN → bump NN, never overwrite handoffs -- About to invoke this skill twice in one cron firing → STOP. One run, one wrap-up, one handoff. -- About to run any `git commit` for generated KB data → STOP. Use the kb-mcp tools. diff --git a/.claude/skills/cron-wrapup/reference/templates/cron-wrapup-summary.md b/.claude/skills/cron-wrapup/reference/templates/cron-wrapup-summary.md deleted file mode 100644 index 59550a9..0000000 --- a/.claude/skills/cron-wrapup/reference/templates/cron-wrapup-summary.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -type: summary -subtype: cron-daily -date: "YYYY-MM-DD" -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: - - wiki/summaries/YYYY/MM/YYYY-MM-DD-memory.md - - wiki/summaries/YYYY/MM/YYYY-MM-DD-opencode-usage.md - - wiki/summaries/YYYY/MM/YYYY-MM-DD-claude-code-usage.md - - wiki/summaries/YYYY/MM/YYYY-MM-DD-hermes-usage.md - - handoffs/YYYY/MM/wiki-daily-build/YYYY-MM-DD_role_handoff_NN.md -tags: [cron-wrapup, ops-daily] ---- - -# Cron Wrap-up — YYYY-MM-DD - -## Status - -OK - -(Replace with OK | DEGRADED | FAILED on the first line. Optional one-sentence elaboration on the second line.) - -## Jobs - -| Job | Schedule | Exit | Output | Notes | -| --- | --- | --- | --- | --- | -| kb-wiki-ttl-sweep | 30 0 * * * | 0 | (log only) | 0 pages expired | -| kb-opencode-daily-report | 10 3 * * * | 0 | wiki/summaries/YYYY/MM/YYYY-MM-DD-opencode-usage.md | N sessions, X.XX USD | -| kb-hermes-daily-report | 15 3 * * * | 0 | wiki/summaries/YYYY/MM/YYYY-MM-DD-hermes-usage.md | N sessions, zombie N | -| kb-claude-code-daily-report | 20 3 * * * | 0 | wiki/summaries/YYYY/MM/YYYY-MM-DD-claude-code-usage.md| N sessions, X.XX USD | -| kb-memory-daily | 30 3 * * * | 0 | wiki/summaries/YYYY/MM/YYYY-MM-DD-memory.md | N new pages | -| kb-wiki-promote | 0 4 * * * | 0 | handoffs/YYYY/MM/wiki-promote/... | N promoted | - -(Omit a row if a job did not run that day, e.g. weekly/monthly on non-eligible days. Do not invent rows.) - -## Insights - -- none - -(Replace with 1-5 user-facing signals from the daily memory page, usage reports, and run handoffs: new improvement pages, recurring error trends, cost spikes, completed work that needs review, newly blocked work, or promotion candidates. Use `- none` only when no meaningful new signal exists.) - -## Action Items - -| Priority | Owner | Item | Source | -| --- | --- | --- | --- | -| none | none | none | none | - -(Replace with concrete follow-ups from daily memory Open Items / Next Run Notes, run handoff Next handoff instructions, and unresolved improvement pages. Owner should be `user`, `agent`, or `unknown`.) - -## Anomalies - -- none - -(Replace with bullet list of anomalies — non-zero exits, missing inputs, lint failures, lock skips, late starts > 10 min. Use the literal line `- none` when empty.) - -## Counters - -| Metric | Value | -| --- | --- | -| pages_created | 0 | -| pages_promoted | 0 | -| pages_ttl_swept | 0 | -| insights_count | 0 | -| action_items_count | 0 | -| lint_errors | 0 | -| lint_warnings | 0 | -| total_usd | 0.00 | -| sessions_total | 0 | - -(Add new snake_case keys only if a follow-up Slack digest needs them.) - -## Links - -- memory: wiki/summaries/YYYY/MM/YYYY-MM-DD-memory.md -- opencode: wiki/summaries/YYYY/MM/YYYY-MM-DD-opencode-usage.md -- claude: wiki/summaries/YYYY/MM/YYYY-MM-DD-claude-code-usage.md -- hermes: wiki/summaries/YYYY/MM/YYYY-MM-DD-hermes-usage.md -- daily-handoff: handoffs/YYYY/MM/wiki-daily-build/YYYY-MM-DD_role_handoff_NN.md -- promote-handoff: handoffs/YYYY/MM/wiki-promote/... - -(Omit a line if the artefact was not produced that day.) diff --git a/.claude/skills/knowledgebase-initialize/reference/optional-global-digest.md b/.claude/skills/knowledgebase-initialize/reference/optional-global-digest.md deleted file mode 100644 index b07e4bf..0000000 --- a/.claude/skills/knowledgebase-initialize/reference/optional-global-digest.md +++ /dev/null @@ -1,40 +0,0 @@ -# Optional Global Digest - -The morning global digest is optional but recommended once `kb-cron-wrapup` is producing stable, DB-backed daily summaries. - -## Purpose - -Create one morning notification that reports the previous night's KnowledgeBase pipeline state without re-reading KnowledgeBase internals. This job is read-only: it reports from the `kb-cron-wrapup` artefact and does not create, edit, lint, or commit KB data. - -The digest should consume only: - -- `data/wiki/summaries/YYYY/MM/YYYY-MM-DD-cron-wrapup.md` -- The fixed H2 sections: `Status`, `Insights`, `Action Items`, `Anomalies`, `Counters`, `Links` - -It should not read: - -- `data/wiki/summaries/.../*-memory.md` -- `data/raw/` (including `data/raw/ops/cron/` — that is the wrap-up's input, not the digest's) -- individual usage report pages - -## Recommended Schedule - -```text -morning-slack-digest: 09:00 every day -``` - -Run it after the `05:00` `kb-cron-wrapup` job so the digest has a single stable, exported artifact to parse. - -## Setup Guidance - -Use whatever scheduler or agent runtime the user already uses for notifications. Keep the job global rather than KnowledgeBase-owned, because it is a delivery concern, not a KB data-generation or data-commit step. - -Recommended prompt shape: - -```text -Read the latest KnowledgeBase cron wrap-up summary for yesterday. Summarize only the fixed H2 contract sections: Status, Insights, Action Items, Anomalies, Counters, and Links. Do not inspect memory pages, raw data, usage pages, or cron logs directly. Do not create, edit, lint, commit, or push KB files. Send a concise morning status digest. -``` - -## Dependency - -Set this up only after at least one `*-cron-wrapup.md` file exists and follows the fixed H2 contract. diff --git a/.claude/skills/memory-report/SKILL.md b/.claude/skills/memory-report/SKILL.md deleted file mode 100644 index 19379eb..0000000 --- a/.claude/skills/memory-report/SKILL.md +++ /dev/null @@ -1,401 +0,0 @@ ---- -name: memory-report -description: Use when running the daily, weekly, or monthly memory workflow — discovering period sources, writing summary + worthy wiki pages through the kb-mcp tools, writing the period handoff, and lint-clean uncommitted output. Covers raw discovery, period dispatch (daily/weekly/monthly), wiki page frontmatter, lint ordering, and canonical lint failures. ---- - -# memory-report - -## DB-Canonical Override - -DB rows are the source of truth. Write summaries, pages, handoffs, and operation -logs through the kb-mcp tools (`upsert_page`, `patch_page`, `create_handoff`, -`create_operation_log`, `create_raw_source`) — these run inside the `opencode run` -cron session where the kb-mcp MCP server is registered. -**Reads** (source discovery in each Step 0) go directly to Postgres via `psql` -(or the read-only kb-mcp `query_sql` tool) — -schema + recipes in `docs/db_informations/state-db-schema-reference.md`. -Markdown under `data/` is generated export. If any older instruction below says -to write files, lint as a write gate, commit `data/`, or discover sources by -scanning `data/`, ignore that instruction and use the DB instead. - -## Overview - -One skill, three periods. The agent receives a target (`YYYY-MM-DD`, `YYYY-WNN`, or `YYYY-MM`) and produces a period summary + zero-or-more wiki pages + one handoff, all lint-clean and uncommitted. - -The shared half (Repo layout, wiki schemas, lint order, common failures) is identical across periods. The per-period half (source discovery, promotion intensity, output filename) differs and lives in three small sections below. - -This SKILL replaces the doc-load pattern where each cron agent re-read 6 markdown files on every run. - -**Bundled reference (self-contained — do not consult docs/ at runtime):** -- `reference/templates/{entity,concept,decision,question,improvement,checklist}.md` — period workflow copies remain here for backward compatibility -- `reference/templates/summaries/{weekly,monthly}.md` — weekly/monthly summary skeletons (daily summary schema is inlined below) - -For new atomic wiki pages, import `wiki-authoring` as the canonical page-authoring contract. Handoff authoring is delegated to `handoff-document` — invoke that skill for the period handoff. This skill does not duplicate handoff schema. - -If anything below contradicts the lint code (`src/kb/lint/wiki.py`), the lint code wins — file an issue and update this skill. - -## Period Dispatch — find your period BEFORE reading sections - -| Period | Cron schedule (KST) | Target arg format | Lock file | Skill section | -|---|---|---|---|---| -| daily | `30 3 * * *` | `YYYY-MM-DD` | `daily.lock` | [§Daily](#daily) | -| weekly | `15 4 * * 1` | `YYYY-WNN` (ISO week) | `weekly.lock` | [§Weekly](#weekly) | -| monthly | `45 4 1 * *` | `YYYY-MM` | `monthly.lock` | [§Monthly](#monthly) | - -If the cron prompt says "daily memory workflow for 2026-05-19", you're in §Daily. Read SHARED CONVENTIONS once, then jump to the matching period section. - -## When to Use - -- Any of the three crons fires -- Manual invocation: "Run the {daily|weekly|monthly} memory workflow for {target}" - -**Do NOT use** for: wiki promotion (separate workflow), raw ingestion (manual / external scripts), or recovery from a previously failed run without reading the prior handoff first. - ---- - -# Shared Conventions - -## Repo Layout (memorize, don't re-discover) - -``` -data/ -├── raw/ -│ ├── github/ # GitHub CLAUDE.md + issues/PRs (flat — date in filename) -│ ├── gmail/{YYYY}/{MM}/ # date-partitioned -│ ├── sessions/{YYYY}/{MM}/ # date-partitioned -│ ├── web/ # flat -│ └── ops/ -│ ├── cron/{YYYY}/{MM}/ # cron-generated usage reports -│ ├── runs/{YYYY}/{MM}/ -│ ├── decisions/{YYYY}/{MM}/ -│ ├── traces/{YYYY}/{MM}/ -│ └── ingest_state/ # state files, not source -├── handoffs/{YYYY}/{MM}// -└── wiki/ - ├── entities/{subject}/{YYYY-MM}/ - ├── improvements/{YYYY-MM}/ - ├── concepts/ # FLAT (timeless ideas) - ├── decisions/ # flat, filename prefixes date: 2026-05-18-foo.md - ├── questions/ # flat - ├── checklists/ # flat - └── summaries/{YYYY}/{MM}/ # YYYY-MM-DD-memory.md, YYYY-WNN-weekly.md, YYYY-MM-monthly.md -``` - -`sources:` frontmatter paths are **relative to `data/`** (e.g. `wiki/summaries/...`, NOT `data/wiki/summaries/...`). - -## Wiki Page Schemas (the only thing lint ERRORs you for) - -All review-tracked types (`entity`, `concept`, `decision`, `improvement`, `checklist`, `question`) inherit `review_status: not_processed` from template. Don't manually set it to anything else. - -### summary - -```yaml ---- -type: summary -subtype: daily | weekly | monthly -date: "YYYY-MM-DD" # daily only -week: "YYYY-WNN" # weekly only -month: "YYYY-MM" # monthly only -period_start: "YYYY-MM-DD" # weekly/monthly -period_end: "YYYY-MM-DD" # weekly/monthly -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: - - wiki/summaries/YYYY/MM/... - - handoffs/YYYY/MM//... -tags: [] ---- -``` -Required: `type, created, updated, sources, tags` + the period-specific date fields. **No `review_status`** — summaries are exempt from approval. - -### entity / concept / question - -```yaml ---- -type: entity # or concept, question -review_status: not_processed -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: [...] -aliases: [] # entity & concept only -tags: [] ---- -``` - -### improvement — **the lint trap** - -```yaml ---- -type: improvement -review_status: not_processed -kind: improvement # one of: improvement | issue | proposal -observed_at: "YYYY-MM-DD" # ISO date — required, lint validates format -domain: dx # one of: cost | correctness | perf | dx | security -severity: med # one of: low | med | high -issue_status: open # one of: open | acknowledged | resolved | wontfix -related: [] # list of wikilinks, lint resolves them -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: [...] -tags: [] ---- -``` - -Missing any of `kind/observed_at/domain/severity/issue_status/related` → ERROR. Invalid enum → ERROR. - -### decision - -```yaml ---- -type: decision -review_status: not_processed -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: [...] -tags: [decision] ---- -``` -Filename convention: `YYYY-MM-DD-.md` (date prefix in filename, not folder). - -### checklist - -```yaml ---- -type: checklist -review_status: not_processed -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -sources: [...] -tags: [] ---- -``` -Body **must** contain `## Items` section with `- [ ] ...` task-list syntax. Lint enforces this. - -## DB Write Order - -The kb-mcp write tools take **structured args** — pass the `frontmatter` as an -object (not YAML text) and `body_md` as the body without the `---` fence, plus -`slug`, `type`, and `export_path`. The same applies to `create_handoff`. - -1. Submit the summary page through the kb-mcp `upsert_page` tool. -2. Submit any new or updated evidence-derived wiki pages through the kb-mcp - `upsert_page` tool (use the kb-mcp `patch_page` tool for partial updates). -3. Submit the run handoff through the kb-mcp `create_handoff` tool. -4. Submit the operation note through the kb-mcp `create_operation_log` tool. -5. Confirm each tool result reports `export.status == success`. On an - `error`/`code` result, handle it (`lint_failed` → fix and retry; - `conflict` → already exists). - -## Common Lint Failures (preemptive fixes) - -| ERROR pattern | Cause | Fix before lint | -|---|---|---| -| `source file not found: wiki/summaries/daily/...` | Cited a legacy path. Daily summaries live at `wiki/summaries/{YYYY}/{MM}/`, not `wiki/summaries/daily/` | Use the partitioned path | -| `source file not found: data/raw/...` | `sources:` paths must be relative to `data/`, not `data/data/` | Drop the `data/` prefix | -| `missing frontmatter field: kind` (improvement) | Used base template, didn't fill improvement-specific fields | All 6 extra fields required | -| `invalid domain: 'xxx'` | Domain not in enum | Use one of: cost, correctness, perf, dx, security | -| `raw file modified after creation (immutability violation)` | Someone (not you) edited a raw file | **Do not touch raw files.** Log in handoff §8; cannot be fixed by this run | -| `stub page — body N chars (< 100)` | WARN only, ignore | — | -| `orphan page — no inbound links` | WARN only, expected for new pages | — | - -## Handoff (defer to `handoff-document`) - -Every run writes one handoff. Use the `handoff-document` skill for filename grammar, frontmatter, body sections, and lint. This skill only specifies the **task folder name** and **content focus** per period (see each period section). - -## Log Format (submit through the kb-mcp `create_operation_log` tool, all periods) - -```markdown - -## YYYY-MM-DD ({period} memory build — target: {TARGET}) - -- **fill**: {TARGET} {period} synthesis - - 소스: - - 출력: wiki/summaries/YYYY/MM/.md (신규) -- **fill**: -- **handoff**: -- **db_write**: pages + handoff + operation log exported successfully -- Note: commit은 사용자가 수동으로 review 후 진행 -``` - -## Workflow Discipline (all periods) - -- **One target per run.** Never extend scope implicitly to "and also yesterday/last week". -- **Never edit raw files.** Raw sources are DB rows; add new raw evidence through the kb-mcp `create_raw_source` tool. -- **Do not auto-promote** wiki pages to `pending_for_approve` — that's the wiki-promote cron's job. -- **Do not git commit.** Markdown is generated export from DB rows. -- **If blocked**, write the handoff with `status: ready`, list the blocker in §8, and exit non-zero. - ---- - -# §Daily - -**Target**: `YYYY-MM-DD` (yesterday). -**Output summary**: `data/wiki/summaries/{YYYY}/{MM}/{TARGET}-memory.md` -**Handoff folder**: `data/handoffs/{YYYY}/{MM}/wiki-daily-build/` -**Promotion intensity**: triage > restructure. Prefer summary; create wiki pages only when source has clear future value. - -## Daily Step 0: Identify yesterday's raw — query the DB, never scan `data/` - -Postgres is the source of truth; `data/` is a human-readable export. Discover sources with `psql`, not by walking the export tree. - -```bash -TARGET="$1" # e.g. 2026-05-19 -psql_kb() { psql "${DATABASE_URL/+psycopg/}" -tAc "$1"; } - -# Raw ingested on TARGET. captured_at is stored in UTC, so range over the KST day. -FROM=$(date -u -d "$TARGET 00:00:00+09:00" +%FT%T) -TO=$(date -u -d "$TARGET 00:00:00+09:00 +1 day" +%FT%T) -psql_kb "SELECT source_key, source_type, captured_at FROM raw_sources - WHERE captured_at >= '$FROM' AND captured_at < '$TO' ORDER BY captured_at;" - -# Ready handoffs for context (prior daily build + anything else still open) -psql_kb "SELECT task_slug, handoff_id, status FROM handoffs - WHERE status='ready' ORDER BY updated_at DESC;" -``` - -Never `find`/`ls`/`grep` under `data/` to discover sources — the export can lag the DB. - -## Daily Workflow (9 steps) - -``` -1. Run Daily Step 0 → raw set + active handoffs for TARGET -2. Read prior wiki-daily-build handoff (status: ready) for context -3. Write data/wiki/summaries/{Y}/{M}/{TARGET}-memory.md -4. Create only obviously-worthy entity/improvement/concept pages - — leave uncertain items in handoff "Promotion Candidates" -5. Mark prior daily handoff status: consumed IF its open items were handled -6. Write new handoff via `handoff-document` under wiki-daily-build/ -7. Submit operation log through the kb-mcp `create_operation_log` tool -8. Confirm each kb-mcp tool result reports export.status == success; on an `error`/`code` result handle it (lint_failed → fix and retry; conflict → already exists) -9. STOP. Do NOT git commit. -``` - -## Daily promotion intensity - -Prefer summary + triage. Create atomic pages only when source has clear future value. Uncertain → handoff's §10 Promotion Candidates, not a new page. - ---- - -# §Weekly - -**Target**: `YYYY-WNN` (last ISO week, e.g. `2026-W20`). -**Output summary**: `data/wiki/summaries/{YYYY}/{MM}/{TARGET}-weekly.md` (where MM = month of week's end) -**Template**: `reference/templates/summaries/weekly.md` -**Handoff folder**: `data/handoffs/{YYYY}/{MM}/wiki-weekly-build/` -**Promotion intensity**: patterns over events. Promote signals **repeated across ≥2 days** or strongly supported by one important source. - -## Weekly Step 0: collect the week's daily summaries and handoffs - -```bash -TARGET="$1" # e.g. 2026-W20 -psql_kb() { psql "${DATABASE_URL/+psycopg/}" -tAc "$1"; } -# Resolve ISO week → Monday date and Sunday date -PERIOD_START=$(date -d "$(echo $TARGET | sed 's/-W/ /')-1" +%F 2>/dev/null \ - || date -d "${TARGET%-W*}-01-01 +$((10#${TARGET#*-W} * 7 - 7)) days" +%F) -PERIOD_END=$(date -d "$PERIOD_START +6 days" +%F) - -# Daily memory summaries falling in the week (slug starts with the date) -psql_kb "SELECT slug FROM pages WHERE type='summary' AND slug LIKE '%-memory' - AND left(slug,10) BETWEEN '$PERIOD_START' AND '$PERIOD_END' ORDER BY slug;" - -# Ready handoffs from the daily build for the period -psql_kb "SELECT handoff_id, status FROM handoffs - WHERE task_slug='wiki-daily-build' AND status='ready' ORDER BY updated_at DESC;" -``` - -Sanity check: expect 7 daily memory summary rows. Document any missing day in the handoff §8; do not synthesize phantom days. - -## Weekly Workflow (10 steps) - -``` -1. Run Weekly Step 0 → seven daily summaries + ready daily handoffs -2. Read prior wiki-weekly-build handoff (status: ready) if any -3. Write data/wiki/summaries/{Y}/{M}/{TARGET}-weekly.md using reference/templates/summaries/weekly.md -4. Promote repeated patterns into concepts, improvements, checklists, or decisions - (see Weekly promotion intensity below) -5. Mark consumed daily handoffs as status: consumed ONLY for items actually handled -6. Write new handoff via `handoff-document` under wiki-weekly-build/ -7. Note monthly candidates in handoff §10 (which patterns are monthly-worthy) -8. Submit operation log through the kb-mcp `create_operation_log` tool -9. Confirm each kb-mcp tool result reports export.status == success; on an `error`/`code` result handle it (lint_failed → fix and retry; conflict → already exists) -10. STOP. Do NOT git commit. -``` - -## Weekly promotion intensity - -- **Promote** when signal repeats across multiple days or has one strong-source support. -- Convert recurring operational mistakes → **checklist** candidates. -- Convert unresolved-but-actionable work → **improvements**. -- Convert closed architectural/workflow choices → **decisions** (only if rationale is captured). -- Don't promote single-day mentions unless source is high-value. - ---- - -# §Monthly - -**Target**: `YYYY-MM` (last month). -**Output summary**: `data/wiki/summaries/{YYYY}/{MM}/{TARGET}-monthly.md` -**Template**: `reference/templates/summaries/monthly.md` -**Handoff folder**: `data/handoffs/{YYYY}/{MM}/wiki-monthly-maintenance/` -**Promotion intensity**: cleanup > add. Consolidate duplicates, close stale, promote stable procedures. - -## Monthly Step 0: collect weekly summaries and prior monthly handoff - -```bash -TARGET="$1" # e.g. 2026-05 -YEAR="${TARGET%-*}" -MONTH="${TARGET#*-}" -psql_kb() { psql "${DATABASE_URL/+psycopg/}" -tAc "$1"; } - -# Weekly summaries exported under this month -psql_kb "SELECT slug FROM pages WHERE type='summary' AND slug LIKE '%-weekly' - AND export_path LIKE 'wiki/summaries/$YEAR/$MONTH/%' ORDER BY slug;" - -# Prior monthly maintenance handoffs (ready only) -psql_kb "SELECT handoff_id, status FROM handoffs - WHERE task_slug='wiki-monthly-maintenance' AND status='ready' ORDER BY updated_at DESC;" - -# Cleanup signal — orphan/stub pages flagged by lint -uv run kb-lint wiki -``` - -## Monthly Workflow (11 steps) - -``` -1. Run Monthly Step 0 → weekly summaries + ready monthly handoff + cleanup signals -2. Review duplicate concepts (same idea, different page), stale improvements - (issue_status: open but observed_at > 30 days ago), open decision candidates -3. Write data/wiki/summaries/{Y}/{M}/{TARGET}-monthly.md using reference/templates/summaries/monthly.md -4. Consolidate duplicated concepts ONLY when sources support the merge - (different evidence → keep both; same evidence → merge with combined sources) -5. Close or defer stale improvements: set issue_status to resolved/wontfix with rationale, - OR leave open and note the blocker -6. Promote stable repeated procedures into checklists -7. Mark consumed weekly handoffs as status: consumed for items handled -8. Write monthly handoff via `handoff-document` under wiki-monthly-maintenance/ - — flag reusable automation as promotion: skill_candidate -9. Submit operation log through the kb-mcp `create_operation_log` tool -10. Confirm each kb-mcp tool result reports export.status == success; on an `error`/`code` result handle it (lint_failed → fix and retry; conflict → already exists) -11. STOP. Do NOT git commit. -``` - -## Monthly promotion intensity - -- **Prefer cleanup** over new pages. Don't add fresh content the weekly should have caught. -- **Don't merge pages** if source evidence differs in meaning — separate facts deserve separate pages. -- **Mark reusable automation patterns** as `promotion: skill_candidate` in handoff §10. Don't write the skill in this run; flag it for the user. -- **Leave human-decision items open** rather than turning them into ADRs. Decisions are user-driven, not synthesized. - ---- - -# Red Flags — STOP and re-check - -- About to `Read data/raw/` for the 3rd time → use Step 0 find command for your period -- About to create a wiki page but haven't checked `sources:` path is `data/`-relative → check before submitting via kb-mcp -- About to write improvement page from `reference/templates/improvement.md` without filling the 6 extra fields → fill them or downgrade to entity/concept -- About to lint exported markdown files instead of using the kb-mcp tools → STOP. Submit through the kb-mcp `upsert_page`/`create_handoff`/`create_operation_log` tools. -- About to `git commit` → STOP. Markdown is generated export from DB rows. -- Lint ERRORs on raw immutability → not your run's fault, log it in handoff §8 and PASS the run -- About to mix periods in one run (e.g. "and also catch up last week's") → STOP. Run one period at a time -- Weekly run with <7 daily summaries → document missing days in handoff; do not synthesize phantom days -- Monthly run with no weekly summaries → STOP. Weekly must run first; write a handoff explaining and exit diff --git a/.claude/skills/memory-report/reference/templates/checklist.md b/.claude/skills/memory-report/reference/templates/checklist.md deleted file mode 100644 index 3c6e791..0000000 --- a/.claude/skills/memory-report/reference/templates/checklist.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -type: checklist -review_status: not_processed -created: "" -updated: "" -sources: [] -tags: [] ---- - -# {{ChecklistTitle}} - -## Purpose - -## Items - -- [ ] item 1 -- [ ] item 2 - -## Notes diff --git a/.claude/skills/memory-report/reference/templates/concept.md b/.claude/skills/memory-report/reference/templates/concept.md deleted file mode 100644 index 5823623..0000000 --- a/.claude/skills/memory-report/reference/templates/concept.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -type: concept -review_status: not_processed -created: "" -updated: "" -sources: [] -aliases: [] -tags: [] ---- - -# {{ConceptName}} - -## Definition - -## Context - -## Related - diff --git a/.claude/skills/memory-report/reference/templates/decision.md b/.claude/skills/memory-report/reference/templates/decision.md deleted file mode 100644 index 59aa025..0000000 --- a/.claude/skills/memory-report/reference/templates/decision.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -type: decision -review_status: not_processed -created: "" -updated: "" -sources: [] -tags: [decision] ---- - -# {{DecisionTitle}} - -## Context - -## Decision - -## Consequences - diff --git a/.claude/skills/memory-report/reference/templates/entity.md b/.claude/skills/memory-report/reference/templates/entity.md deleted file mode 100644 index 1a07b84..0000000 --- a/.claude/skills/memory-report/reference/templates/entity.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -type: entity -review_status: not_processed -created: "" -updated: "" -sources: [] -aliases: [] -tags: [] ---- - -# {{EntityName}} - -## Overview - -## Key Details - -## Related - diff --git a/.claude/skills/memory-report/reference/templates/improvement.md b/.claude/skills/memory-report/reference/templates/improvement.md deleted file mode 100644 index 24865d9..0000000 --- a/.claude/skills/memory-report/reference/templates/improvement.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -type: improvement -review_status: not_processed -kind: improvement -observed_at: "" -domain: "" -severity: "" -issue_status: open -related: [] -created: "" -updated: "" -sources: [] -tags: [] ---- - -# {{ImprovementTitle}} - -## Observation - -## Impact - -## Proposed Action - -## Notes diff --git a/.claude/skills/memory-report/reference/templates/question.md b/.claude/skills/memory-report/reference/templates/question.md deleted file mode 100644 index 9c30f3e..0000000 --- a/.claude/skills/memory-report/reference/templates/question.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -type: question -review_status: not_processed -created: "" -updated: "" -sources: [] -tags: [] ---- - -# {{QuestionTitle}} - -## Question - -## Answer - -## Context - -## Related diff --git a/.claude/skills/memory-report/reference/templates/summaries/monthly.md b/.claude/skills/memory-report/reference/templates/summaries/monthly.md deleted file mode 100644 index 707f806..0000000 --- a/.claude/skills/memory-report/reference/templates/summaries/monthly.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -type: summary -subtype: monthly -month: "" -period_start: "" -period_end: "" -created: "" -updated: "" -sources: [] -tags: [] ---- - -# Monthly Report — {{YYYY-MM}} - -## Summary - -- Total sessions: N / active days: N (empty days: N) -- Total tokens: N / average daily tokens: N -- Average cache hit rate: N% -- Average TODO completion rate: N% - -## Weekly Trends - -| Week | Sessions | Tokens | Cache Hit Rate | TODO Completion | Notes | -|------|------|------|------------|------------|------| -| W01 | | | | | | -| W02 | | | | | | -| W03 | | | | | | -| W04 | | | | | | - -## Monthly Insights - -### What Went Well - - - -### Problem Patterns - - - -### Action Items - - - - -## Observations - - - diff --git a/.claude/skills/memory-report/reference/templates/summaries/weekly.md b/.claude/skills/memory-report/reference/templates/summaries/weekly.md deleted file mode 100644 index c9f3235..0000000 --- a/.claude/skills/memory-report/reference/templates/summaries/weekly.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -type: summary -subtype: weekly -week: "" -period_start: "" -period_end: "" -created: "" -updated: "" -sources: [] -tags: [] ---- - -# Weekly Summary - {{YYYY-WNN}} - -## Summary - -- Period: {{YYYY-MM-DD}} to {{YYYY-MM-DD}} -- Source reports reviewed: N -- Active systems: OpenCode / Hermes / Claude Code / other -- Main pattern: - -## Daily Coverage - -| Date | Memory | OpenCode | Hermes | Claude Code | Notes | -|---|---|---|---|---|---| -| YYYY-MM-DD | present/missing | present/missing/N/A | present/missing/N/A | present/missing/N/A | | - -## Signals - -### Work Patterns - - - -### Quality / Reliability - - - -### Cost / Efficiency - - - -## Promotions - -### Concepts - - - -### Improvements - - - -### Checklists - - - -### Decisions - - - -## Open Items - - - -## Next Week - - diff --git a/.claude/skills/usage-report-setup/SKILL.md b/.claude/skills/usage-report-setup/SKILL.md deleted file mode 100644 index 6bed3fc..0000000 --- a/.claude/skills/usage-report-setup/SKILL.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -name: usage-report-setup -description: Use when choosing OpenCode, Hermes, or Claude Code usage report modes — creating or auditing source-specific daily report cron jobs, configuring report output paths, handling missing telemetry sources, or wiring usage reports before daily memory builds. ---- - -# Usage Report Setup - -Use this skill as the runtime contract for source-specific usage report setup. Do not look for a workflow doc during execution; this skill is the complete operating surface. - -## Policy - -Generate separate reports by source. Do not create a combined usage report in this layer; daily memory synthesis may read multiple source reports and write one combined memory summary later. -Reports are written in-process through the `kb.service` layer (lint → DB → Markdown -export). Markdown under `data/wiki/` is generated export, not the source of truth. -Set `DATABASE_URL` (and optionally `KB_DATA_DIR`) before running any non-dry-run -report command. There is no token to set. - -| Source | Command | DB Output | -|---|---|---| -| OpenCode | `uv run kb-opencode-daily-report --date YYYY-MM-DD --lint` | CLI writes metrics + summary page in-process via the service layer | -| Hermes | `uv run kb-hermes-daily-report --date YYYY-MM-DD --lint` | CLI writes metrics + summary page in-process via the service layer | -| Claude Code | `uv run kb-claude-code-daily-report --date YYYY-MM-DD --lint` | CLI writes metrics + summary page in-process via the service layer | - -Metrics are canonical in the DB, written by the CLI through the service layer; a derived `.metrics.json` is exported under `data/ops/reports/`, not authored by hand. - -## Setup Workflow - -1. Inspect which report commands exist: - ```bash - uv run kb-opencode-daily-report --help - uv run kb-hermes-daily-report --help - uv run kb-claude-code-daily-report --help - ``` -2. Ask the user which modes to enable if not already specified: - - none - - OpenCode only - - Hermes only - - Claude Code only - - multiple separate reports -3. Verify matching wrapper scripts under `scripts/cron/`. -4. Ensure usage report jobs run before daily memory build. -5. Propose crontab entries. Do not install them unless the user explicitly approves installation. -6. Run a manual dry run only for selected sources. -7. Verify service-layer write success, generated markdown export, and metrics JSON output. - -## Cron Schedule - -Recommended KST ordering: - -```cron -10 3 * * * /scripts/cron/kb-opencode-daily-report.sh -15 3 * * * /scripts/cron/kb-hermes-daily-report.sh -20 3 * * * /scripts/cron/kb-claude-code-daily-report.sh -30 3 * * * /scripts/cron/kb-memory-daily.sh -``` - -Enable only selected sources. - -## Wrapper Contract - -Each wrapper must: - -1. Resolve `KB_ROOT` from the script location. -2. Create `.cron/locks`. -3. Use a source-specific `flock` lock. -4. Compute target date in KST as yesterday. -5. Compute `LOG_FILE="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)/${TARGET_DATE}_kb--daily-report.log"` and `mkdir -p` its parent. -6. Run the source-specific report command with `--lint`, redirecting stdout/stderr to `$LOG_FILE`. -7. Submit the completed wrapper log through `uv run kb-submit-cron-run`. -8. Exit non-zero if the report command or DB log submission fails. - -Example command inside a wrapper: - -```bash -uv run kb-opencode-daily-report --date "$TARGET_DATE" --lint -``` - -## Missing Source Handling - -If a selected source is not installed or has no database/telemetry: - -- do not fail unrelated source report jobs -- do not generate a misleading empty success report unless the user requested empty-day reports -- write a clear wrapper log entry -- if an agent is involved, write a handoff or operation log through the service layer - -## Output Rules - -Daily usage report markdown must: - -- use `type: summary` -- use `subtype: daily` -- set `date`, `created`, and `updated` -- use `sources: []` for local DB/telemetry inputs -- avoid dollar signs; write `N USD` -- be accepted by the service-layer write (the same path the report CLI uses); Markdown is exported after the DB write - -## Validation - -For each enabled source: - -```bash -uv run kb--daily-report --date YYYY-MM-DD --lint -# DB is canonical and write-only (no read endpoints). The command prints -# `export.status: success` on a good write; confirm the derived exports landed: -ls data/wiki/summaries/YYYY/MM/*--usage.md -ls data/ops/reports/YYYY/MM/*-usage.metrics.json -``` - -Do not commit `data/`; it is generated export. - -## Red Flags - -- About to create a combined usage report command in this layer. -- About to treat a missing optional source as failure for all reports. -- About to run daily memory before selected usage reports. -- About to install crontab entries without explicit user approval. diff --git a/.claude/skills/wiki-approval/SKILL.md b/.claude/skills/wiki-approval/SKILL.md deleted file mode 100644 index 9921d88..0000000 --- a/.claude/skills/wiki-approval/SKILL.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -name: wiki-approval -description: Use when promoting DB-backed wiki pages from not_processed to pending_for_approve — approving or rejecting pages, running TTL sweep, updating approved pages, and writing wiki-promote handoffs/logs through kb-mcp tools. ---- - -# Wiki Approval - -## DB-Canonical Override - -Approval state lives in DB pages. Use the kb-mcp tools (`promote_page`, -`approve_page`, `reject_page`, `ttl_sweep_pages`, `create_handoff`, -`create_operation_log`) for promote/approve/reject/TTL sweep, handoffs, and -operation logs. Markdown under `data/` is generated export. If any older -instruction below says to lint as a write gate or commit `data/`, prefer the -kb-mcp tools. - -Use this skill as the runtime contract for the KnowledgeBase review lifecycle. Do not look for a workflow doc during execution; this skill is the complete operating surface. - -## Status Model - -Applies only to wiki page types `entity`, `concept`, `decision`, `improvement`, `checklist`, and `question`. - -| Status | Meaning | Next | -|---|---|---| -| `not_processed` | AI wrote or semantically changed the page | promote or TTL reject | -| `pending_for_approve` | Waiting for human review | approve or reject | -| `approved` | Official wiki content | semantic edit self-resets to `not_processed` | -| `rejected` | Preserved outside wiki under `data/rejected/` | terminal | - -`summary` and `index` pages are outside the approval lifecycle. - -## Commands - -**Reads** go directly to Postgres (set `DATABASE_URL`; schema + recipes in -`docs/db_informations/state-db-schema-reference.md`). The read-only kb-mcp -`query_sql` tool is an equivalent alternative: - -```bash -psql "${DATABASE_URL/+psycopg/}" -tAc \ - "SELECT slug, type, created_at FROM pages WHERE review_status='not_processed' ORDER BY updated_at DESC;" -``` - -**Writes** go through the kb-mcp tools: - -- call the kb-mcp `promote_page` tool with `slug=`, `feedback=""`, `source="agent"` -- call the kb-mcp `approve_page` tool with `slug=`, `feedback=""`, `source="user"` -- call the kb-mcp `reject_page` tool with `slug=`, `feedback=""`, `source="user"` -- call the kb-mcp `ttl_sweep_pages` tool with `days=7` - -State machine: `promote_page` requires the page be in `not_processed`, `approve_page` -requires `pending_for_approve`, and `reject_page` works from `pending_for_approve` -or `not_processed`. A tool result with `code: conflict` means the page was not in -the expected `review_status` for that transition. - -`` is the filename without `.md`. - -## Reserved Body Section - -`## User Feedback` is CLI-owned. Never use that exact heading as normal wiki content. Use `## Feedback`, `## Reviewer Notes`, or another heading for authored content. - -## Promotion Workflow - -Use this for the daily `wiki-promote` cron or a manual promotion run. - -1. List the not_processed queue (direct Postgres read): - ```bash - psql "${DATABASE_URL/+psycopg/}" -tAc \ - "SELECT slug, type, created_at FROM pages WHERE review_status='not_processed' ORDER BY updated_at DESC;" - ``` -2. Prioritize recent `not_processed` pages. -3. Promote only when all are true: - - source paths are clear, real, and verifiable - - future lookup value exists - - page is durable knowledge, not a raw event dump -4. Promote worthy pages: call the kb-mcp `promote_page` tool with `slug=`, - `feedback=""`, `source="agent"`. The page must be in `not_processed`; a - `code: conflict` result means it was not. -5. Leave borderline pages untouched. Do not reject manually from an agent promotion run. -6. Confirm each tool result reports `export.status == success`; on an `error`/`code` - result handle it (conflict → page not in the expected `review_status`; - not_found → wrong slug). -7. Write a promotion handoff via the kb-mcp `create_handoff` tool. -8. Append the operation note via the kb-mcp `create_operation_log` tool. -9. Do not commit `data/`; it is generated export. - -## Human Approval / Rejection - -For user-requested page review: - -1. List pending pages (direct Postgres read): - ```bash - psql "${DATABASE_URL/+psycopg/}" -tAc \ - "SELECT slug, type FROM pages WHERE review_status='pending_for_approve';" - ``` -2. Read the page and its `sources:` files. -3. Approve when the page is correct and useful: call the kb-mcp `approve_page` - tool with `slug=`, `feedback="..."`, `source="user"`. The page must be - in `pending_for_approve`; a `code: conflict` result means it was not. -4. Reject only when the user has decided it should not enter the wiki: call the - kb-mcp `reject_page` tool with `slug=`, `feedback="..."`, `source="user"`. - This works from `pending_for_approve` or `not_processed`. -5. Confirm the tool result reports `export.status == success`; on an `error`/`code` - result handle it (conflict → page not in the expected `review_status`; - not_found → wrong slug). - -Rejected pages are marked in DB with `review_status: rejected`, `rejected_at`, -and `rejected_by`; export writes them under `data/rejected/...`. - -## TTL Sweep - -Cron-safe deterministic cleanup: call the kb-mcp `ttl_sweep_pages` tool with `days=7`. - -It rejects `not_processed` pages older than the threshold with `rejected_by: auto_ttl`. The wrapper may run this directly without an agent because no judgment is needed. - -## Editing Approved Pages - -When an agent edits an `approved` page: - -- typo, formatting, link-only cleanup: keep `review_status: approved` -- factual change, new information, changed conclusion, changed scope: set `review_status: not_processed` - -There is no deterministic drift detector. The editing agent must make this call. - -## Subject Hub Rule - -`INDEX.md` is generated. Subject `_index.md` hub files are not auto-synchronized. After approval, a user or companion agent may add a manual `- [[]]` line where useful. Missing hub entries are warnings, not blockers. - -## Handoff Quick Contract - -Promotion runs write under: - -```text -data/handoffs/YYYY/MM/wiki-promote/ -``` - -Use a filename that passes handoff validation (the kb-mcp `create_handoff` tool validates on submission), for example: - -```text -wiki-promote_opencode_handoff_01.md -``` - -Required frontmatter: - -```yaml ---- -handoff_id: "wiki-promote:wiki-promote:opencode:01" -task_slug: "wiki-promote" -subject: "wiki-promote" -role: opencode -handoff_seq: 1 -created: "YYYY-MM-DD" -updated: "YYYY-MM-DD" -status: ready -security: - contains_secrets: false - redaction_status: unchecked -promotion: null ---- -``` - -Include these body sections: - -```markdown -## 1. Assignment -## 2. Context received -## 3. Work performed -## 4. Tool trace -## 5. Findings / decisions -## 6. Outputs -## 7. Verification -## 8. Risks / uncertainties -## 9. Next handoff instructions -## 10. Promotion candidates -``` - -`## 4. Tool trace` table must have 7 columns: - -```markdown -| # | Tool | Purpose | Input summary | Output summary | Status | Side effect | -| --- | --- | --- | --- | --- | --- | --- | -``` - -## Log Format - -Submit through the kb-mcp `create_operation_log` tool (export may update `data/log.md`, but the DB row is canonical): - -```markdown - -## YYYY-MM-DD (wiki promotion) - -- **promoted**: -- **left**: -- **handoff**: handoffs/YYYY/MM/wiki-promote/.md -- **db_write**: promoted pages + handoff + operation log exported successfully -``` - -## Red Flags - -- About to reject during promotion run: stop. Agents promote or leave; users reject. -- About to use `## User Feedback` as authored content: rename the heading. -- About to commit from the outer repo: stop. Do not commit data/; DB is canonical. -- About to approve without reading sources: read the cited files first. diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 170866b..0000000 --- a/DESIGN.md +++ /dev/null @@ -1,129 +0,0 @@ - ---- -name: KnowledgeBase Review Console -description: Calm, precise review-and-insights surface for an LLM-fed wiki, handoffs, and rejection patterns. ---- - -# Design System: KnowledgeBase Review Console - -## 1. Overview - -**Creative North Star: "The Workshop Bench"** - -This is a single-operator review console for a personal LLM-fed wiki. The metaphor is a well-lit workshop bench: tools laid out exactly where the hand expects them, surfaces clean, nothing decorative. Wiki reading happens elsewhere (Antigravity / Obsidian); this surface exists strictly to **decide** — approve, reject, inspect a pattern — and to surface the *shape* of those decisions over time. - -The aesthetic family is the restrained-operations cluster: Linear, Plausible, Stripe Dashboard, Raycast. Tinted slate-blue neutrals carry almost the whole UI. One accent appears on ≤5% of any screen, and only for live state (focus, current selection, the row a destructive action is about to land on). Density is deliberate; whitespace is structural, not decorative. Status is conveyed by position, glyph, and label — never by color alone. - -The system explicitly rejects: the four-up KPI strip, the giant-number-with-trend-arrow card, gradient accents, "Welcome back!" greetings, decorative sparklines, soft pastels, rounded-everything, illustration-heavy empty states, and the Filament / Django-Admin / Retool look. If a screen could be confused with a generic SaaS dashboard at a glance, it has failed. - -**Key Characteristics:** -- Restrained color strategy: tinted slate-blue neutrals + a single signal accent ≤5% -- Single technical/geometric sans across the entire UI -- Responsive motion language: feedback + short transitions; no choreography -- Operator-grade density; whitespace is structural, not breathing room -- Lint-grade honesty: state is truthful, never decorative - -## 2. Colors - -A near-monochrome cool-slate palette. Neutrals do almost all the work. The accent earns its place by appearing rarely. - -### Primary -- **Signal** *(`[to be resolved during implementation]`)*: the one accent. Used only for live state — focus rings, the currently-selected review row, the active filter chip, the destructive-action confirmation. Target use: ≤5% of any screen. Hue direction: a single saturated step away from the neutral ramp's hue, leaning slightly cool. Not violet (Linear's lane), not red (Raycast's lane); pick a neighboring tone during implementation. - -### Neutral -The whole surface and text system lives here. All neutrals are **tinted toward a single cool-slate hue** (small chroma, ~0.005–0.012); never pure greys, never `#000` or `#fff`. - -- **Bench** *(deepest surface)*: the page background. Slightly warmer than panel surfaces so panels feel raised without shadow. -- **Panel** *(primary surface)*: cards, the review pane, the dashboard panels. -- **Rail** *(secondary surface)*: nav rail, secondary toolbars, condensed table rows. -- **Hairline** *(divider)*: the project's primary delineator. 1px, low-contrast, used liberally instead of shadows. -- **Ink** *(body text)*: ≥4.5:1 against Panel; the main reading color. -- **Ink-Muted** *(secondary text)*: timestamps, paths, frontmatter keys, byline-style metadata. ≥4.5:1 in the contexts it appears. -- **Ink-Dim** *(tertiary text)*: shortcut hints, "n items" counts, disabled labels. ≥3:1 minimum; never used for primary content. - -*Hex / OKLCH values to be resolved during implementation. Generate the ramp in OKLCH around a single cool-slate hue.* - -### Status (semantic, not decorative) -- **Accepted**: not a color. A solid check glyph and a position change (the row leaves the queue). If a tint is needed, a quiet positive-leaning neutral; never a saturated green. -- **Rejected**: a small ring or hollow glyph, again not pure red. Pair every rejected pill with its TTL countdown so red never means "alarm" — it means "scheduled deletion". -- **Pending / Awaiting**: no color at all. Position in the queue is the status. -- **Expiring soon (TTL ≤ 24h)**: the Signal accent, used here as a deadline cue, not decoration. - -### Named Rules - -**The One Accent Rule.** Across any single screen, Signal occupies ≤5% of the pixels. If it covers more, something has been treated as decoration. Remove it. - -**The Tinted Neutrals Rule.** Every neutral is tinted toward the cool-slate hue. `#000`, `#fff`, and untinted greys are forbidden. The tint is the brand. - -**The Color-Plus-Glyph Rule.** No status — accepted, rejected, pending, expiring — is communicated by color alone. Color is paired with a glyph, a label, or position. Color-blind users see the same state as anyone else. - -## 3. Typography - -**Display Font:** a single technical/geometric sans across the entire UI *(family to be chosen at implementation — Inter, Geist, IBM Plex Sans, and Söhne are all in-register; pick one with a mono companion or excellent tabular figures)*. -**Body Font:** same family. -**Mono companion:** *(to be chosen at implementation; should pair with the display/body family)*. Used for paths, slugs, hashes, frontmatter keys, lint codes, TTL countdowns. - -**Character:** One typeface across the UI gives the console a single, neutral voice. The mono companion is used sparingly and only for content that is *literally a string of code or path* — never for emphasis. Tabular figures are mandatory for any column of numbers (counts, percentages, TTLs). - -### Hierarchy -- **Display** *(weight 500, ~28–32px, line-height 1.15)*: page headings (e.g. "Review Queue", "Dashboard"). Single instance per screen. -- **Headline** *(weight 500, ~20px, line-height 1.25)*: section headings inside a screen. -- **Title** *(weight 500, ~15px, line-height 1.3)*: card / row titles (wiki page title in the review queue, handoff doc title, dashboard panel header). -- **Body** *(weight 400, 14–15px, line-height 1.5, max width 65–75ch)*: descriptive text, page summaries, dashboard prose. Body never goes below 14px effective. -- **Body-Mono** *(weight 400, ~13–14px)*: paths, slugs, lint codes, TTL strings. Tabular figures on. -- **Label** *(weight 500, 11–12px, letter-spacing +0.04em, uppercase)*: small metadata labels — "type", "source", "captured\_at". Used sparingly; never for body content. - -### Named Rules - -**The One Family Rule.** A single typographic family carries the entire UI. Pairing a display serif with a body sans is in-register for editorial brand work, not for this operations console; it is prohibited here. - -**The Tabular Figures Rule.** Every column of numbers — counts, percentages, TTL countdowns, IDs — uses tabular figures. Numbers must align vertically. Proportional figures in a stat column are a defect. - -**The Mono-for-Strings Rule.** Mono is reserved for content that *is* a literal string: paths, slugs, hashes, lint codes, frontmatter keys. Mono used for "technical feel" or emphasis is prohibited. - -## 4. Elevation - -Flat by default. Depth is conveyed by **tonal layering** between Bench (page), Panel (primary surface), and Rail (secondary surface) — never by shadow on resting elements. Hairline dividers do almost all the structural work that shadows would do in a more decorative system. - -The only acceptable shadows are **state shadows**, applied transiently: -- A subtle focus halo on the currently-focused interactive element (paired with the Signal accent ring). -- A faint lift on a row when it's the target of a hover-pending destructive action. -- Modals, when they exist, sit on a low-opacity scrim, not a glassy blur. Glassmorphism is forbidden anywhere in this system. - -### Named Rules - -**The Flat-By-Default Rule.** Surfaces are flat at rest. Shadow appears only as a response to state (focus, destructive-target hover). Persistent decorative shadows on cards, panels, or rails are prohibited. - -**The Hairline-Not-Shadow Rule.** Where another design would reach for a soft shadow to separate surfaces, this system reaches for a 1px hairline. Hairlines are the project's primary tool for structure. - -## 5. Components - -*Component specification deferred until implementation. The console has no components yet; the next pass of `/impeccable document` will run in scan mode and capture the real primitives — review row, dashboard panel, handoff card, TTL countdown, filter chip, command palette — directly from the codebase.* - -## 6. Do's and Don'ts - -### Do: -- **Do** tint every neutral toward the cool-slate hue (chroma ~0.005–0.012). Pure greys, `#000`, and `#fff` are forbidden. -- **Do** keep the Signal accent on ≤5% of any screen, reserved for live state and the next destructive action. -- **Do** pair every status (accepted / rejected / pending / expiring) with a glyph or label, never color alone. -- **Do** use tabular figures in every column of numbers — counts, percentages, TTL strings, IDs. -- **Do** prefer 1px hairlines over shadows to separate surfaces. -- **Do** keep body text ≥14px effective and body line length within 65–75ch. -- **Do** honor `prefers-reduced-motion: reduce`: remove transitions, fall back to opacity / instant state changes only. -- **Do** put real values on screen — real counts, real timestamps, real lint codes. Fake skeletons or rounded-down numbers that hide errors are forbidden. - -### Don't: -- **Don't** build the four-up KPI strip (Total / Active / Pending / +12% MoM). It is the SaaS-dashboard cliché the PRODUCT.md explicitly rejects. -- **Don't** use the giant-number-with-trend-arrow card. A correctly chosen number in its own context beats a hero metric every time. -- **Don't** use decorative sparklines. Charts must explain a rejection or acceptance pattern; if a chart doesn't answer a question, delete it. -- **Don't** apply gradients anywhere — on text (`background-clip: text` is banned), on backgrounds, on accents. -- **Don't** use side-stripe borders (`border-left` or `border-right` > 1px as a colored accent). On rows, status pills, callouts: forbidden. -- **Don't** apply glassmorphism — backdrop-blur as decoration, "glassy" panels, frosted nav. Forbidden anywhere in this system. -- **Don't** wrap content in identical card grids. Cards are the lazy answer; nested cards are always wrong. -- **Don't** write "Welcome back!", "Awesome!", or any encouragement copy. The voice is short, factual, second-person only when necessary; never marketing. -- **Don't** show illustration-heavy empty states. Empty states are short factual sentences (e.g. "No pages awaiting review. Last cron completed at 03:17.") with the next action, if any, in plain text. -- **Don't** drift toward Filament / Django-Admin / Retool aesthetics — uniform tables, generic action menus, framework-default forms. The PRODUCT.md anti-reference is the line. -- **Don't** drift toward consumer-warm notebook aesthetics — soft pastels, rounded-everything, friendly serifs, mascots. Wrong register for this tool. -- **Don't** add Notion/WYSIWYG editing affordances — slash menus, drag handles, block editors. This is a review surface, not an editing surface. -- **Don't** animate CSS layout properties. Animate transform and opacity only. No bounce, no elastic; ease out with quart/quint/expo curves. -- **Don't** rely on color alone to communicate any status, ever. diff --git a/PRODUCT.md b/PRODUCT.md deleted file mode 100644 index 8c92b29..0000000 --- a/PRODUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Product - -## Register - -product - -## Users - -A single owner-operator (the repo author) running KnowledgeBase as a personal LLM-fed wiki and operations system. The user already lives in the CLI (`kb-lint-wiki`, `kb-wiki-review`, daily report scripts) and reads finished wiki pages inside Antigravity or Obsidian — those tools own the reading experience. - -The user comes to this frontend in short, focused sessions, usually after the daily ingest/fill cron has produced new candidate wiki pages, rejections, and handoff updates. The job is to: - -1. Review pending AI-written wiki pages and approve, reject, or send back for revision. -2. Track handoff documents in flight — what's open, what's been promoted, what's stalled. -3. Read the dashboard for rejection and acceptance patterns: which wiki types get rejected most, which sources produce noisy raw data, which rejected pages are about to be deleted by the TTL sweep. - -Context of use: desktop browser on a local network, alongside an editor and terminal. Sessions are minutes long, not hours. No phone use, no on-the-go review. - -## Product Purpose - -A precise, calm review-and-insights surface for an LLM-fed wiki the user has already built around `kb-wiki-review`, handoff documents, and lint. The frontend exists to: - -- Replace the parts of `kb-wiki-review` that are slow or blind in a terminal: side-by-side review of pending pages with their raw sources, batch decisions, and the "what's about to expire" view. -- Make handoff status legible at a glance instead of grepping `data/handoffs/`. -- Surface the patterns in accept / reject decisions that no individual page review can show: rejection rate by wiki type, by source, by author-agent, by time-of-week; TTL countdown for rejected pages; what's about to be permanently deleted. - -Out of scope for now: wiki page reading (Antigravity/Obsidian do this), editing wiki bodies, raw source browsing, content authoring, multi-user collaboration, mobile. - -Success looks like: a one-screen weekly read tells the user *"the AI is over-producing concept pages from conversation sources and 60% are getting rejected for thin sourcing"* — a finding the CLI cannot produce, and that changes upstream prompts. - -## Brand Personality - -Calm, precise, expert-tool. Three words: **considered, accurate, quiet**. - -The reference cluster is Linear, Raycast, Stripe Dashboard, Plausible — interfaces that respect the user's existing expertise, do not explain themselves, and earn trust through restraint and accuracy rather than personality. The voice in the UI is short, factual, second-person only when necessary. No mascots, no encouragement, no exclamation marks, no "Awesome!" empty states. - -Emotionally the frontend should feel like a well-organized workshop bench: tools laid out, surfaces clean, nothing decorative. The user should never feel marketed to by their own tool. - -## Anti-references - -What this must explicitly NOT look or feel like: - -- **Heavy admin frameworks** — Django Admin, Filament, Retool, generic CRUD UIs. Tell-tale signs: dense uniform tables, generic action menus, framework-default forms, no opinion about what matters on a screen. KnowledgeBase has strong opinions about wiki types and handoff lifecycle; the UI must reflect that, not flatten it into rows. -- **Generic SaaS dashboard clichés** — big hero metric cards (Total / Active / +12% MoM), three identical KPI tiles in a row, gradient accents, "Welcome back, user!" greetings, illustration-heavy empty states, Vercel/Tailwind-template hero blocks. Specifically banned shapes: the four-up KPI strip, the giant-number-with-trend-arrow card, decorative sparklines. -- **Notion / WYSIWYG editor energy** — slash commands, drag handles, block menus. This is a *review* surface, not an editing surface. -- **Consumer-warm notebook aesthetics** — soft pastels, rounded-everything, friendly illustrations, cozy serif headings paired with playful sans body. Wrong register for an operations tool. - -## Design Principles - -1. **Review, don't read.** Wiki reading lives in Antigravity and Obsidian. Every screen here is for *deciding* (approve / reject / inspect a pattern), not for consuming long-form content. If a screen starts looking like a reader, it has drifted out of scope. - -2. **Patterns over piles.** A list of rejected pages is the floor, not the ceiling. The dashboard's real job is to make the *shape* of decisions legible — rejection rate by type, by source, by time — so the user can fix upstream prompts and ingestion, not just clear a queue. - -3. **Precision over polish.** A correctly chosen number, a sharp diff, a real TTL countdown are worth more than animated charts and gradient cards. Borrow the Linear/Stripe rule: if you can't justify a visual element to the operator-user, remove it. - -4. **One user, no shims.** This is a single-operator tool. No role badges, no "your team" copy, no shared-cursor presence, no permission UI. Designing for a hypothetical future team would inject SaaS clichés the user explicitly rejected. - -5. **Lint-grade honesty.** The wiki is enforced by lint; the UI must match that ethos. No fake totals, no rounded-down counts that hide errors, no skeleton placeholders that imply data exists when it doesn't, no "Everything looks great!" empty states when the cron actually failed at 03:17. - -## Implementation Stack - -- **Frontend:** Vite + React + TypeScript. Single-page application loaded locally; no SSR, no SEO. -- **Backend:** small FastAPI service inside the existing `src/kb/` Python tree. Read-mostly endpoints over the markdown content of `data/wiki/`, `data/handoffs/`, `data/log.md`, and over `kb-wiki-review` actions for approve / reject / TTL queries. -- **Data:** the markdown files in `data/` remain the source of truth. The API reads them with frontmatter parsing on demand; no separate database. Approve / reject mutations go through the existing `kb-wiki-review` machinery, not a parallel write path. -- **Runtime posture:** local-only, single user, single machine. Same privacy stance as the rest of the repo: never deployed, never exposed beyond localhost. -- **Why this stack over HTMX:** the review console's keyboard-first interactions (command palette, multi-select queue actions, optimistic approve / reject with TTL hint motion) and tabular data with live filters are native in React and possible-but-grafted in HTMX. The two-runtime cost (uv + npm) is accepted in exchange for that fit. - -This is the only place this decision is recorded. No separate ADR. - -## Accessibility & Inclusion - -- WCAG 2.1 AA as the floor: 4.5:1 contrast on body text, 3:1 on UI components and large text, visible focus rings on every interactive element, semantic landmarks, labelled form controls, error messages tied to inputs with `aria-describedby`. -- `prefers-reduced-motion: reduce` honored throughout. Default motion is already minimal (no parallax, no hero animations); the reduced variant removes the few remaining transitions and uses opacity/instant state changes only. -- Keyboard navigable across all primary flows (review queue, handoff list, dashboard filters). Standard tab order, no keyboard traps. -- No reliance on color alone to convey status — accepted / rejected / pending / expiring also carry a glyph, label, or position. -- Body text never below 14px effective; line length capped at 65–75ch. diff --git a/docs/db_informations/hermes-monitoring-guide.md b/docs/db_informations/hermes-monitoring-guide.md deleted file mode 100644 index 77f6d9b..0000000 --- a/docs/db_informations/hermes-monitoring-guide.md +++ /dev/null @@ -1,106 +0,0 @@ -# Hermes Monitoring Guide — Daily & Weekly Report - -Updated: 2026-05-11 - -## 1. Synopsis - -- **Purpose**: Hermes DB에서 어떤 데이터를 왜 모니터링해야 하는지 정의. 쿼리는 에이전트가 이 문서를 보고 생성. -- **DB**: `~/.hermes/state.db` + `~/.hermes/profiles/*/state.db` - ---- - -## 2. 모니터링 지표 정의 - -### Layer 1: 비용·효율 (Cost & Efficiency) - -**목적**: 얼마나 썼나, 낭비는 없나. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 일일 총 비용 | `COALESCE(actual_cost_usd, estimated_cost_usd)` | 전날 대비 20%+ 증가 시 드릴다운 | -| 모델별 비용 점유율 | `sessions.model` + 비용 | 비싼 모델이 단순 task에 쓰이는지 확인 | -| 캐시 히트율 | `cache_read_tokens / (input_tokens + cache_read_tokens) × 100` | 낮으면 system prompt 구조 재검토 | -| reasoning 토큰 비율 | `reasoning_tokens / output_tokens` | thinking 모델에서 단순 task에 낭비되는지 확인 | -| API 호출 횟수 | `sessions.api_call_count` | 세션당 과도하면 루프 탈출 실패 의심 | - -### Layer 2: 작업 품질 (Task Quality) - -**목적**: 작업이 완료됐나, 어디서 막혔나. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 완료율 | `end_reason = 'cron_complete', 'cli_close', 'api_complete' 등` / 전체 | 실측: `cli_close` 31건, NULL(비정상) 24건 | -| end_reason 분포 | `sessions.end_reason` | `context_limit` 多 → compaction 설정 조정, `error` 多 → tool 실패 분석 | -| 진행 중 세션 수 | `ended_at IS NULL` | 비정상적으로 많으면 좀비 세션 의심 | -| 도구 호출 횟수 | `sessions.tool_call_count` | 세션당 평균 대비 3배+ = 루프 또는 반복 재시도 | -| context compression 발생 | `end_reason = 'compression'` 또는 `parent_session_id IS NOT NULL` | 세션 분할 또는 컨텍스트 정리 전략 수립 | - -### Layer 3: 행동 패턴 (Behavioral Patterns) - -**목적**: 언제, 어떻게, 어디서 일하나. - -| 지표 | 소스 | 의미 | -|------|------|------| -| source별 세션 분포 | `sessions.source` | 실측: cli(37) > api_server(18) > telegram(12) > cron(11) > tui(10) | -| 시간대별 세션 분포 | `strftime('%H', started_at, 'unixepoch')` | 피크 시간대 파악, cron 스케줄 최적화 | -| 평균 세션 시간 | `AVG(ended_at - started_at)` | 길어지면 context 폭발 또는 tool 루프 의심 | -| 도구 사용 Top N | `messages.tool_calls` JSON 파싱 | 가장 많이 쓰는 도구 패턴 파악 | -| 세션당 도구 호출 빈도 | `tool_call_count / COUNT(sessions)` | 특정 도구가 평소의 3배면 루프 징후 | - -### Layer 4: 프로파일별 분포 (Multi-Agent Focus) - -**목적**: 어느 에이전트가 얼마나 일하는가. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 프로파일별 세션·비용 | 각 `profiles/*/state.db` | 에이전트별 부하 분산 확인 | -| 프로파일별 모델 사용 | `sessions.model` per profile | 에이전트별 모델 라우팅 정책 검증 | -| root vs subagent 비율 | `parent_session_id IS NULL` | 실측: root 85개 vs subagent 4개 | - ---- - -## 3. 신호 → 액션 매핑 - -| 신호 | 임계값 | 액션 | -|------|--------|------| -| 일일 비용 급증 | 전날 대비 +20% | 해당일 세션 Top 5 드릴다운 | -| 캐시 히트율 저하 | < 30% | system prompt 구조 재검토 | -| NULL end_reason 비율 | > 20% | 비정상 종료 원인 분석 | -| 세션당 API 호출 | 평균 대비 3배+ | 루프 탈출 실패 여부 확인 | -| reasoning 토큰 비율 | > 30% | `max_thinking_tokens` 조정 검토 | -| context compression | 주간 3회+ | 세션 분할 전략 수립 | -| 특정 도구 호출 급증 | 평소 대비 3배+ | 루프 또는 반복 재시도 징후 | - ---- - -## 4. Daily vs Weekly 분리 기준 - -``` -Daily Report (어제 기준) Weekly Report (지난 7일) -───────────────────────── ────────────────────────────── -• 총 비용 / 세션 수 / 완료율 • 일별 비용·토큰 트렌드 -• source별 사용 분포 • 캐시 히트율 추이 (개선/악화) -• end_reason 분포 • 모델별 비용 점유율 -• 도구 사용 Top 10 • 도구별 calls_per_session 트렌드 -• 비정상 종료 세션 목록 • 피크 시간대 패턴 -• 진행 중(좀비) 세션 수 • 프로파일별 부하 분산 -``` - ---- - -## Appendix - -### A. OpenCode와의 차이점 - -| 항목 | Hermes | OpenCode | -|------|--------|----------| -| 타임스탬프 단위 | **초** (float) | **밀리초** (integer) | -| 토큰 저장 위치 | `sessions` 테이블 컬럼 | `part` 테이블 JSON | -| 비용 컬럼 | `actual_cost_usd` / `estimated_cost_usd` 분리 | `part.step-finish.cost` | -| 도구 호출 기록 | `messages.tool_calls` JSON 배열 | `part.tool` JSON | -| 멀티 DB | 프로파일별 독립 DB | 단일 DB | -| FTS 지원 | ✅ (FTS5 + trigram) | ❌ | - -### B. 관련 문서 - -- `hermes-schema-reference.md` — 테이블·컬럼·JSON 구조 상세 diff --git a/docs/db_informations/hermes-schema-reference.md b/docs/db_informations/hermes-schema-reference.md deleted file mode 100644 index b5d982d..0000000 --- a/docs/db_informations/hermes-schema-reference.md +++ /dev/null @@ -1,169 +0,0 @@ -# Hermes SQLite Schema Reference - -Updated: 2026-05-11 - -## 1. Synopsis - -- **Purpose**: Hermes DB 스키마 레퍼런스. 에이전트가 쿼리 작성 전 참조하는 원천 문서. -- **DB**: `~/.hermes/state.db` (SQLite, WAL 모드, FTS5, schema v11) - ---- - -## 2. DB 위치 - -| 경로 | 설명 | -|------|------| -| `~/.hermes/state.db` | **메인 DB** — 모든 세션·메시지 기록 | -| `~/.hermes/profiles/*/state.db` | 프로파일별 독립 DB (멀티-에이전트) | -| `~/.hermes/kanban.db` | 칸반 태스크 DB | - -**프로파일 목록**: `main-gateway`, `research-agent`, `execution-agent`, `structuring-agent`, `verification-agent` - -> 프로파일 DB는 메인 DB와 **완전히 별개**. 합산 시 개별 조회 후 UNION 필요. - ---- - -## 3. 테이블 관계도 - -``` -sessions ──< messages - └──< messages_fts (FTS5 전문 검색) - └──< messages_fts_trigram (CJK/부분 문자열 검색) -state_meta (키-값 메타데이터) -schema_version (마이그레이션 버전) -``` - -- `sessions.id` → `messages.session_id` -- `sessions.parent_session_id` → `sessions.id` (self-referential, context compression 분기) - ---- - -## 4. 테이블 스키마 - -### sessions - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `id` | TEXT PK | 세션 식별자 (`YYYYMMDD_HHMMSS_hex`) | -| `source` | TEXT | `cli` \| `tui` \| `telegram` \| `slack` \| `cron` \| `api_server` | -| `user_id` | TEXT | 사용자 식별자 (NULL 가능) | -| `model` | TEXT | 사용 모델명 (예: `claude-opus-4-7`, `Qwen3.6-35B-A3B-FP8`) | -| `parent_session_id` | TEXT FK | NULL = root 세션, 값 있음 = context compression 분기 | -| `started_at` | REAL | Unix epoch **초** (float) | -| `ended_at` | REAL | Unix epoch 초. NULL = 진행 중 | -| `end_reason` | TEXT | 종료 이유 (아래 참조) | -| `message_count` | INTEGER | 세션 내 메시지 수 | -| `tool_call_count` | INTEGER | 도구 호출 횟수 | -| `api_call_count` | INTEGER | API 호출 횟수 | -| `input_tokens` | INTEGER | 입력 토큰 합계 | -| `output_tokens` | INTEGER | 출력 토큰 합계 | -| `cache_read_tokens` | INTEGER | 캐시 읽기 토큰 | -| `cache_write_tokens` | INTEGER | 캐시 쓰기 토큰 | -| `reasoning_tokens` | INTEGER | Extended thinking 토큰 | -| `billing_provider` | TEXT | 청구 제공자 | -| `billing_mode` | TEXT | 청구 방식 | -| `estimated_cost_usd` | REAL | 추정 비용 | -| `actual_cost_usd` | REAL | 실제 청구 비용 | -| `cost_status` | TEXT | 비용 계산 상태 (`unknown` 등) | -| `title` | TEXT | 세션 제목 (UNIQUE, NULL 가능) | - -> **비용 컬럼 우선순위**: `COALESCE(actual_cost_usd, estimated_cost_usd)` -> **캐시 히트율**: `cache_read_tokens / (input_tokens + cache_read_tokens) × 100` - -**`end_reason` 값 목록** - -| 값 | 설명 | -|----|------| -| `cli_close` | CLI 세션 종료 | -| `tui_close` / `tui_shutdown` | TUI 세션 종료 | -| `cron_complete` | 크론 작업 완료 | -| `api_complete` | API 서버 요청 정상 완료 | -| `api_error` | API 서버 요청 중 에러/크래시 | -| `api_cancelled` | API 서버 요청 서버 취소 (shutdown 등) | -| `api_client_disconnect` | API 서버 SSE 클라이언트 연결 끊김 | -| `session_reset` | 세션 리셋 | -| `new_session` | 새 세션으로 전환 | -| `compression` | 컨텍스트 압축 | -| NULL | 비정상 종료 또는 진행 중 | - -### messages - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `id` | INTEGER PK | 자동 증가 | -| `session_id` | TEXT FK | → `sessions.id` | -| `role` | TEXT | `user` \| `assistant` \| `tool` \| `session_meta` | -| `content` | TEXT | 메시지 본문 | -| `tool_calls` | TEXT (JSON) | 도구 호출 배열 (role=assistant일 때) | -| `tool_name` | TEXT | 호출된 도구 이름 (role=tool일 때) | -| `timestamp` | REAL | Unix epoch 초 | -| `finish_reason` | TEXT | `stop` \| `tool_calls` | -| `reasoning` | TEXT | Extended thinking 원문 | -| `token_count` | INTEGER | 해당 메시지 토큰 수 | - ---- - -## 5. messages.tool_calls JSON 구조 - -```json -[ - { - "id": "call_function_xxx", - "call_id": "call_function_xxx", - "type": "function", - "function": { - "name": "terminal", - "arguments": "{\"command\": \"ls -la\"}" - } - } -] -``` - -> 도구명 추출: `json_extract(value, '$.function.name')` + `json_each(tool_calls)` -> `tool_name` 컬럼은 role=tool 행에만 채워짐. role=assistant의 도구 호출은 `tool_calls` JSON에서 추출. - ---- - -## Appendix - -### A. 공통 제약 - -- 타임스탬프는 **초 단위 float** → `date(col, 'unixepoch')` (OpenCode의 밀리초와 다름) -- 비용: `COALESCE(actual_cost_usd, estimated_cost_usd)` 패턴 사용 -- 루트 세션만 집계: `WHERE parent_session_id IS NULL` (중복 집계 방지) -- 진행 중 세션 제외: `WHERE ended_at IS NOT NULL` -- 스키마 버전 확인: `SELECT version FROM schema_version;` (현재: 11) - -### B. 인덱스 (성능 참고) - -``` -idx_sessions_source ON sessions(source) -idx_sessions_parent ON sessions(parent_session_id) -idx_sessions_started ON sessions(started_at DESC) -idx_messages_session ON messages(session_id, timestamp) -``` - -### C. FTS5 검색 - -```sql --- 전문 검색 (영문) -SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'keyword'; - --- 부분 문자열 / CJK 검색 -SELECT rowid FROM messages_fts_trigram WHERE messages_fts_trigram MATCH '검색어'; -``` - -> FTS5 인덱스는 `content + tool_name + tool_calls` 를 합쳐서 색인. - -### D. 프로파일 DB 통합 조회 패턴 - -```bash -for db in ~/.hermes/state.db ~/.hermes/profiles/*/state.db; do - [ -f "$db" ] || continue - sqlite3 "$db" "SELECT ..." -done -``` - -### E. 관련 문서 - -- `hermes-monitoring-guide.md` — 무엇을 왜 모니터링할지 지표 정의 diff --git a/docs/db_informations/opencode-monitoring-guide.md b/docs/db_informations/opencode-monitoring-guide.md deleted file mode 100644 index c00a1d6..0000000 --- a/docs/db_informations/opencode-monitoring-guide.md +++ /dev/null @@ -1,84 +0,0 @@ -# OpenCode Monitoring Guide — Daily & Weekly Report - -Updated: 2026-05-11 - -## 1. Synopsis - -- **Purpose**: OpenCode DB에서 어떤 데이터를 왜 모니터링해야 하는지 정의. 쿼리는 에이전트가 이 문서를 보고 생성. -- **DB**: `~/.local/share/opencode/opencode.db` — 스키마 상세는 `opencode-schema-reference.md` 참조. - ---- - -## 2. 모니터링 지표 정의 - -### Layer 1: 비용·효율 (Cost & Efficiency) - -**목적**: 얼마나 썼나, 낭비는 없나. - -> **모델별 토큰·비용은 `message` 테이블이 유일한 소스.** -> `session.model`은 subagent 세션에서 NULL이지만, `message.data.modelID`에는 root/subagent 구분 없이 실제 모델명이 정확히 기록된다. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 모델별 일일 토큰 | `message.data` (modelID, tokens.*) | root+subagent 포함 실제 모델 분포 | -| 기록 비용 | `message.data.cost` | OpenCode가 모든 모델에 대해 자동 집계. vllm(자체호스팅)은 0 | -| 캐시 히트율 | `cache_read / (input_cache_miss + cache_read) × 100` | 높을수록 비용 효율 좋음. `input`은 캐시 미스분만이므로 반드시 cache_read와 합산해서 분모 계산 | -| API 호출 횟수 | SQLite `part.type=step-finish` count per session | 51회+ = 루프 탈출 실패 의심 | -| reasoning 토큰 비율 | `message.tokens.reasoning` / 전체 | Extended thinking 남용 여부 | - -### Layer 2: 작업 품질 (Task Quality) - -**목적**: 작업이 완료됐나, 어디서 막혔나. - -| 지표 | 소스 | 의미 | -|------|------|------| -| Todo 완료율 | `todo.status` | 전체 평균 80.9%. 세션 완료율 0% = 중단된 작업 | -| 도구별 에러율 | `tool.state.status = error` | webfetch 12.8%, write 5.1%, edit 3.9% | -| 연속 에러 세션 | 같은 도구가 동일 세션에서 2회+ 실패 | 에이전트가 막혀서 반복 실패하는 구간 | -| compaction 발생 | `part.type = compaction` | 세션이 너무 길어짐 → 분할 검토 | -| step-finish reason=error | `step-finish.reason = error` | API 레벨 실패 (드묾, 실측 2건) | - -### Layer 3: 행동 패턴 (Behavioral Patterns) - -**목적**: 언제, 어떻게 일하나 — 습관 최적화. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 시간대별 세션 분포 | `session.time_created` hour | 실측: 새벽 1~3시 집중(150개), 낮 저활동 | -| 요일별 활동량 | `session.time_created` weekday | 실측: 월요일 136개로 압도적 1위 | -| root vs subagent 비율 | `session.parent_id IS NULL` | 실측: root 86개 vs subagent 249개 (위임율 74%) | -| 도구 사용 믹스 | `tool.tool` 분포 | read > bash > grep 순. 탐색 vs 실행 비율 | -| 세션 길이 분포 | `step-finish` count per session | short(35%) / medium(40%) / long(18%) / very long(6%) | - -### Layer 4: 집중도·안정성 (Focus & Stability) - -**목적**: 어디에 집중했나, 무엇이 불안정한가. - -| 지표 | 소스 | 의미 | -|------|------|------| -| 일일 프로젝트 전환 수 | SQLite `session.project_id` distinct per day | 3개+ = 컨텍스트 스위칭 과다 | -| 핫 파일 (주간 수정 빈도) | SQLite `patch.files[]` | 반복 수정 = 설계 불안정. 실측: mail_server.py 46회 | -| 세션당 수정 파일 수 | SQLite `patch.files[]` distinct per session | 과도하게 많으면 범위 초과 작업 | -| 모델별 세션 분포 (전체) | `message.data.modelID` GROUP BY | root+subagent 포함 실제 모델 분포. session.model 사용 금지 | - ---- - -## 4. 신호 → 액션 매핑 - -| 신호 | 임계값 | 액션 | -|------|--------|------| -| API 호출 횟수 | 세션 > 50회 | 루프 탈출 실패 여부 확인, 세션 분할 검토 | -| 도구 에러율 | 도구별 > 5% | retry 로직 추가 또는 대체 도구 검토 | -| Todo 완료율 | 세션 < 50% | 중단 원인 분석 (계획 과다? 실행 실패?) | -| compaction 발생 | 1회 이상 | 세션 분할 또는 컨텍스트 정리 전략 수립 | -| 일일 프로젝트 전환 | > 3개 | 집중도 저하 경고 | -| 핫 파일 수정 횟수 | 주간 > 10회 | 설계 불안정 → 리팩토링 우선순위 상향 | -| reasoning 토큰 비율 | > 30% | Extended thinking 사용 타당성 검토 | - ---- - -## Appendix - -### A. 관련 문서 - -- `opencode-schema-reference.md` — 테이블·컬럼·JSON 구조 상세 diff --git a/docs/db_informations/opencode-schema-reference.md b/docs/db_informations/opencode-schema-reference.md deleted file mode 100644 index fe0f0ec..0000000 --- a/docs/db_informations/opencode-schema-reference.md +++ /dev/null @@ -1,215 +0,0 @@ -# OpenCode SQLite Schema Reference - -Updated: 2026-05-11 - -## 1. Synopsis - -- **Purpose**: OpenCode DB 스키마 레퍼런스. 에이전트가 쿼리 작성 전 참조하는 원천 문서. -- **DB**: `~/.local/share/opencode/opencode.db` (SQLite 3.51.2, Drizzle ORM, WAL 모드) - ---- - -## 2. 테이블 관계도 - -``` -project ──< session ──< part - └──< message ──< part - └──< todo - └──< session_share -``` - -- `project.id` → `session.project_id` -- `session.id` → `part.session_id`, `message.session_id`, `todo.session_id` -- `message.id` → `part.message_id` - ---- - -## 3. 테이블 스키마 - -### project - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `id` | TEXT PK | 프로젝트 식별자 | -| `worktree` | TEXT | 작업 디렉토리 경로 | -| `name` | TEXT | 프로젝트 이름 (NULL 가능) | -| `time_created` | INTEGER | Unix epoch 밀리초 | -| `time_updated` | INTEGER | Unix epoch 밀리초 | - -### session - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `id` | TEXT PK | 세션 식별자 (`ses_...`) | -| `project_id` | TEXT FK | → `project.id` | -| `parent_id` | TEXT | NULL = root 세션, 값 있음 = subagent 세션 | -| `title` | TEXT | 세션 제목 | -| `model` | TEXT (JSON) | root 세션만 기록. **subagent 세션은 NULL** (upstream 미기록) | -| `agent` | TEXT | 에이전트 이름 (예: `CodeRabbit-Dev`) | -| `directory` | TEXT | 작업 디렉토리 경로 | -| `time_created` | INTEGER | Unix epoch 밀리초 | -| `time_updated` | INTEGER | Unix epoch 밀리초 | - -> ⚠️ **`session.model`은 모델 집계에 사용하지 말 것.** subagent 세션은 NULL이므로 불완전하다. -> 실제 모델명은 `message.data.modelID`에 root/subagent 구분 없이 정확히 기록된다. - -### part - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `id` | TEXT PK | 파트 식별자 (`prt_...`) | -| `message_id` | TEXT FK | → `message.id` | -| `session_id` | TEXT FK | → `session.id` | -| `time_created` | INTEGER | Unix epoch 밀리초 | -| `data` | TEXT (JSON) | `data.type`으로 분기 (아래 참조) | - -**`data.type` 값 목록** - -| type | 비율 | 설명 | -|------|------|------| -| `tool` | 34% | 도구 호출 1건 | -| `step-start` | 19% | API 호출 시작 | -| `step-finish` | 19% | API 호출 완료 — 토큰·비용의 유일한 소스 | -| `reasoning` | 14% | Extended thinking 내용 | -| `text` | 13% | 텍스트 응답 | -| `patch` | 2% | 파일 변경 스냅샷 | -| `file` | <1% | 첨부 파일 | -| `compaction` | <1% | 컨텍스트 압축 발생 | - -### todo - -| 컬럼 | 타입 | 설명 | -|------|------|------| -| `session_id` | TEXT FK | → `session.id` | -| `content` | TEXT | 할 일 내용 | -| `status` | TEXT | `pending` \| `in_progress` \| `completed` \| `cancelled` | -| `priority` | TEXT | `high` \| `medium` \| `low` | -| `position` | INTEGER | 순서 (PK 구성요소) | -| `time_created` | INTEGER | Unix epoch 밀리초 | - ---- - -## 4. part.data JSON 구조 (타입별) - -### step-finish -```json -{ - "type": "step-finish", - "reason": "tool-calls", - "snapshot": "a2ff2a1b...", - "tokens": { - "total": 42537, - "input": 1592, - "output": 113, - "reasoning": 0, - "cache": { "write": 0, "read": 40832 } - }, - "cost": 0 -} -``` -> `total = input + cache.read + output` -> **`input` = 캐시 미스 토큰만** (새로 읽힌 것). `cache.read` = 캐시 히트 토큰. -> 실제 LLM 입력량 = `input + cache.read`. 캐시 히트율 = `cache.read / (input + cache.read)` -> `cost`는 OpenCode가 모든 모델에 대해 자동 집계해서 기록. - -### tool -```json -{ - "type": "tool", - "tool": "bash", - "callID": "toolu_...", - "state": { - "status": "completed", - "input": { "command": "ls -la" }, - "metadata": { "output": "..." } - } -} -``` -> `state.status`: `completed` | `error` | `running` | `pending` - -### patch -```json -{ - "type": "patch", - "hash": "a2ff2a1b...", - "files": ["/absolute/path/to/file.py"] -} -``` - -### reasoning -```json -{ - "type": "reasoning", - "thinking": "..." -} -``` - -### compaction -```json -{ - "type": "compaction" -} -``` -> 세션이 너무 길어져 컨텍스트 압축이 발생했음을 나타냄. - ---- - -## Appendix - -### A. 공통 제약 - -- 모든 `time_*` 컬럼은 **밀리초 epoch** → `datetime(col/1000,'unixepoch')` -- `session.model`은 root 세션만 기록, subagent는 NULL → **모델 집계에 사용 금지** -- **모델별 토큰·비용은 `message` 테이블에서 집계** (아래 §B 참조) -- `patch.files`는 JSON 배열 → `json_each(json_extract(data,'$.files'))`로 펼치기 - -### B. message 테이블 — 토큰·비용 집계 (핵심) - -`message.data` (JSON)에 root/subagent 구분 없이 실제 모델명과 토큰이 기록된다. - -```sql --- 날짜별 모델별 토큰·비용 집계 (KST 기준) -SELECT - json_extract(m.data, '$.modelID') AS modelID, - json_extract(m.data, '$.providerID') AS providerID, - SUM(json_extract(m.data, '$.tokens.input')) AS input_cache_miss, -- 캐시 미스 토큰만 - SUM(json_extract(m.data, '$.tokens.cache.read')) AS cache_read, -- 캐시 히트 토큰 - SUM(json_extract(m.data, '$.tokens.output')) AS output, - SUM(json_extract(m.data, '$.tokens.cache.write')) AS cache_write, - SUM(json_extract(m.data, '$.tokens.reasoning')) AS reasoning, - SUM(json_extract(m.data, '$.tokens.input') - + json_extract(m.data, '$.tokens.cache.read')) AS total_input, -- 실제 LLM 입력량 - SUM(json_extract(m.data, '$.cost')) AS actual_cost -FROM message m -JOIN session s ON m.session_id = s.id -WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = 'YYYY-MM-DD' - AND json_extract(m.data, '$.role') = 'assistant' -GROUP BY modelID, providerID -ORDER BY total_input DESC; -``` - -> **캐시 히트율** = `cache_read / (input_cache_miss + cache_read) × 100` - -**providerID별 비용 처리 규칙:** - -| providerID | 비용 소스 | -|------------|----------| -| `anthropic` | `message.data.cost` (OpenCode 자동 집계) | -| `openai` | `message.data.cost` (OpenCode 자동 집계) | -| `opencode-go` | `message.data.cost` (실청구) | -| `google` | `message.data.cost` (OpenCode 자동 집계) | -| `vllm` | 0 USD (자체호스팅, 집계 제외) | - -### B. 인덱스 (성능 참고) - -``` -part_session_idx ON part(session_id) -part_message_id_id_idx ON part(message_id, id) -session_project_idx ON session(project_id) -session_parent_idx ON session(parent_id) -todo_session_idx ON todo(session_id) -``` - -### C. 관련 문서 - -- `opencode-monitoring-guide.md` — 무엇을 왜 모니터링할지 지표 정의 diff --git a/docs/db_informations/report-pipeline.md b/docs/db_informations/report-pipeline.md deleted file mode 100644 index 68222a2..0000000 --- a/docs/db_informations/report-pipeline.md +++ /dev/null @@ -1,109 +0,0 @@ -# Report Pipeline — Source-Specific Daily Reports - -Updated: 2026-05-18 - -## 1. Synopsis - -- **Purpose**: Define the source-specific daily report pipeline for OpenCode, Hermes, and Claude Code. -- **Format**: All reports are Markdown. Storage root: `data/wiki/summaries/`. - ---- - -## 2. Pipeline Structure - -``` -OpenCode DB -> data/wiki/summaries/YYYY/MM/YYYY-MM-DD-opencode-usage.md -Hermes DB -> data/wiki/summaries/YYYY/MM/YYYY-MM-DD-hermes-usage.md -Claude telemetry -> data/wiki/summaries/YYYY/MM/YYYY-MM-DD-claude-code-usage.md - -Daily memory cron reads source-specific reports and writes: -data/wiki/summaries/YYYY/MM/YYYY-MM-DD-memory.md -``` - -### Source Responsibilities - -| Source | Metrics | Reason | -|------|----------|------| -| `message.data` (modelID, tokens, cost) | Model tokens, recorded cost, shadow cost inputs | Records actual model names across root and subagent sessions. | -| `session` (parent_id, time_created) | Session count, root/subagent split, hourly distribution, project distribution | Session structure source. | -| `part` (type=tool, patch, compaction) | Tool error rate, hot files, compaction | In-session behavior data. | -| `todo` | TODO completion rate | Work quality proxy. | -| SQLite `hermes/state.db` | Hermes metrics | Source of truth for Hermes source-specific reports. | - ---- - -## 3. Common Rules - -### Date Boundary -- Default boundary: one local day in KST (UTC+9) unless configured otherwise. -- Session attribution: `session.time_created` or source-equivalent session start time. -- Query range: local day start <= session start < next local day start. -- OpenCode: `time_created` is Unix epoch milliseconds. -- Hermes: `started_at` is Unix epoch seconds. - -### Missing Source Handling -- If a selected source DB/telemetry is missing, skip only that source report. -- Continue generating other selected source reports. -- Record the skip reason in the wrapper log or handoff. - -### Filename Rules -- OpenCode: `YYYY-MM-DD-opencode-usage.md` -- Hermes: `YYYY-MM-DD-hermes-usage.md` -- Claude Code: `YYYY-MM-DD-claude-code-usage.md` -- Daily memory synthesis: `YYYY-MM-DD-memory.md` -- Weekly synthesis: `YYYY-WNN-weekly.md` -- Monthly synthesis: `YYYY-MM-monthly.md` - -### Weekly Synthesis Method -- Do not re-query DBs. -- Read source-specific daily reports and daily memory summaries. -- Mark missing sources as `not configured` or `missing`. - ---- - -## 4. Templates - -- Source-specific daily reports are rendered by CLI commands. -- Memory synthesis summaries follow `.claude/skills/memory-report/SKILL.md`. - -**Empty-day handling**: If a configured source has no activity, write `no activity`. If the source itself is missing, do not create that report. - -## 5. Cost Calculation - -### Cost Source - -- **Recorded cost**: `SUM(message.data.cost)`. OpenCode records this across models. - -### providerID Handling - -| providerID | Cost Source | -|------------|----------| -| `anthropic` | `message.data.cost` (OpenCode recorded cost) | -| `openai` | `message.data.cost` (OpenCode recorded cost) | -| `google` | `message.data.cost` (OpenCode recorded cost) | -| `opencode-go` | `message.data.cost` (actual recorded cost) | -| `vllm` | 0 USD (self-hosted, excluded from cost aggregation) | - ---- - -## Appendix - -### A. Full Storage Layout - -``` -data/wiki/summaries/ -└── 2026/ - └── 05/ - ├── 2026-05-10-opencode-usage.md - ├── 2026-05-10-hermes-usage.md - ├── 2026-05-10-claude-code-usage.md - ├── 2026-05-10-memory.md - ├── 2026-W19-weekly.md - └── 2026-05-monthly.md -``` - -### B. Signal-To-Action References - -Use these thresholds when interpreting daily/weekly signals: -- `opencode-monitoring-guide.md` §4 -- `hermes-monitoring-guide.md` §3 diff --git a/docs/raw/ops_cron.md b/docs/raw/ops_cron.md deleted file mode 100644 index e6b4d92..0000000 --- a/docs/raw/ops_cron.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -source_url: "" -type: ops_cron -captured_at: "" -contributor: nam-young-woo -job_name: "" -schedule: "" -status: success | failed | skipped -duration_sec: 0 -tags: [] ---- - -# Cron: {{job_name}} — {{YYYY-MM-DD HH:MM}} KST - -## Result - -- **Status**: success | failed | skipped -- **Started at**: -- **Ended at**: -- **Duration**: N seconds - -## Actions - - - -## Outputs - - - -| Item | Path / Value | -|------|----------| -| | | - -## Errors And Warnings - - - -## Follow-Up - - - diff --git a/docs/workflows.md b/docs/workflows.md deleted file mode 100644 index c5149b7..0000000 --- a/docs/workflows.md +++ /dev/null @@ -1,110 +0,0 @@ -# Workflows (At-a-Glance Map) - -Updated: 2026-06-02 - -## 1. Synopsis - -- **Purpose**: Give a one-look mental model of how KnowledgeBase moves: what the - nightly automation does and how a raw source becomes an approved wiki page. -- **Scope**: Overview only. This is **not** an execution guide — every cron job's - behavior is governed by a skill. For step-level detail, read the relevant - `.claude/skills//SKILL.md` (see [§5](#5-where-the-detail-lives)). - -## 2. Legend - -The same conventions apply to every diagram below: - -| Style | Meaning | -|---|---| -| 🟦 blue box | **Deterministic** job — pure CLI (`uv run kb-*`), no LLM | -| 🟪 purple box | **LLM-driven** job — runs `opencode run` against a skill contract | -| ⬦ diamond | **Gate / decision** (e.g. lint pass?) — blocks the flow on failure | -| 👤 | **Human-run** step (not a cron job) | -| dashed arrow | read-only / report (no artefact produced or moved) | - -## 3. Nightly cron pipeline (KST) - -The whole night collapses to: *collect → organize → DB write + export*, -with a separate morning read-out to Slack. - -```mermaid -flowchart LR - classDef det fill:#e3f2fd,stroke:#1565c0,color:#0d47a1; - classDef llm fill:#f3e5f5,stroke:#6a1b9a,color:#4a148c; - classDef ext fill:#fff8e1,stroke:#f9a825,color:#5d4037,stroke-dasharray:4 3; - classDef gate fill:#ffffff,stroke:#555,color:#000; - - ttl["00:30
wiki-ttl-sweep"]:::det - subgraph R["03:10–03:20 · usage reports"] - direction TB - oc["opencode"]:::det - hm["hermes"]:::det - cc["claude-code"]:::det - end - mem["03:30
memory-daily"]:::llm - prom["04:00
wiki-promote"]:::llm - wrap["05:00
cron-wrapup"]:::llm - papers["10:05
ingest-daily-papers"]:::det - slack["09:00 · morning-slack-digest 👤
(Hermes agent — not a KB cron)"]:::ext - Slack[("Slack")] - - wk["04:15 memory-weekly
(Mon)"]:::llm - mo["04:45 memory-monthly
(1st)"]:::llm - - ttl --> R --> mem --> prom --> wrap - prom -. "Mon" .-> wk -.-> wrap - prom -. "1st" .-> mo -.-> wrap - wrap -. "reads committed wrap-up" .-> slack -.-> Slack - papers -``` - -Intuition: nothing pushes mid-night — `cron-wrapup` (05:00) writes the night's -work through the DB API as one checkpoint. `ingest-daily-papers` (10:05) and the -`morning-slack-digest` (09:00, runs with the Hermes agent) sit outside the main chain. - -## 4. Data flow & review lifecycle - -How a captured source becomes durable, indexed knowledge — and what `INDEX.md` lists. - -```mermaid -flowchart TB - classDef raw fill:#eceff1,stroke:#546e7a,color:#263238; - classDef state fill:#e8f5e9,stroke:#2e7d32,color:#1b5e20; - classDef term fill:#ffebee,stroke:#c62828,color:#b71c1c; - - src["sources
github · chat · calendar · web · manual"] - raw["data/raw/
immutable · create-only"]:::raw - auth["wiki-authoring / memory-report
write page"] - np(["not_processed"]):::state - pend(["pending_for_approve"]):::state - appr(["approved"]):::state - idx["wiki/INDEX.md
(approved pages only)"] - rej["rejected/
audit trail"]:::term - ttl["TTL sweep (7d)
→ expire"]:::term - - src --> raw --> auth --> np - np -- "wiki-promote" --> pend - pend -- "👤 approve" --> appr --> idx - pend -- "reject" --> rej - np -. "stale" .-> ttl - pend -. "stale" .-> ttl -``` - -Intuition: `data/raw/` is write-once evidence; wiki pages climb an approval ladder -(`not_processed → pending_for_approve → approved`), and only **approved** pages land -in `INDEX.md`. Stale unprocessed/pending pages are swept by TTL; explicit rejections -are kept as an audit trail. - -## 5. Where the detail lives - -This map is intentionally shallow. Each job's real contract — inputs, outputs, lint -order, edge cases — lives in its skill: - -- LLM-driven jobs load their behavior from a `SKILL.md` at runtime - (`memory-*` → `memory-report`; `wiki-promote` → `wiki-approval`; - `cron-wrapup` → `cron-wrapup`). -- Deterministic jobs are CLIs, but their setup and operating contract are still - documented in skills (e.g. usage reports → `usage-report-setup`). - -Browse `.claude/skills/` for the full set; read the matching `SKILL.md` for any box -in the diagrams above. diff --git a/pyproject.toml b/pyproject.toml index 75740b1..39ab827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,4 @@ build-backend = "hatchling.build" [project.scripts] kb-lint = "kb.cli.lint:main" -kb-opencode-daily-report = "kb.cli.opencode_daily_report:main" -kb-hermes-daily-report = "kb.cli.hermes_daily_report:main" -kb-claude-code-daily-report = "kb.cli.claude_code_daily_report:main" -kb-submit-cron-run = "kb.cli.submit_cron_run:main" -kb-db-ttl-sweep = "kb.cli.db_ttl_sweep:main" kb-mcp = "kb.mcp.server:main" diff --git a/scripts/cron/_db.sh b/scripts/cron/_db.sh deleted file mode 100644 index 21350f1..0000000 --- a/scripts/cron/_db.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# Load host environment (DATABASE_URL, KB_DATA_DIR) when present. -# Sourced by every cron wrapper after KB_ROOT is set. - -# Hermes cron may run with a minimal PATH and a profile-scoped HOME -# (for example /home/spow12/.hermes/home). Resolve the operator home from -# passwd data first so user-local tools are found consistently. -CRON_OPERATOR_USER="${LOGNAME:-${USER:-spow12}}" -CRON_OPERATOR_HOME="${CRON_OPERATOR_HOME:-$(getent passwd "$CRON_OPERATOR_USER" 2>/dev/null | cut -d: -f6)}" -CRON_OPERATOR_HOME="${CRON_OPERATOR_HOME:-/home/spow12}" - -# opencode and other user tools read config/credentials from HOME. Hermes cron -# can provide a profile-scoped HOME, so normalize it for project cron children. -export HOME="$CRON_OPERATOR_HOME" - -# Keep all wrappers able to find uv, opencode, psql, and user-local shims -# without relying on an interactive shell. -export PATH="$CRON_OPERATOR_HOME/.opencode/bin:$CRON_OPERATOR_HOME/.local/bin:$CRON_OPERATOR_HOME/anaconda3/bin:$CRON_OPERATOR_HOME/.bun/bin:$CRON_OPERATOR_HOME/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" - -# Use explicit binary paths so nested non-interactive cron shells do not depend on -# login-shell startup files. Allow overrides for fresh hosts or tests. -export UV_BIN="${UV_BIN:-$CRON_OPERATOR_HOME/anaconda3/bin/uv}" -export OPENCODE_BIN="${OPENCODE_BIN:-$CRON_OPERATOR_HOME/.opencode/bin/opencode}" -export KB_OPENCODE_MODEL="${KB_OPENCODE_MODEL:-anthropic/claude-sonnet-4-6}" - -if [[ -n "${KB_ROOT:-}" && -f "$KB_ROOT/.env" ]]; then - set -a - # shellcheck disable=SC1091 - . "$KB_ROOT/.env" - set +a -fi - -kb_finish_cron_run() { - local job_name="$1" - local target="$2" - local run_exit="$3" - local log_file="$4" - local status="success" - local submit_exit=0 - local log_path="" - local args=() - - if [[ "$run_exit" -ne 0 ]]; then - status="failed" - fi - - if [[ "$log_file" == "$KB_ROOT/data/"* ]]; then - log_path="${log_file#"$KB_ROOT/data/"}" - fi - - args=( - --job-name "$job_name" - --target "$target" - --status "$status" - --exit-code "$run_exit" - --log-file "$log_file" - ) - if [[ -n "$log_path" ]]; then - args+=(--log-path "$log_path") - fi - - ( - cd "$KB_ROOT" - env -u VIRTUAL_ENV "$UV_BIN" run kb-submit-cron-run "${args[@]}" - ) || submit_exit=$? - - if [[ "$run_exit" -ne 0 ]]; then - return "$run_exit" - fi - return "$submit_exit" -} diff --git a/scripts/cron/kb-claude-code-daily-report.sh b/scripts/cron/kb-claude-code-daily-report.sh deleted file mode 100755 index edf17c7..0000000 --- a/scripts/cron/kb-claude-code-daily-report.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-claude-code-daily-report.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/claude-code-report.lock" \ - bash -c ' - cd "$1" - env -u VIRTUAL_ENV "$UV_BIN" run kb-claude-code-daily-report --date "$2" --lint - ' bash "$KB_ROOT" "$TARGET_DATE" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -if [[ "$RUN_EXIT" -ne 0 ]]; then - echo "[$(date -Iseconds)] ERROR: kb-claude-code-daily-report failed for $TARGET_DATE" >&2 -fi -kb_finish_cron_run "kb-claude-code-daily-report" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-cron-wrapup.sh b/scripts/cron/kb-cron-wrapup.sh deleted file mode 100755 index 73f941b..0000000 --- a/scripts/cron/kb-cron-wrapup.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="${KB_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/../.." && pwd)}" -source "$SCRIPT_DIR/_db.sh" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -INFLIGHT_LOG_DIR="$KB_ROOT/.cron/logs" -INFLIGHT_LOG="$INFLIGHT_LOG_DIR/cron-wrapup.log" -ARCHIVE_LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOCK_DIR="$KB_ROOT/.cron/locks" - -mkdir -p "$INFLIGHT_LOG_DIR" "$LOCK_DIR" - -PROMPT="Run the KB cron wrap-up for $TARGET_DATE. Import and follow .claude/skills/cron-wrapup/SKILL.md as the runtime contract. Import .claude/skills/handoff-document/SKILL.md for the run handoff. Do not read docs as runtime instructions. Write all durable state through the kb-mcp tools. Do not run git, do not commit data/, and do not push. If blocked, write the wrap-up with Status: FAILED and a handoff with status: ready, then exit non-zero." - -SESSION_EXIT=0 -ARCHIVE_LOG="$ARCHIVE_LOG_DIR/${TARGET_DATE}_kb-cron-wrapup.log" -flock -n "$LOCK_DIR/cron-wrapup.lock" bash -c ' - set -uo pipefail - KB_ROOT="$1"; PROMPT="$2"; TARGET_DATE="$3"; INFLIGHT_LOG="$4" - ARCHIVE_LOG_DIR="$5"; ARCHIVE_LOG="$6" - rc=0 - cd "$KB_ROOT" - "$OPENCODE_BIN" run --model "$KB_OPENCODE_MODEL" --dangerously-skip-permissions \ - --dir "$KB_ROOT" "$PROMPT" >> "$INFLIGHT_LOG" 2>&1 || rc=$? - - # Archive the completed run log as generated export evidence; DB is canonical. - archive_rc=0 - mkdir -p "$ARCHIVE_LOG_DIR" || archive_rc=$? - [ "$archive_rc" -ne 0 ] \ - || cp "$INFLIGHT_LOG" "$ARCHIVE_LOG" || archive_rc=$? - [ "$archive_rc" -eq 0 ] \ - || echo "WARN: failed to archive run log to $ARCHIVE_LOG_DIR (rc=$archive_rc)" >> "$INFLIGHT_LOG" - - [ "$rc" -ne 0 ] || [ "$archive_rc" -eq 0 ] || rc=$archive_rc - exit $rc -' bash "$KB_ROOT" "$PROMPT" "$TARGET_DATE" "$INFLIGHT_LOG" "$ARCHIVE_LOG_DIR" "$ARCHIVE_LOG" \ - || SESSION_EXIT=$? - -if [[ ! -f "$ARCHIVE_LOG" ]]; then - ARCHIVE_LOG="$INFLIGHT_LOG" -fi -kb_finish_cron_run "kb-cron-wrapup" "$TARGET_DATE" "$SESSION_EXIT" "$ARCHIVE_LOG" -exit $? diff --git a/scripts/cron/kb-hermes-daily-report.sh b/scripts/cron/kb-hermes-daily-report.sh deleted file mode 100755 index 3ee762f..0000000 --- a/scripts/cron/kb-hermes-daily-report.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-hermes-daily-report.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/hermes-report.lock" \ - bash -c ' - cd "$1" - env -u VIRTUAL_ENV "$UV_BIN" run kb-hermes-daily-report --date "$2" --lint - ' bash "$KB_ROOT" "$TARGET_DATE" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -if [[ "$RUN_EXIT" -ne 0 ]]; then - echo "[$(date -Iseconds)] ERROR: kb-hermes-daily-report failed for $TARGET_DATE" >&2 -fi -kb_finish_cron_run "kb-hermes-daily-report" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-ingest-papers.sh b/scripts/cron/kb-ingest-papers.sh deleted file mode 100755 index a2b887e..0000000 --- a/scripts/cron/kb-ingest-papers.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-ingest-papers.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/ingest-papers.lock" \ - bash -c ' - cd "$1" - env -u VIRTUAL_ENV "$UV_BIN" run python scripts/ingest-daily-papers.py - ' bash "$KB_ROOT" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -if [[ "$RUN_EXIT" -ne 0 ]]; then - echo "[$(date -Iseconds)] ERROR: kb-ingest-papers failed for $TARGET_DATE" >&2 -fi -kb_finish_cron_run "kb-ingest-papers" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-memory-daily.sh b/scripts/cron/kb-memory-daily.sh deleted file mode 100755 index d29dac1..0000000 --- a/scripts/cron/kb-memory-daily.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-memory-daily.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/daily.lock" bash -c " - cd '$KB_ROOT' - "$OPENCODE_BIN" run \ - --model "$KB_OPENCODE_MODEL" \ - --dangerously-skip-permissions \ - --dir '$KB_ROOT' \ - 'Run the daily memory workflow for $TARGET_DATE. Import and follow .claude/skills/memory-report/SKILL.md §Daily as the runtime contract. Import .claude/skills/wiki-authoring/SKILL.md for any wiki page edits and .claude/skills/handoff-document/SKILL.md for the handoff. Do not read docs as runtime instructions. Read canonical state from Postgres and write durable state through the kb-mcp tools only. Do not run git or edit data/ as source of truth. If blocked, write a handoff with status: ready and submit an operation log through the kb-mcp create_operation_log tool before exiting.' -" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -kb_finish_cron_run "kb-memory-daily" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-memory-monthly.sh b/scripts/cron/kb-memory-monthly.sh deleted file mode 100755 index 757d329..0000000 --- a/scripts/cron/kb-memory-monthly.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_MONTH="$(TZ=Asia/Seoul date -d 'last month' +%Y-%m)" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-memory-monthly.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/monthly.lock" bash -c " - cd '$KB_ROOT' - "$OPENCODE_BIN" run \ - --model "$KB_OPENCODE_MODEL" \ - --dangerously-skip-permissions \ - --dir '$KB_ROOT' \ - 'Run the monthly memory workflow for $TARGET_MONTH. Import and follow .claude/skills/memory-report/SKILL.md §Monthly as the runtime contract. Import .claude/skills/wiki-authoring/SKILL.md for wiki cleanup/page edits and .claude/skills/handoff-document/SKILL.md for the handoff. Do not read docs as runtime instructions. Read canonical state from Postgres and write durable state through the kb-mcp tools only. Do not run git or edit data/ as source of truth. If blocked, write a handoff with status: ready and submit an operation log through the kb-mcp create_operation_log tool before exiting.' -" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -kb_finish_cron_run "kb-memory-monthly" "$TARGET_MONTH" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-memory-weekly.sh b/scripts/cron/kb-memory-weekly.sh deleted file mode 100755 index bf77220..0000000 --- a/scripts/cron/kb-memory-weekly.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_WEEK="$(TZ=Asia/Seoul date -d 'last week' +%G-W%V)" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-memory-weekly.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/weekly.lock" bash -c " - cd '$KB_ROOT' - "$OPENCODE_BIN" run \ - --model "$KB_OPENCODE_MODEL" \ - --dangerously-skip-permissions \ - --dir '$KB_ROOT' \ - 'Run the weekly memory workflow for $TARGET_WEEK. Import and follow .claude/skills/memory-report/SKILL.md §Weekly as the runtime contract. Import .claude/skills/wiki-authoring/SKILL.md for any wiki page edits and .claude/skills/handoff-document/SKILL.md for the handoff. Do not read docs as runtime instructions. Read canonical state from Postgres and write durable state through the kb-mcp tools only. Do not run git or edit data/ as source of truth. If blocked, write a handoff with status: ready and submit an operation log through the kb-mcp create_operation_log tool before exiting.' -" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -kb_finish_cron_run "kb-memory-weekly" "$TARGET_WEEK" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-opencode-daily-report.sh b/scripts/cron/kb-opencode-daily-report.sh deleted file mode 100755 index 6f311f4..0000000 --- a/scripts/cron/kb-opencode-daily-report.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-opencode-daily-report.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/opencode-report.lock" \ - bash -c ' - cd "$1" - env -u VIRTUAL_ENV "$UV_BIN" run kb-opencode-daily-report --date "$2" --lint - ' bash "$KB_ROOT" "$TARGET_DATE" >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -if [[ "$RUN_EXIT" -ne 0 ]]; then - echo "[$(date -Iseconds)] ERROR: kb-opencode-daily-report failed for $TARGET_DATE" >&2 -fi -kb_finish_cron_run "kb-opencode-daily-report" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-wiki-promote.sh b/scripts/cron/kb-wiki-promote.sh deleted file mode 100755 index e5dc969..0000000 --- a/scripts/cron/kb-wiki-promote.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-wiki-promote.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -{ - echo "[$(TZ=Asia/Seoul date --iso-8601=seconds)] kb-wiki-promote start target=$TARGET_DATE" - - set +e - flock -n "$LOCK_DIR/wiki-promote.lock" bash -c " - cd '$KB_ROOT' - env -u VIRTUAL_ENV timeout --kill-after=30s 540s "$OPENCODE_BIN" run \ - --model "$KB_OPENCODE_MODEL" \ - --dangerously-skip-permissions \ - --dir '$KB_ROOT' \ - 'Run the KnowledgeBase wiki promotion workflow for $TARGET_DATE. Import and follow .claude/skills/wiki-approval/SKILL.md as the runtime contract. Import .claude/skills/wiki-authoring/SKILL.md only if page metadata/content fixes are needed before promotion. Do not read docs as runtime instructions. Promote worthy not_processed pages through the kb-mcp tools, leave borderline pages for TTL, write the wiki-promote handoff and operation log through the kb-mcp tools. Do not run git, commit data/, or push.' - " - status=$? - set -e - - if [[ $status -ne 0 ]]; then - echo "[$(TZ=Asia/Seoul date --iso-8601=seconds)] kb-wiki-promote failed status=$status" - if [[ $status -eq 1 ]]; then - echo "lock busy: another wiki-promote process is still holding $LOCK_DIR/wiki-promote.lock" - fuser -v "$LOCK_DIR/wiki-promote.lock" || true - elif [[ $status -eq 124 ]]; then - echo "opencode timed out after 540s; timeout should have terminated the child before Hermes cron's 600s script limit" - fi - RUN_EXIT="$status" - else - echo "[$(TZ=Asia/Seoul date --iso-8601=seconds)] kb-wiki-promote complete target=$TARGET_DATE" - fi -} >> "$LOG_FILE" 2>&1 -kb_finish_cron_run "kb-wiki-promote" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/scripts/cron/kb-wiki-ttl-sweep.sh b/scripts/cron/kb-wiki-ttl-sweep.sh deleted file mode 100755 index f63ce6a..0000000 --- a/scripts/cron/kb-wiki-ttl-sweep.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KB_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -source "$SCRIPT_DIR/_db.sh" -LOCK_DIR="$KB_ROOT/.cron/locks" -TARGET_DATE="$(TZ=Asia/Seoul date -d 'yesterday' +%F)" -LOG_DIR="$KB_ROOT/data/raw/ops/cron/$(TZ=Asia/Seoul date -d "$TARGET_DATE" +%Y/%m)" -LOG_FILE="$LOG_DIR/${TARGET_DATE}_kb-wiki-ttl-sweep.log" - -mkdir -p "$LOG_DIR" "$LOCK_DIR" - -RUN_EXIT=0 -flock -n "$LOCK_DIR/wiki-ttl-sweep.lock" \ - bash -c 'cd "$1" && env -u VIRTUAL_ENV "$UV_BIN" run kb-db-ttl-sweep --days 7' bash "$KB_ROOT" \ - >> "$LOG_FILE" 2>&1 || RUN_EXIT=$? -if [[ "$RUN_EXIT" -ne 0 ]]; then - echo "[$(date -Iseconds)] ERROR: kb-wiki-ttl-sweep failed" >&2 -fi -kb_finish_cron_run "kb-wiki-ttl-sweep" "$TARGET_DATE" "$RUN_EXIT" "$LOG_FILE" -exit $? diff --git a/src/kb/cli/_payloads.py b/src/kb/cli/_payloads.py deleted file mode 100644 index a4275ca..0000000 --- a/src/kb/cli/_payloads.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Pure payload parsers for KB CLI consumers. - -These functions parse Markdown (with YAML frontmatter) into structured dicts -suitable for passing to service-layer functions. They have no I/O side effects. - -Extracted from the now-removed ``kb.cli.db_api`` so that in-process consumers -can import them without pulling in an HTTP client. -""" - -from __future__ import annotations - -from typing import Any - -import yaml - - -class PayloadError(ValueError): - """Raised when a Markdown document cannot be parsed into a valid payload.""" - - -def _split_frontmatter(markdown: str) -> tuple[dict[str, Any], str]: - if not markdown.startswith("---\n"): - raise PayloadError("markdown must start with YAML frontmatter") - try: - _, fm_text, body = markdown.split("---", 2) - except ValueError as exc: - raise PayloadError("markdown frontmatter closing delimiter is missing") from exc - frontmatter = yaml.safe_load(fm_text) or {} - if not isinstance(frontmatter, dict): - raise PayloadError("markdown frontmatter must be a mapping") - return frontmatter, body - - -def _first_heading(body: str, fallback: str) -> str: - for line in body.splitlines(): - if line.startswith("# "): - return line[2:].strip() or fallback - return fallback - - -def markdown_page_payload( - *, - markdown: str, - export_path: str, - slug: str, - origin: str, - source: str, -) -> dict[str, Any]: - """Parse a Markdown page into a dict suitable for ``service.pages.upsert_page``. - - Returns keys: slug, type, title, category, review_status, origin, - frontmatter, body_md, export_path, source. - """ - frontmatter, body = _split_frontmatter(markdown) - page_type = frontmatter.get("type") - if not isinstance(page_type, str) or not page_type: - raise PayloadError("page frontmatter requires non-empty type") - return { - "slug": slug, - "type": page_type, - "title": frontmatter.get("title"), - "category": frontmatter.get("category"), - "review_status": frontmatter.get("review_status"), - "origin": origin, - "frontmatter": frontmatter, - "body_md": body, - "export_path": export_path, - "source": source, - } - - -def raw_source_payload( - *, - markdown: str, - source_key: str, - source_type: str | None = None, -) -> dict[str, Any]: - """Parse a raw-source Markdown document into a dict for ``service.sources.create_raw_source``. - - Returns keys: source_key, source_type, source_url, title, captured_at, - frontmatter, content_md. - """ - frontmatter, body = _split_frontmatter(markdown) - raw_type = source_type or frontmatter.get("type") or "raw" - return { - "source_key": source_key, - "source_type": raw_type, - "source_url": frontmatter.get("source_url"), - "title": _first_heading(body, source_key), - "captured_at": frontmatter.get("captured_at"), - "frontmatter": frontmatter, - "content_md": body, - } diff --git a/src/kb/cli/_submit.py b/src/kb/cli/_submit.py deleted file mode 100644 index 53a946e..0000000 --- a/src/kb/cli/_submit.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Shared in-process submit helper for the daily report CLIs. - -Replaces the HTTP ``submit_markdown_page`` + ``submit_metrics`` pair with a -single in-process transaction via the service layer. -""" - -from __future__ import annotations - -from typing import Any - -from kb.cli._payloads import markdown_page_payload -from kb.service import ops as service_ops -from kb.service import pages as service_pages -from kb.service.session import session_scope - - -def submit_page_and_metrics( - *, - report: str, - export_path: str, - slug: str, - report_date: str, - report_type: str, - metrics: dict[str, Any], - **metric_fields: Any, -) -> None: - """Upsert the rendered report page + its metrics in one session scope. - - Two internal commits share the session (the page is committed before the - metrics), so this is not a single atomic transaction. - """ - payload = markdown_page_payload( - markdown=report, - export_path=export_path, - slug=slug, - origin="ingested", - source="cli", - ) - with session_scope() as (session, data_dir): - service_pages.upsert_page(session, data_dir, **payload) - service_ops.upsert_metrics( - session, - data_dir, - report_date=report_date, - report_type=report_type, - metrics_json=metrics, - **metric_fields, - ) diff --git a/src/kb/cli/claude_code_daily_report.py b/src/kb/cli/claude_code_daily_report.py deleted file mode 100644 index 716ae53..0000000 --- a/src/kb/cli/claude_code_daily_report.py +++ /dev/null @@ -1,673 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic Claude Code daily usage report generator.""" - -from __future__ import annotations - -import argparse -import json -import urllib.parse -import urllib.request -from collections import defaultdict -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any, Callable - -from kb import REPO_ROOT as BASEDIR -from kb.cli._submit import submit_page_and_metrics -from kb.cli.usage_reports.render import _fmt, _int, _num, _pct - -DEFAULT_PROM = "http://127.0.0.1:9090" -DEFAULT_LOKI = "http://127.0.0.1:3110" -KST = timezone(timedelta(hours=9)) -DEFERRED_METRICS = ["Task Completion Rate", "pass@k", "pass^k"] - - -def _kst_day_bounds(date_str: str) -> tuple[float, float]: - d = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=KST) - return d.timestamp(), (d + timedelta(days=1)).timestamp() - - -def _query_prom(base: str, expr: str, ts: float) -> list[dict[str, Any]]: - params = urllib.parse.urlencode({"query": expr, "time": f"{ts:.3f}"}) - req = urllib.request.Request(f"{base}/api/v1/query?{params}") - with urllib.request.urlopen(req, timeout=15) as resp: - body = json.load(resp) - return ( - body.get("data", {}).get("result", []) - if body.get("status") == "success" - else [] - ) - - -def _query_loki( - base: str, - expr: str, - start_ns: int, - end_ns: int, - limit: int = 5000, - step: str | None = None, -) -> list[dict[str, Any]]: - payload: dict[str, str] = { - "query": expr, - "start": str(start_ns), - "end": str(end_ns), - "limit": str(limit), - "direction": "forward", - } - if step: - payload["step"] = step - params = urllib.parse.urlencode(payload) - req = urllib.request.Request(f"{base}/loki/api/v1/query_range?{params}") - with urllib.request.urlopen(req, timeout=60) as resp: - body = json.load(resp) - return ( - body.get("data", {}).get("result", []) - if body.get("status") == "success" - else [] - ) - - -def _query_loki_instant( - base: str, - expr: str, - ts_ns: int, - limit: int = 5000, -) -> list[dict[str, Any]]: - """Run a Loki instant query at a single timestamp. - - Use for aggregations like `count_over_time({...}[1d])` that must be - evaluated once at the window end — query_range with default step - creates overlapping sliding windows that inflate counts. - """ - params = urllib.parse.urlencode( - {"query": expr, "time": str(ts_ns), "limit": str(limit)} - ) - req = urllib.request.Request(f"{base}/loki/api/v1/query?{params}") - with urllib.request.urlopen(req, timeout=60) as resp: - body = json.load(resp) - return ( - body.get("data", {}).get("result", []) - if body.get("status") == "success" - else [] - ) - - -def _vec_sum( - vec: list[dict[str, Any]], - key_fn: Callable[[dict[str, str]], Any] = lambda m: "_all", -) -> dict[Any, float]: - out: dict[Any, float] = defaultdict(float) - for r in vec: - out[key_fn(r["metric"])] += float(r["value"][1]) - return dict(out) - - -def _collect_prometheus(target_date: str, prom: str) -> dict[str, Any]: - start_ts, end_ts = _kst_day_bounds(target_date) - - def inc(metric: str, grouping: str | None = None) -> str: - body = f"increase({metric}[24h] @ {end_ts:.0f})" - if grouping: - body = f"sum by ({grouping}) ({body})" - return body - - token_vec = _query_prom( - prom, - inc("claude_code_token_usage_tokens_total", "user_email,model,type"), - end_ts, - ) - cost_vec = _query_prom( - prom, inc("claude_code_cost_usage_USD_total", "user_email,model"), end_ts - ) - # session_count is emitted once per session-start, so increase() yields 0. - # Counting unique series under the day window via a 24h-bracketed selector is more reliable. - session_series = _query_prom( - prom, - f"count(count by (session_id) (last_over_time(claude_code_session_count_total[24h] @ {end_ts:.0f})))", - end_ts, - ) - sessions = float(session_series[0]["value"][1]) if session_series else 0.0 - active = _vec_sum( - _query_prom(prom, inc("claude_code_active_time_seconds_total", "type"), end_ts), - lambda m: m.get("type", "?"), - ) - lines = _vec_sum( - _query_prom(prom, inc("claude_code_lines_of_code_count_total", "type"), end_ts), - lambda m: m.get("type", "?"), - ) - decisions = _vec_sum( - _query_prom( - prom, inc("claude_code_code_edit_tool_decision_total", "decision"), end_ts - ), - lambda m: m.get("decision", "?"), - ) - qsrc = _vec_sum( - _query_prom( - prom, inc("claude_code_token_usage_tokens_total", "query_source"), end_ts - ), - lambda m: m.get("query_source", "?"), - ) - - # Pivot tokens to (user_email, model) -> {type: count} - tokens_by_pair: dict[tuple[str, str], dict[str, float]] = defaultdict( - lambda: defaultdict(float) - ) - for r in token_vec: - m = r["metric"] - pair = (m.get("user_email", "?"), m.get("model", "?")) - tokens_by_pair[pair][m.get("type", "?")] += float(r["value"][1]) - - cost_by_pair: dict[tuple[str, str], float] = defaultdict(float) - for r in cost_vec: - m = r["metric"] - cost_by_pair[(m.get("user_email", "?"), m.get("model", "?"))] += float( - r["value"][1] - ) - - model_usage = [] - totals = defaultdict(float) - for pair, type_map in sorted( - tokens_by_pair.items(), key=lambda kv: -sum(kv[1].values()) - ): - user_email, model = pair - input_t = type_map.get("input", 0.0) - output = type_map.get("output", 0.0) - cache_read = type_map.get("cacheRead", 0.0) - cache_write = type_map.get("cacheCreation", 0.0) - total_input = input_t + cache_read - total = input_t + output + cache_read + cache_write - cost = cost_by_pair.get(pair, 0.0) - model_usage.append( - { - "user_email": user_email, - "model": model, - "input": int(total_input), - "cache_miss": int(input_t), - "cache_read": int(cache_read), - "output": int(output), - "cache_write": int(cache_write), - "total": int(total), - "cost": round(cost, 6), - "cache_hit_pct": _pct_or_none(cache_read, total_input), - } - ) - totals["input"] += input_t - totals["output"] += output - totals["cache_read"] += cache_read - totals["cache_write"] += cache_write - totals["cost"] += cost - totals["total"] += total - - total_input_all = totals["input"] + totals["cache_read"] - cache_hit_pct = _pct_or_none(totals["cache_read"], total_input_all) - - return { - "available": bool(token_vec or cost_vec), - "sessions": {"total": int(sessions)}, - "tokens": { - "input": int(totals["input"] + totals["cache_read"]), - "cache_miss": int(totals["input"]), - "cache_read": int(totals["cache_read"]), - "output": int(totals["output"]), - "cache_write": int(totals["cache_write"]), - "total": int(totals["total"]), - "cache_hit_pct": cache_hit_pct, - }, - "cost": { - "recorded_usd": round(totals["cost"], 6), - "cost_per_session_usd": ( - round(totals["cost"] / sessions, 6) if sessions else None - ), - }, - "model_usage": model_usage, - "active_time_seconds": {k: round(v, 2) for k, v in active.items()}, - "lines_of_code": {k: int(v) for k, v in lines.items()}, - "code_edit_decisions": {k: int(v) for k, v in decisions.items()}, - "query_source": {k: int(v) for k, v in qsrc.items()}, - } - - -def _pct_or_none(numerator: float, denominator: float) -> float | None: - if denominator <= 0: - return None - return round(numerator * 100 / denominator, 2) - - -def _collect_loki(target_date: str, loki: str) -> dict[str, Any]: - start_ts, end_ts = _kst_day_bounds(target_date) - start_ns, end_ns = int(start_ts * 1e9), int(end_ts * 1e9) - - # event_name / tool_name / success are structured metadata (not stream labels), - # so they need pipe filters and structured-metadata-aware grouping. - # Use an instant query at end_ns so the [24h] window is evaluated once; - # query_range would emit overlapping sliding windows and the sum across - # them multiplies each event by ~step_count, inflating counts ~10-100x. - tool_streams = _query_loki_instant( - loki, - 'sum by (tool_name, success) (count_over_time({service_name="claude-code"} | event_name="tool_result" [24h]))', - end_ns, - limit=5000, - ) - tool_breakdown: dict[str, dict[str, int]] = defaultdict( - lambda: {"calls": 0, "errors": 0} - ) - for stream in tool_streams: - labels = stream.get("metric") or stream.get("stream") or {} - tool = labels.get("tool_name", "unknown") - success = labels.get("success", "true") - value = stream.get("value") - count = int(float(value[1])) if value else 0 - tool_breakdown[tool]["calls"] += count - if success != "true": - tool_breakdown[tool]["errors"] += count - tool_rows = sorted( - ( - {"tool": t, **v, "error_rate_pct": _pct_or_none(v["errors"], v["calls"])} - for t, v in tool_breakdown.items() - ), - key=lambda r: (-r["calls"], r["tool"]), - ) - total_calls = sum(v["calls"] for v in tool_breakdown.values()) - total_errors = sum(v["errors"] for v in tool_breakdown.values()) - - hourly_streams = _query_loki( - loki, - 'sum by (session_id) (count_over_time({service_name="claude-code"}[1h]))', - start_ns, - end_ns, - limit=5000, - step="1h", - ) - hourly: dict[str, set[str]] = defaultdict(set) - sessions_seen: set[str] = set() - for stream in hourly_streams: - labels = stream.get("metric") or stream.get("stream") or {} - sid = labels.get("session_id", "?") - sessions_seen.add(sid) - for ts, val in stream.get("values", []): - # Loki matrix response: timestamp is float seconds, value is a string count. - if float(val) <= 0: - continue - hour_kst = datetime.fromtimestamp(float(ts), tz=KST).strftime("%H") - hourly[hour_kst].add(sid) - hourly_rows = [ - {"hour": h, "sessions": len(sids)} for h, sids in sorted(hourly.items()) - ] - - duration_streams = _query_loki( - loki, - 'avg by (tool_name) (avg_over_time({service_name="claude-code"} | event_name="tool_result" | unwrap duration_ms [24h]))', - start_ns, - end_ns, - limit=2000, - ) - latency_by_tool = [] - for stream in duration_streams: - labels = stream.get("metric") or stream.get("stream") or {} - vals = [ - float(v[1]) - for v in stream.get("values", []) - if v[1] not in ("NaN", "+Inf", "-Inf") - ] - if not vals: - continue - latency_by_tool.append( - { - "tool": labels.get("tool_name", "?"), - "avg_ms": round(sum(vals) / len(vals), 2), - "max_ms": round(max(vals), 2), - } - ) - latency_by_tool.sort(key=lambda r: -r["avg_ms"]) - - # Count distinct sessions per terminal_type, not raw entry counts. - terminal_streams = _query_loki( - loki, - 'sum by (terminal_type, session_id) (count_over_time({service_name="claude-code"}[24h]))', - start_ns, - end_ns, - limit=5000, - ) - terminal_sessions: dict[str, set[str]] = defaultdict(set) - for s in terminal_streams: - labels = s.get("metric") or s.get("stream") or {} - terminal_sessions[labels.get("terminal_type", "?")].add( - labels.get("session_id", "?") - ) - terminal = {k: len(v) for k, v in terminal_sessions.items()} - - # Instant query at end_ns — same rationale as tool_streams above. - prompts = _query_loki_instant( - loki, - 'sum by (session_id) (count_over_time({service_name="claude-code"} | event_name="user_prompt" [24h]))', - end_ns, - limit=5000, - ) - n_turns = 0 - for s in prompts: - value = s.get("value") - if value: - n_turns += int(float(value[1])) - - return { - "available": bool(tool_streams or hourly_streams), - "sessions": {"loki_distinct": len(sessions_seen)}, - "n_turns": n_turns, - "n_toolcalls": total_calls, - "error_rate": { - "tool_calls": total_calls, - "tool_errors": total_errors, - "rate_pct": _pct_or_none(total_errors, total_calls), - }, - "tool_breakdown": tool_rows, - "tool_latency": latency_by_tool[:10], - "hourly_sessions": hourly_rows, - "terminal_distribution": terminal, - } - - -def collect_claude_code_metrics( - target_date: str, prom: str = DEFAULT_PROM, loki: str = DEFAULT_LOKI -) -> dict[str, Any]: - p = _collect_prometheus(target_date, prom) - lk = _collect_loki(target_date, loki) - return { - "date": target_date, - "generated_at": datetime.now(timezone.utc).isoformat(), - "deferred_metrics": DEFERRED_METRICS, - "claude_code": { - **p, - **{k: v for k, v in lk.items() if k != "sessions"}, - "sessions": {**p.get("sessions", {}), **lk.get("sessions", {})}, - }, - "policy_compliance": { - "passed": 0, - "total": 0, - "rate_pct": None, - "status": "not_evaluated_until_write", - }, - } - - -def _model_table(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Model usage: N/A" - out = [ - "| user | model | total | total_input | cache_miss | cache_read | output | cache_write | cache_hit | cost USD |", - "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|", - ] - for r in rows: - out.append( - f"| {r.get('user_email')} | {r.get('model')} | {_int(r.get('total'))} | {_int(r.get('input'))} | " - f"{_int(r.get('cache_miss'))} | {_int(r.get('cache_read'))} | {_int(r.get('output'))} | " - f"{_int(r.get('cache_write'))} | {_pct(r.get('cache_hit_pct'))} | {_fmt(r.get('cost'))} |" - ) - return "\n".join(out) - - -def _tool_table(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Tool usage: N/A" - out = ["| tool | calls | errors | error_rate |", "|---|---:|---:|---:|"] - for r in rows: - out.append( - f"| {r.get('tool')} | {_int(r.get('calls'))} | {_int(r.get('errors'))} | {_pct(r.get('error_rate_pct'))} |" - ) - return "\n".join(out) - - -def _latency_table(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Average tool duration: N/A" - out = ["| tool | avg_ms | max_ms |", "|---|---:|---:|"] - for r in rows: - out.append( - f"| {r.get('tool')} | {_fmt(r.get('avg_ms'))} | {_fmt(r.get('max_ms'))} |" - ) - return "\n".join(out) - - -def _hourly_lines(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Hourly distribution: N/A" - return "\n".join( - f" - {r.get('hour')}: {_fmt(r.get('sessions'))} sessions" for r in rows - ) - - -def _dict_lines(d: dict[str, Any], suffix: str = "") -> str: - if not d: - return " - N/A" - return "\n".join( - f" - {k}: {_fmt(v)}{suffix}" - for k, v in sorted(d.items(), key=lambda kv: -_num(kv[1])) - ) - - -def _observations(metrics: dict[str, Any]) -> list[str]: - cc = metrics["claude_code"] - obs: list[str] = [] - tokens = cc.get("tokens", {}) - hit = tokens.get("cache_hit_pct") - if hit is not None and hit >= 90: - obs.append( - f"Cache hit rate is {_pct(hit)}; repeated-context work is likely reducing cost. Monitor cache_read share changes." - ) - elif hit is not None and hit < 50 and tokens.get("total", 0) > 0: - obs.append( - f"Cache hit rate is low at {_pct(hit)}; many new sessions or frequent context invalidation may be reducing cost efficiency." - ) - err = cc.get("error_rate", {}) - if err.get("rate_pct") is not None and err.get("rate_pct") > 5: - obs.append( - f"Overall tool error rate is {_pct(err.get('rate_pct'))}; review precheck and tool-call patterns." - ) - decisions = cc.get("code_edit_decisions", {}) - accept = _num(decisions.get("accept")) - reject = _num(decisions.get("reject")) - if accept + reject > 0: - accept_rate = _pct_or_none(accept, accept + reject) - obs.append( - f"Code edit accept rate is {_pct(accept_rate)} ({int(accept)}/{int(accept + reject)}); low rates may signal prompt or tool-result quality issues." - ) - hourly = cc.get("hourly_sessions", []) - if hourly: - peak = max(hourly, key=lambda r: _num(r.get("sessions"))) - obs.append( - f"Usage peak at hour {peak.get('hour')} with {_fmt(peak.get('sessions'))} sessions; useful for automation and focus-pattern tracking." - ) - qsrc = cc.get("query_source", {}) - aux = _num(qsrc.get("auxiliary")) - main = _num(qsrc.get("main")) - if main + aux > 0: - aux_pct = _pct_or_none(aux, main + aux) - obs.append( - f"Auxiliary query share is {_pct(aux_pct)}; this is a rough proxy for sub-agent/background workload." - ) - if not obs: - obs.append("No notable signals. Current values can be used as baseline.") - return obs - - -def render_daily_report(metrics: dict[str, Any]) -> str: - d = metrics["date"] - cc = metrics["claude_code"] - pc = metrics["policy_compliance"] - sessions = cc.get("sessions", {}) - sessions.get("total") or sessions.get("loki_distinct") or 0 - tokens = cc.get("tokens", {}) - cost = cc.get("cost", {}) - err = cc.get("error_rate", {}) - decisions = cc.get("code_edit_decisions", {}) - lines = cc.get("lines_of_code", {}) - active = cc.get("active_time_seconds", {}) - active_total = sum(_num(v) for v in active.values()) - observations = "\n".join(f"- {line}" for line in _observations(metrics)) - - return f"""--- -type: summary -subtype: daily -date: \"{d}\" -created: \"{d}\" -updated: \"{d}\" -sources: [] -tags: [agent-usage, deterministic-report, claude-code] ---- - -# Claude Code Daily Report — {d} - -## Summary - -- Sessions: counter {_fmt(sessions.get('total'))} / Loki distinct {_fmt(sessions.get('loki_distinct'))} -- Turns (user_prompt): {_int(cc.get('n_turns'))} -- Tool calls: {_int(cc.get('n_toolcalls'))} (errors {_int(err.get('tool_errors'))}, rate {_pct(err.get('rate_pct'))}) -- Total tokens: {_int(tokens.get('total'))} (input {_int(tokens.get('input'))} / output {_int(tokens.get('output'))} / cache_read {_int(tokens.get('cache_read'))} / cache_write {_int(tokens.get('cache_write'))}) -- Recorded cost: {_fmt(cost.get('recorded_usd'))} USD -- Cache hit rate: {_pct(tokens.get('cache_hit_pct'))} -- Code edit decisions: accept {_int(decisions.get('accept'))} / reject {_int(decisions.get('reject'))} -- Lines of code: added {_int(lines.get('added'))} / removed {_int(lines.get('removed'))} -- Active time: {_fmt(round(active_total, 1))}s ({_fmt(round(active_total / 60, 1))}m) - -## Development/Evaluation Metrics - -| Metric | Claude Code | Status | -|---|---:|---| -| Task Completion Rate | code_edit accept rate {_pct(_pct_or_none(_num(decisions.get('accept')), _num(decisions.get('accept')) + _num(decisions.get('reject'))))} proxy | true task schema needed | -| pass@k / pass^k | deferred | eval-run schema needed | -| Error Rate | {_int(err.get('tool_errors'))}/{_int(err.get('tool_calls'))} ({_pct(err.get('rate_pct'))}) | deterministic | -| Hallucinated Parameters | N/A | Claude Code tools are validated client-side | -| n_toolcalls / n_turns | {_int(cc.get('n_toolcalls'))} / {_int(cc.get('n_turns'))} | deterministic | -| Total Token Usage / Cost | {_int(tokens.get('total'))} tok / {_fmt(cost.get('recorded_usd'))} USD ({_fmt(cost.get('cost_per_session_usd'))} USD/session) | deterministic | -| Policy Compliance Rate | {_pct(pc.get('rate_pct'))} | {pc.get('status')} | - -## Layer 1: Cost and Efficiency - -- Total tokens: {_int(tokens.get('total'))} (input {_int(tokens.get('input'))} / output {_int(tokens.get('output'))} / cache_read {_int(tokens.get('cache_read'))} / cache_write {_int(tokens.get('cache_write'))}) -- Cache hit rate: {_pct(tokens.get('cache_hit_pct'))} (cache_read / (cache_miss + cache_read)) -- Recorded cost: {_fmt(cost.get('recorded_usd'))} USD ({_fmt(cost.get('cost_per_session_usd'))} USD/session) -- User/model usage: - -{_model_table(cc.get('model_usage', []))} - -## Layer 2: Work Quality - -- Tool calls: {_int(cc.get('n_toolcalls'))}, errors: {_int(err.get('tool_errors'))} ({_pct(err.get('rate_pct'))}) -- Tool usage: - -{_tool_table(cc.get('tool_breakdown', []))} - -- Code edit decisions: accept {_int(decisions.get('accept'))} / reject {_int(decisions.get('reject'))} -- Lines of code: added {_int(lines.get('added'))} / removed {_int(lines.get('removed'))} - -## Layer 3: Behavior Pattern - -- Hourly distribution (unique sessions): -{_hourly_lines(cc.get('hourly_sessions', []))} -- Query source distribution (token-weighted): -{_dict_lines(cc.get('query_source', {}), suffix=' tokens')} -- Average tool duration top 10: - -{_latency_table(cc.get('tool_latency', []))} - -## Layer 4: Focus and Stability - -- Terminal type distribution (log entry count): -{_dict_lines(cc.get('terminal_distribution', {}))} -- Active time by type (seconds): -{_dict_lines(active)} -- Counter sessions: {_fmt(sessions.get('total'))}, Loki distinct sessions: {_fmt(sessions.get('loki_distinct'))} - -## Problems / Improvement Candidates - -{observations} -""" - - -def _write_policy(report_path: Path, metrics_path: Path, report: str) -> dict[str, Any]: - checks = [ - report_path.as_posix().endswith("-claude-code-usage.md"), - "/data/wiki/summaries/" in report_path.as_posix(), - "/data/ops/reports/" in metrics_path.as_posix(), - "sources: []" in report, - "$" not in report, - ] - passed = sum(1 for c in checks if c) - return { - "passed": passed, - "total": len(checks), - "rate_pct": _pct_or_none(passed, len(checks)), - "status": "evaluated", - } - - -def write_outputs( - metrics: dict[str, Any], base_dir: Path = BASEDIR -) -> dict[str, object]: - d = metrics["date"] - year, month, _ = d.split("-") - report_path = ( - base_dir / "data/wiki/summaries" / year / month / f"{d}-claude-code-usage.md" - ) - report = render_daily_report(metrics) - metrics["policy_compliance"] = _write_policy(report_path, report_path, report) - report = render_daily_report(metrics) - export_path = report_path.relative_to(base_dir / "data").as_posix() - cc = metrics.get("claude_code", {}) - submit_page_and_metrics( - report=report, - export_path=export_path, - slug=report_path.stem, - report_date=d, - report_type="claude_code", - metrics=metrics, - session_count=_int_or_none(cc.get("sessions", {}).get("total")), - token_total=_int_or_none(cc.get("tokens", {}).get("total")), - cost_usd=_float_or_none(cc.get("cost", {}).get("recorded_usd")), - tool_error_count=_int_or_none(cc.get("error_rate", {}).get("tool_errors")), - ) - return {"report": report_path, "metrics": "db"} - - -def _int_or_none(val: Any) -> int | None: - try: - return int(val) - except (TypeError, ValueError): - return None - - -def _float_or_none(val: Any) -> float | None: - try: - return float(val) - except (TypeError, ValueError): - return None - - -def _default_target_date() -> str: - return (datetime.now(KST).date() - timedelta(days=1)).isoformat() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--date", default=_default_target_date()) - parser.add_argument("--prom", default=DEFAULT_PROM) - parser.add_argument("--loki", default=DEFAULT_LOKI) - parser.add_argument("--base-dir", type=Path, default=BASEDIR) - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--lint", action="store_true") - args = parser.parse_args(argv) - - metrics = collect_claude_code_metrics(args.date, args.prom, args.loki) - if args.dry_run: - print(render_daily_report(metrics)) - return 0 - outputs = write_outputs(metrics, args.base_dir) - print("generated:") - for key, path in outputs.items(): - print(f"- {key}: {path}") - if args.lint: - print("lint: skipped; service layer validates and exports Markdown on write") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/kb/cli/db_ttl_sweep.py b/src/kb/cli/db_ttl_sweep.py deleted file mode 100644 index 7b81d2a..0000000 --- a/src/kb/cli/db_ttl_sweep.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Run the DB-canonical wiki TTL sweep.""" - -from __future__ import annotations - -import argparse - -from kb.service import pages as service_pages -from kb.service.session import session_scope - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--days", type=int, default=7) - args = parser.parse_args(argv) - with session_scope() as (session, data_dir): - result = service_pages.ttl_sweep(session, data_dir, days=args.days) - print(f"swept: {result.get('swept', 0)}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/kb/cli/hermes_daily_report.py b/src/kb/cli/hermes_daily_report.py deleted file mode 100644 index 96530c0..0000000 --- a/src/kb/cli/hermes_daily_report.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic Hermes daily usage report generator.""" - -from __future__ import annotations - -import argparse -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -from kb import REPO_ROOT as BASEDIR -from kb.cli._submit import submit_page_and_metrics -from kb.cli.usage_reports.collect import ( - DEFAULT_HERMES_DB, - DEFERRED_METRICS, - _collect_hermes, - _pct, -) -from kb.cli.usage_reports.render import ( - _fmt, - _hourly_lines, - _int, - _kv_lines, - _model_table, - _pct as _pct_text, - _schema_cell, -) - -KST = timezone(timedelta(hours=9)) - - -def _summary_dir(base_dir: Path, target_date: str) -> Path: - year, month, _ = target_date.split("-") - return base_dir / "data/wiki/summaries" / year / month - - -def _metrics_dir(base_dir: Path, target_date: str) -> Path: - year, month, _ = target_date.split("-") - return base_dir / "data/ops/reports" / year / month - - -def collect_metrics( - target_date: str, db_path: Path = DEFAULT_HERMES_DB -) -> dict[str, Any]: - return { - "date": target_date, - "generated_at": datetime.now(timezone.utc).isoformat(), - "deferred_metrics": DEFERRED_METRICS, - "hermes": _collect_hermes(target_date, Path(db_path)), - "policy_compliance": { - "passed": 0, - "total": 0, - "rate_pct": None, - "status": "not_evaluated_until_write", - }, - } - - -def _observations(metrics: dict[str, Any]) -> list[str]: - he = metrics["hermes"] - if not he.get("available", True): - return [f"Hermes DB unavailable: {he.get('reason')}"] - obs: list[str] = [] - if float(he.get("sessions", {}).get("zombie") or 0) > 0: - obs.append( - f"Zombie sessions {_fmt(he.get('sessions', {}).get('zombie'))} found. Check TUI/cron shutdown handling." - ) - if not obs: - obs.append("No notable signals. Current values can be used as baseline.") - return obs - - -def render_report(metrics: dict[str, Any]) -> str: - d = metrics["date"] - he = metrics["hermes"] - pc = metrics["policy_compliance"] - observations = "\n".join(f"- {line}" for line in _observations(metrics)) - return f"""--- -type: summary -subtype: daily -date: "{d}" -created: "{d}" -updated: "{d}" -sources: [] -tags: [agent-usage, deterministic-report, hermes] ---- - -# Hermes Daily Report - {d} - -## Summary - -- Root sessions: {_fmt(he.get('sessions', {}).get('root'))} -- Zombie sessions: {_fmt(he.get('sessions', {}).get('zombie'))} -- Turns: {_int(he.get('n_turns'))} -- Tool calls: {_int(he.get('n_toolcalls'))} -- Total tokens: {_int(he.get('tokens', {}).get('total'))} -- Recorded cost: {_fmt(he.get('cost', {}).get('recorded_usd'))} USD - -## Development/Evaluation Metrics - -| Metric | Hermes | Status | -|---|---:|---| -| Task Completion Rate | N/A | task schema needed | -| Error Rate | N/A | source schema needed | -| Hallucinated Parameters | {_schema_cell(he)} | deterministic where available | -| n_toolcalls / n_turns | {_fmt(he.get('n_toolcalls'))} / {_fmt(he.get('n_turns'))} | deterministic | -| Policy Compliance Rate | {_pct_text(pc.get('rate_pct'))} | {pc.get('status')} | - -## Layer 1: Cost and Efficiency - -- Tokens: {_int(he.get('tokens', {}).get('total'))} (input {_int(he.get('tokens', {}).get('input'))} / output {_int(he.get('tokens', {}).get('output'))} / cache_read {_int(he.get('tokens', {}).get('cache_read'))} / cache_write {_int(he.get('tokens', {}).get('cache_write'))} / reasoning {_int(he.get('tokens', {}).get('reasoning'))}) -- Recorded cost: {_fmt(he.get('cost', {}).get('recorded_usd'))} USD -- Average session time: {_fmt(he.get('latency', {}).get('avg_session_sec'))}s / max {_fmt(he.get('latency', {}).get('max_session_sec'))}s -- Model usage: - -{_model_table(he.get('model_usage', []), include_sessions=True)} - -## Layer 2: Work Quality - -- End reason distribution: -{_kv_lines(he.get('end_reason_distribution', []), 'end_reason')} -- Source distribution: -{_kv_lines(he.get('source_distribution', []), 'source')} -- Zombie sessions: {_fmt(he.get('sessions', {}).get('zombie'))} -- Hallucinated Parameters: {_schema_cell(he)} - -## Layer 3: Behavior Pattern - -- Hourly sessions (KST): -{_hourly_lines(he.get('hourly_sessions', []))} -- n_toolcalls / n_turns: {_fmt(he.get('n_toolcalls'))} / {_fmt(he.get('n_turns'))} - -## Layer 4: Focus and Stability - -- Model sessions: -{_kv_lines(he.get('model_usage', []), 'model')} -- Max session latency: {_fmt(he.get('latency', {}).get('max_session_sec'))}s - -## Problems / Improvement Candidates - -{observations} -""" - - -def _write_policy(report_path: Path, metrics_path: Path, report: str) -> dict[str, Any]: - checks = [ - report_path.as_posix().endswith("-hermes-usage.md"), - "/data/wiki/summaries/" in report_path.as_posix(), - "/data/ops/reports/" in metrics_path.as_posix(), - "sources: []" in report, - "$" not in report, - ] - passed = sum(1 for c in checks if c) - return { - "passed": passed, - "total": len(checks), - "rate_pct": _pct(passed, len(checks)), - "status": "evaluated", - } - - -def write_outputs( - metrics: dict[str, Any], base_dir: Path = BASEDIR -) -> dict[str, object]: - d = metrics["date"] - report_path = _summary_dir(base_dir, d) / f"{d}-hermes-usage.md" - report = render_report(metrics) - metrics["policy_compliance"] = _write_policy(report_path, report_path, report) - report = render_report(metrics) - export_path = report_path.relative_to(base_dir / "data").as_posix() - he = metrics.get("hermes", {}) - submit_page_and_metrics( - report=report, - export_path=export_path, - slug=report_path.stem, - report_date=d, - report_type="hermes", - metrics=metrics, - token_total=_int_or_none(he.get("tokens", {}).get("total")), - cost_usd=_float_or_none(he.get("cost", {}).get("recorded_usd")), - ) - return {"report": report_path, "metrics": "db"} - - -def _int_or_none(val: Any) -> int | None: - try: - return int(val) - except (TypeError, ValueError): - return None - - -def _float_or_none(val: Any) -> float | None: - try: - return float(val) - except (TypeError, ValueError): - return None - - -def _default_target_date() -> str: - return (datetime.now(KST).date() - timedelta(days=1)).isoformat() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--date", default=_default_target_date()) - parser.add_argument("--hermes-db", type=Path, default=DEFAULT_HERMES_DB) - parser.add_argument("--base-dir", type=Path, default=BASEDIR) - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--lint", action="store_true") - args = parser.parse_args(argv) - - metrics = collect_metrics(args.date, args.hermes_db) - if args.dry_run: - print(render_report(metrics)) - return 0 - outputs = write_outputs(metrics, args.base_dir) - print("generated:") - for key, path in outputs.items(): - print(f"- {key}: {path}") - if args.lint: - print("lint: skipped; service layer validates and exports Markdown on write") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/kb/cli/opencode_daily_report.py b/src/kb/cli/opencode_daily_report.py deleted file mode 100644 index 033c2d1..0000000 --- a/src/kb/cli/opencode_daily_report.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic OpenCode daily usage report generator.""" - -from __future__ import annotations - -import argparse -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -from kb import REPO_ROOT as BASEDIR -from kb.cli._submit import submit_page_and_metrics -from kb.cli.usage_reports.collect import ( - DEFAULT_OPENCODE_DB, - DEFERRED_METRICS, - _collect_opencode, - _pct, -) -from kb.cli.usage_reports.render import ( - _fmt, - _hot_files, - _hourly_lines, - _int, - _model_table, - _pct as _pct_text, - _schema_cell, - _tool_table, -) - -KST = timezone(timedelta(hours=9)) - - -def _summary_dir(base_dir: Path, target_date: str) -> Path: - year, month, _ = target_date.split("-") - return base_dir / "data/wiki/summaries" / year / month - - -def _metrics_dir(base_dir: Path, target_date: str) -> Path: - year, month, _ = target_date.split("-") - return base_dir / "data/ops/reports" / year / month - - -def collect_metrics( - target_date: str, db_path: Path = DEFAULT_OPENCODE_DB -) -> dict[str, Any]: - return { - "date": target_date, - "generated_at": datetime.now(timezone.utc).isoformat(), - "deferred_metrics": DEFERRED_METRICS, - "opencode": _collect_opencode(target_date, Path(db_path)), - "policy_compliance": { - "passed": 0, - "total": 0, - "rate_pct": None, - "status": "not_evaluated_until_write", - }, - } - - -def _observations(metrics: dict[str, Any]) -> list[str]: - oc = metrics["opencode"] - obs: list[str] = [] - if not oc.get("available", True): - return [f"OpenCode DB unavailable: {oc.get('reason')}"] - cache_models = [ - r for r in oc.get("model_usage", []) if float(r.get("cache_hit_pct") or 0) >= 95 - ] - if cache_models: - names = ", ".join(str(r.get("model")) for r in cache_models[:3]) - obs.append( - f"High cache-hit models detected ({names}); this suggests repeated-context work patterns." - ) - todo = oc.get("todo", {}) - if todo.get("total"): - obs.append( - f"TODO completion rate is {_pct_text(todo.get('completion_rate_pct'))} ({_fmt(todo.get('completed'))}/{_fmt(todo.get('total'))})." - ) - errors = [ - r for r in oc.get("tool_breakdown", []) if float(r.get("errors") or 0) > 0 - ] - if errors: - worst = sorted( - errors, - key=lambda r: ( - float(r.get("error_rate_pct") or 0), - float(r.get("errors") or 0), - ), - reverse=True, - )[0] - obs.append( - f"Tool errors are most visible in {worst.get('tool')}: {_fmt(worst.get('errors'))}/{_fmt(worst.get('calls'))} ({_pct_text(worst.get('error_rate_pct'))})." - ) - if not obs: - obs.append("No notable signals. Current values can be used as baseline.") - return obs - - -def render_report(metrics: dict[str, Any]) -> str: - d = metrics["date"] - oc = metrics["opencode"] - pc = metrics["policy_compliance"] - todo = oc.get("todo", {}) - delegation_rate = None - if float(oc.get("sessions", {}).get("total") or 0): - delegation_rate = round( - float(oc.get("sessions", {}).get("subagent") or 0) - * 100 - / float(oc.get("sessions", {}).get("total") or 0), - 2, - ) - observations = "\n".join(f"- {line}" for line in _observations(metrics)) - return f"""--- -type: summary -subtype: daily -date: "{d}" -created: "{d}" -updated: "{d}" -sources: [] -tags: [agent-usage, deterministic-report, opencode] ---- - -# OpenCode Daily Report - {d} - -## Summary - -- Sessions: total {_fmt(oc.get('sessions', {}).get('total'))} (root {_fmt(oc.get('sessions', {}).get('root'))} / subagent {_fmt(oc.get('sessions', {}).get('subagent'))}) -- Projects: {_fmt(oc.get('sessions', {}).get('projects'))} -- Total tokens: {_int(oc.get('tokens', {}).get('total'))} -- Recorded cost: {_fmt(oc.get('cost', {}).get('recorded_usd'))} USD -- TODO completion: {_pct_text(todo.get('completion_rate_pct'))} ({_fmt(todo.get('completed'))}/{_fmt(todo.get('total'))}) -- Delegation rate: {_pct_text(delegation_rate)} - -## Development/Evaluation Metrics - -| Metric | OpenCode | Status | -|---|---:|---| -| Task Completion Rate | TODO proxy {_pct_text(todo.get('completion_rate_pct'))} ({_fmt(todo.get('completed'))}/{_fmt(todo.get('total'))}) | task schema needed for true eval | -| Error Rate | {_fmt(oc.get('error_rate', {}).get('tool_errors'))}/{_fmt(oc.get('error_rate', {}).get('tool_calls'))} ({_fmt(oc.get('error_rate', {}).get('rate_pct'))}%) | deterministic | -| Hallucinated Parameters | {_schema_cell(oc)} | deterministic schema check | -| n_toolcalls / n_turns | {_fmt(oc.get('n_toolcalls'))} / {_fmt(oc.get('n_turns'))} | deterministic | -| Policy Compliance Rate | {_pct_text(pc.get('rate_pct'))} | {pc.get('status')} | - -## Layer 1: Cost and Efficiency - -- Tokens: {_int(oc.get('tokens', {}).get('total'))} (input {_int(oc.get('tokens', {}).get('input'))} / output {_int(oc.get('tokens', {}).get('output'))} / cache_read {_int(oc.get('tokens', {}).get('cache_read'))} / cache_write {_int(oc.get('tokens', {}).get('cache_write'))} / reasoning {_int(oc.get('tokens', {}).get('reasoning'))}) -- Recorded cost: {_fmt(oc.get('cost', {}).get('recorded_usd'))} USD -- Average session time: {_fmt(oc.get('latency', {}).get('avg_session_sec'))}s / max {_fmt(oc.get('latency', {}).get('max_session_sec'))}s -- Model usage: - -{_model_table(oc.get('model_usage', []))} - -## Layer 2: Work Quality - -- TODO completion: {_pct_text(todo.get('completion_rate_pct'))} ({_fmt(todo.get('completed'))}/{_fmt(todo.get('total'))}) -- Tool error rate: {_fmt(oc.get('error_rate', {}).get('rate_pct'))}% ({_fmt(oc.get('error_rate', {}).get('tool_errors'))}/{_fmt(oc.get('error_rate', {}).get('tool_calls'))}) -- Tool errors: - -{_tool_table(oc.get('tool_breakdown', []))} - -- Hallucinated Parameters: {_schema_cell(oc)} -- Compactions: {_fmt(oc.get('sessions', {}).get('compactions'))} - -## Layer 3: Behavior Pattern - -- Hourly sessions (KST): -{_hourly_lines(oc.get('hourly_sessions', []))} -- root vs subagent: root {_fmt(oc.get('sessions', {}).get('root'))} / subagent {_fmt(oc.get('sessions', {}).get('subagent'))} - -## Layer 4: Focus and Stability - -- Project distribution: -{chr(10).join(f" - {r.get('project')}: {_fmt(r.get('sessions'))}" for r in oc.get('projects', [])[:8]) or ' - N/A'} -- Hot files: -{_hot_files(oc.get('hot_files', []))} - -## Problems / Improvement Candidates - -{observations} -""" - - -def _write_policy(report_path: Path, metrics_path: Path, report: str) -> dict[str, Any]: - checks = [ - report_path.as_posix().endswith("-opencode-usage.md"), - "/data/wiki/summaries/" in report_path.as_posix(), - "/data/ops/reports/" in metrics_path.as_posix(), - "sources: []" in report, - "$" not in report, - ] - passed = sum(1 for c in checks if c) - return { - "passed": passed, - "total": len(checks), - "rate_pct": _pct(passed, len(checks)), - "status": "evaluated", - } - - -def write_outputs( - metrics: dict[str, Any], base_dir: Path = BASEDIR -) -> dict[str, object]: - d = metrics["date"] - report_path = _summary_dir(base_dir, d) / f"{d}-opencode-usage.md" - report = render_report(metrics) - metrics["policy_compliance"] = _write_policy(report_path, report_path, report) - report = render_report(metrics) - export_path = report_path.relative_to(base_dir / "data").as_posix() - oc = metrics.get("opencode", {}) - submit_page_and_metrics( - report=report, - export_path=export_path, - slug=report_path.stem, - report_date=d, - report_type="opencode", - metrics=metrics, - session_count=_int_or_none(oc.get("sessions", {}).get("total")), - token_total=_int_or_none(oc.get("tokens", {}).get("total")), - cost_usd=_float_or_none(oc.get("cost", {}).get("recorded_usd")), - tool_error_count=_int_or_none(oc.get("error_rate", {}).get("tool_errors")), - ) - return {"report": report_path, "metrics": "db"} - - -def _int_or_none(val: Any) -> int | None: - try: - return int(val) - except (TypeError, ValueError): - return None - - -def _float_or_none(val: Any) -> float | None: - try: - return float(val) - except (TypeError, ValueError): - return None - - -def _default_target_date() -> str: - return (datetime.now(KST).date() - timedelta(days=1)).isoformat() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--date", default=_default_target_date()) - parser.add_argument("--opencode-db", type=Path, default=DEFAULT_OPENCODE_DB) - parser.add_argument("--base-dir", type=Path, default=BASEDIR) - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--lint", action="store_true") - args = parser.parse_args(argv) - - metrics = collect_metrics(args.date, args.opencode_db) - if args.dry_run: - print(render_report(metrics)) - return 0 - outputs = write_outputs(metrics, args.base_dir) - print("generated:") - for key, path in outputs.items(): - print(f"- {key}: {path}") - if args.lint: - print("lint: skipped; service layer validates and exports Markdown on write") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/kb/cli/submit_cron_run.py b/src/kb/cli/submit_cron_run.py deleted file mode 100644 index 6799a76..0000000 --- a/src/kb/cli/submit_cron_run.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Submit a cron run log to the KB database (in-process, no HTTP).""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -from kb.service import ops as service_ops -from kb.service.session import session_scope - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Submit a cron run log to the KB DB.") - parser.add_argument("--job-name", required=True) - parser.add_argument("--target", required=True) - parser.add_argument("--status", required=True, choices=["success", "failed"]) - parser.add_argument("--exit-code", type=int) - parser.add_argument("--log-path") - parser.add_argument("--log-file", type=Path, required=True) - parser.add_argument("--started-at") - parser.add_argument("--finished-at") - args = parser.parse_args(argv) - - with session_scope() as (session, data_dir): - service_ops.create_cron_run( - session, - data_dir, - job_name=args.job_name, - target=args.target, - status=args.status, - log_body=args.log_file.read_text(encoding="utf-8"), - exit_code=args.exit_code, - log_path=args.log_path, - started_at=args.started_at, - finished_at=args.finished_at, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/kb/cli/usage_reports/__init__.py b/src/kb/cli/usage_reports/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/kb/cli/usage_reports/collect.py b/src/kb/cli/usage_reports/collect.py deleted file mode 100644 index fd5ea4a..0000000 --- a/src/kb/cli/usage_reports/collect.py +++ /dev/null @@ -1,640 +0,0 @@ -#!/usr/bin/env python3 -"""Shared collectors for deterministic usage reports.""" - -from __future__ import annotations - -import json -import sqlite3 -from pathlib import Path -from typing import Any - -DEFAULT_OPENCODE_DB = Path.home() / ".local/share/opencode/opencode.db" -DEFAULT_HERMES_DB = Path.home() / ".hermes/state.db" -DEFERRED_METRICS = ["Task Completion Rate", "pass@k", "pass^k"] - -TOOL_SCHEMAS: dict[str, dict[str, set[str]]] = { - # Hermes tools - "terminal": { - "required": {"command"}, - "allowed": { - "command", - "timeout", - "workdir", - "background", - "pty", - "notify_on_complete", - "watch_patterns", - }, - }, - "read_file": {"required": {"path"}, "allowed": {"path", "offset", "limit"}}, - "search_files": { - "required": {"pattern"}, - "allowed": { - "pattern", - "target", - "path", - "file_glob", - "limit", - "offset", - "output_mode", - "context", - }, - }, - "write_file": {"required": {"path", "content"}, "allowed": {"path", "content"}}, - "patch": { - "required": {"mode"}, - "allowed": {"mode", "path", "old_string", "new_string", "replace_all", "patch"}, - }, - "skill_view": {"required": {"name"}, "allowed": {"name", "file_path"}}, - "skill_manage": { - "required": {"action", "name"}, - "allowed": { - "action", - "name", - "content", - "old_string", - "new_string", - "replace_all", - "category", - "file_path", - "file_content", - "absorbed_into", - }, - }, - "todo": {"required": set(), "allowed": {"todos", "merge"}}, - "cronjob": { - "required": {"action"}, - "allowed": { - "action", - "job_id", - "prompt", - "schedule", - "name", - "repeat", - "deliver", - "skills", - "model", - "script", - "no_agent", - "context_from", - "enabled_toolsets", - "workdir", - }, - }, - # OpenCode tools - "bash": { - "required": {"command"}, - "allowed": {"command", "description", "timeout", "workdir"}, - }, - "read": {"required": {"filePath"}, "allowed": {"filePath", "offset", "limit"}}, - "write": {"required": {"filePath", "content"}, "allowed": {"filePath", "content"}}, - "edit": { - "required": {"filePath", "oldString", "newString"}, - "allowed": {"filePath", "oldString", "newString", "replaceAll"}, - }, - "todowrite": {"required": {"todos"}, "allowed": {"todos"}}, - "webfetch": {"required": {"url"}, "allowed": {"url", "format", "timeout"}}, -} - - -def _connect(path: Path) -> sqlite3.Connection | None: - if not path.exists(): - return None - con = sqlite3.connect(path) - con.row_factory = sqlite3.Row - return con - - -def _one(con: sqlite3.Connection, sql: str, args: tuple[Any, ...]) -> dict[str, Any]: - row = con.execute(sql, args).fetchone() - return dict(row) if row else {} - - -def _rows( - con: sqlite3.Connection, sql: str, args: tuple[Any, ...] -) -> list[dict[str, Any]]: - return [dict(row) for row in con.execute(sql, args).fetchall()] - - -def _has_table(con: sqlite3.Connection, name: str) -> bool: - row = con.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,) - ).fetchone() - return row is not None - - -def _num(value: Any) -> float: - return float(value or 0) - - -def _pct(numerator: float, denominator: float) -> float | None: - if denominator <= 0: - return None - return round(numerator * 100 / denominator, 2) - - -def _validate_tool_call(tool: str | None, args: Any) -> list[str]: - if not tool or tool not in TOOL_SCHEMAS: - return [] - if isinstance(args, str): - try: - args = json.loads(args) if args else {} - except json.JSONDecodeError: - return ["arguments_json_parse_error"] - if not isinstance(args, dict): - return ["arguments_not_object"] - schema = TOOL_SCHEMAS[tool] - errors: list[str] = [] - missing = schema["required"] - set(args) - extra = set(args) - schema["allowed"] - if missing: - errors.append("missing:" + ",".join(sorted(missing))) - if extra: - errors.append("extra:" + ",".join(sorted(extra))) - return errors - - -def _tool_schema_summary(calls: list[tuple[str | None, Any]]) -> dict[str, Any]: - validated = invalid = unvalidated = 0 - examples: list[str] = [] - for tool, args in calls: - if not tool or tool not in TOOL_SCHEMAS: - unvalidated += 1 - continue - validated += 1 - errors = _validate_tool_call(tool, args) - if errors: - invalid += 1 - if len(examples) < 5: - examples.append(f"{tool}: {'; '.join(errors)}") - return { - "validated_calls": validated, - "invalid_calls": invalid, - "unvalidated_calls": unvalidated, - "compliance_rate_pct": _pct(validated - invalid, validated), - "examples": examples, - } - - -def _collect_opencode(target_date: str, db_path: Path) -> dict[str, Any]: - con = _connect(db_path) - if con is None: - return {"available": False, "reason": f"missing db: {db_path}"} - sessions = _one( - con, - """ - SELECT count(*) total, - coalesce(sum(parent_id IS NULL), 0) root, - coalesce(sum(parent_id IS NOT NULL), 0) subagent, - count(distinct project_id) projects, - avg((time_updated - time_created) / 1000.0) avg_session_sec, - max((time_updated - time_created) / 1000.0) max_session_sec, - coalesce(sum(coalesce(summary_files, 0)), 0) summary_files, - coalesce(sum(time_compacting IS NOT NULL), 0) compactions - FROM session - WHERE date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ) - tokens = _one( - con, - """ - SELECT coalesce(sum(json_extract(m.data, '$.tokens.input')), 0) input, - coalesce(sum(json_extract(m.data, '$.tokens.output')), 0) output, - coalesce(sum(json_extract(m.data, '$.tokens.reasoning')), 0) reasoning, - coalesce(sum(json_extract(m.data, '$.tokens.cache.read')), 0) cache_read, - coalesce(sum(json_extract(m.data, '$.tokens.cache.write')), 0) cache_write, - coalesce(sum(json_extract(m.data, '$.cost')), 0) recorded_usd - FROM message m - JOIN session s ON m.session_id = s.id - WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = ? - AND json_extract(m.data, '$.role') = 'assistant' - """, - (target_date,), - ) - model_rows = _rows( - con, - """ - SELECT coalesce(json_extract(m.data, '$.modelID'), s.model, 'unknown') model, - coalesce(json_extract(m.data, '$.providerID'), 'unknown') provider, - coalesce(sum(json_extract(m.data, '$.tokens.input')), 0) input, - coalesce(sum(json_extract(m.data, '$.tokens.output')), 0) output, - coalesce(sum(json_extract(m.data, '$.tokens.reasoning')), 0) reasoning, - coalesce(sum(json_extract(m.data, '$.tokens.cache.read')), 0) cache_read, - coalesce(sum(json_extract(m.data, '$.tokens.cache.write')), 0) cache_write, - coalesce(sum(json_extract(m.data, '$.cost')), 0) cost, - count(*) messages - FROM message m - JOIN session s ON m.session_id = s.id - WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = ? - AND json_extract(m.data, '$.role') = 'assistant' - GROUP BY 1, 2 - ORDER BY (input + output + reasoning + cache_read + cache_write) DESC - """, - (target_date,), - ) - turns = _one( - con, - """ - SELECT count(*) n_turns - FROM message m - JOIN session s ON m.session_id = s.id - WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ) - tool = _one( - con, - """ - SELECT count(*) tool_calls, - coalesce(sum(json_extract(data, '$.state.status') = 'error'), 0) tool_errors - FROM part - WHERE json_extract(data, '$.type') = 'tool' - AND date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ) - tool_breakdown = _rows( - con, - """ - SELECT coalesce(json_extract(data, '$.tool'), 'unknown') tool, - count(*) calls, - coalesce(sum(json_extract(data, '$.state.status') = 'error'), 0) errors - FROM part - WHERE json_extract(data, '$.type') = 'tool' - AND date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - GROUP BY tool - ORDER BY calls DESC, tool ASC - """, - (target_date,), - ) - tool_rows = _rows( - con, - """ - SELECT json_extract(data, '$.tool') tool, - json_extract(data, '$.state.input') input - FROM part - WHERE json_extract(data, '$.type') = 'tool' - AND date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ) - hourly = _rows( - con, - """ - SELECT strftime('%H', datetime(time_created/1000, 'unixepoch', '+9 hours')) hour, - count(*) sessions, - coalesce(sum(parent_id IS NULL), 0) root, - coalesce(sum(parent_id IS NOT NULL), 0) subagent - FROM session - WHERE date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - GROUP BY hour - ORDER BY hour - """, - (target_date,), - ) - project_rows = _rows( - con, - """ - SELECT coalesce(p.name, replace(s.directory, rtrim(s.directory, replace(s.directory, '/', '')), ''), s.project_id) project, - coalesce(p.worktree, s.directory) path, - count(*) sessions - FROM session s - LEFT JOIN project p ON p.id = s.project_id - WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = ? - GROUP BY s.project_id - ORDER BY sessions DESC, project ASC - """, - (target_date,), - ) - if _has_table(con, "todo"): - todo_rows = _rows( - con, - """ - SELECT status, count(*) count - FROM todo t - JOIN session s ON t.session_id = s.id - WHERE date(datetime(s.time_created/1000, 'unixepoch', '+9 hours')) = ? - GROUP BY status - """, - (target_date,), - ) - else: - todo_rows = [] - hot_files = _rows( - con, - """ - SELECT json_extract(data, '$.state.input.filePath') file, - count(*) edits - FROM part - WHERE json_extract(data, '$.type') = 'tool' - AND json_extract(data, '$.tool') IN ('write', 'edit') - AND json_extract(data, '$.state.input.filePath') IS NOT NULL - AND date(datetime(time_created/1000, 'unixepoch', '+9 hours')) = ? - GROUP BY file - ORDER BY edits DESC, file ASC - LIMIT 10 - """, - (target_date,), - ) - con.close() - total_tokens = sum( - _num(tokens.get(k)) - for k in ("input", "output", "reasoning", "cache_read", "cache_write") - ) - model_usage = [] - for row in model_rows: - total = sum( - _num(row.get(k)) - for k in ("input", "output", "reasoning", "cache_read", "cache_write") - ) - cache_miss = _num(row.get("input")) - cache_read = _num(row.get("cache_read")) - total_input = cache_miss + cache_read - model_usage.append( - { - "model": row.get("model"), - "provider": row.get("provider"), - "input": int(total_input), - "cache_miss": int(cache_miss), - "cache_read": int(cache_read), - "output": int(_num(row.get("output"))), - "reasoning": int(_num(row.get("reasoning"))), - "cache_write": int(_num(row.get("cache_write"))), - "total": int(total), - "cost": round(_num(row.get("cost")), 6), - "cache_hit_pct": _pct(cache_read, total_input), - "messages": int(_num(row.get("messages"))), - } - ) - todo_counts = {str(r.get("status")): int(_num(r.get("count"))) for r in todo_rows} - todo_total = sum(todo_counts.values()) - todo_completed = todo_counts.get("completed", 0) - return { - "available": True, - "sessions": { - "total": int(_num(sessions.get("total"))), - "root": int(_num(sessions.get("root"))), - "subagent": int(_num(sessions.get("subagent"))), - "projects": int(_num(sessions.get("projects"))), - "summary_files": int(_num(sessions.get("summary_files"))), - "compactions": int(_num(sessions.get("compactions"))), - }, - "n_turns": int(_num(turns.get("n_turns"))), - "n_toolcalls": int(_num(tool.get("tool_calls"))), - "error_rate": { - "tool_calls": int(_num(tool.get("tool_calls"))), - "tool_errors": int(_num(tool.get("tool_errors"))), - "rate_pct": _pct( - _num(tool.get("tool_errors")), _num(tool.get("tool_calls")) - ), - }, - "tool_schema": _tool_schema_summary( - [(r.get("tool"), r.get("input")) for r in tool_rows] - ), - "tool_breakdown": [ - { - "tool": r.get("tool"), - "calls": int(_num(r.get("calls"))), - "errors": int(_num(r.get("errors"))), - "error_rate_pct": _pct(_num(r.get("errors")), _num(r.get("calls"))), - } - for r in tool_breakdown - ], - "todo": { - "total": todo_total, - "completed": todo_completed, - "pending": todo_counts.get("pending", 0), - "in_progress": todo_counts.get("in_progress", 0), - "cancelled": todo_counts.get("cancelled", 0), - "completion_rate_pct": _pct(todo_completed, todo_total), - }, - "model_usage": model_usage, - "hourly_sessions": [ - { - "hour": r.get("hour"), - "sessions": int(_num(r.get("sessions"))), - "root": int(_num(r.get("root"))), - "subagent": int(_num(r.get("subagent"))), - } - for r in hourly - ], - "projects": [ - { - "project": r.get("project"), - "path": r.get("path"), - "sessions": int(_num(r.get("sessions"))), - } - for r in project_rows - ], - "hot_files": [ - {"file": r.get("file"), "edits": int(_num(r.get("edits")))} - for r in hot_files - ], - "tokens": { - "input": int(_num(tokens.get("input"))), - "output": int(_num(tokens.get("output"))), - "reasoning": int(_num(tokens.get("reasoning"))), - "cache_read": int(_num(tokens.get("cache_read"))), - "cache_write": int(_num(tokens.get("cache_write"))), - "total": int(total_tokens), - }, - "latency": { - "avg_session_sec": round(_num(sessions.get("avg_session_sec")), 2), - "max_session_sec": round(_num(sessions.get("max_session_sec")), 2), - }, - "cost": { - "recorded_usd": round(_num(tokens.get("recorded_usd")), 6), - "cost_per_session_usd": ( - round(_num(tokens.get("recorded_usd")) / _num(sessions.get("total")), 6) - if _num(sessions.get("total")) - else None - ), - }, - } - - -def _collect_hermes(target_date: str, db_path: Path) -> dict[str, Any]: - con = _connect(db_path) - if con is None: - return {"available": False, "reason": f"missing db: {db_path}"} - sessions = _one( - con, - """ - SELECT count(*) root, - coalesce(sum(ended_at IS NULL), 0) zombie, - coalesce(sum(tool_call_count), 0) toolcalls, - coalesce(sum(message_count), 0) turns, - coalesce(sum(input_tokens), 0) input, - coalesce(sum(output_tokens), 0) output, - coalesce(sum(cache_read_tokens), 0) cache_read, - coalesce(sum(cache_write_tokens), 0) cache_write, - coalesce(sum(reasoning_tokens), 0) reasoning, - coalesce(sum(coalesce(actual_cost_usd, estimated_cost_usd, 0)), 0) cost, - avg(CASE WHEN ended_at IS NOT NULL THEN ended_at - started_at END) avg_session_sec, - max(CASE WHEN ended_at IS NOT NULL THEN ended_at - started_at END) max_session_sec - FROM sessions - WHERE parent_session_id IS NULL - AND date(datetime(started_at, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ) - model_rows = _rows( - con, - """ - SELECT coalesce(model, 'unknown') model, - coalesce(billing_provider, 'unknown') provider, - count(*) sessions, - coalesce(sum(input_tokens), 0) input, - coalesce(sum(output_tokens), 0) output, - coalesce(sum(cache_read_tokens), 0) cache_read, - coalesce(sum(cache_write_tokens), 0) cache_write, - coalesce(sum(reasoning_tokens), 0) reasoning, - coalesce(sum(coalesce(actual_cost_usd, estimated_cost_usd, 0)), 0) cost - FROM sessions - WHERE parent_session_id IS NULL - AND date(datetime(started_at, 'unixepoch', '+9 hours')) = ? - GROUP BY 1, 2 - ORDER BY (input + output + cache_read + cache_write + reasoning) DESC - """, - (target_date,), - ) - source_rows = _rows( - con, - """ - SELECT source, count(*) sessions - FROM sessions - WHERE parent_session_id IS NULL - AND date(datetime(started_at, 'unixepoch', '+9 hours')) = ? - GROUP BY source - ORDER BY sessions DESC, source ASC - """, - (target_date,), - ) - end_reason_rows = _rows( - con, - """ - SELECT coalesce(end_reason, 'NULL') end_reason, count(*) sessions - FROM sessions - WHERE parent_session_id IS NULL - AND date(datetime(started_at, 'unixepoch', '+9 hours')) = ? - GROUP BY end_reason - ORDER BY sessions DESC, end_reason ASC - """, - (target_date,), - ) - hourly = _rows( - con, - """ - SELECT strftime('%H', datetime(started_at, 'unixepoch', '+9 hours')) hour, - count(*) sessions - FROM sessions - WHERE parent_session_id IS NULL - AND date(datetime(started_at, 'unixepoch', '+9 hours')) = ? - GROUP BY hour - ORDER BY hour - """, - (target_date,), - ) - message_schema = _rows(con, "PRAGMA table_info(messages)", ()) - has_tool_calls = any(row.get("name") == "tool_calls" for row in message_schema) - calls: list[tuple[str | None, Any]] = [] - if has_tool_calls: - for row in _rows( - con, - """ - SELECT tool_calls - FROM messages - WHERE tool_calls IS NOT NULL - AND date(datetime(timestamp, 'unixepoch', '+9 hours')) = ? - """, - (target_date,), - ): - try: - entries = json.loads(row.get("tool_calls") or "[]") - except json.JSONDecodeError: - entries = [] - for entry in entries if isinstance(entries, list) else []: - fn = entry.get("function", {}) if isinstance(entry, dict) else {} - calls.append((fn.get("name"), fn.get("arguments"))) - con.close() - total_tokens = sum( - _num(sessions.get(k)) - for k in ("input", "output", "reasoning", "cache_read", "cache_write") - ) - model_usage = [] - for row in model_rows: - cache_miss = _num(row.get("input")) - cache_read = _num(row.get("cache_read")) - total_input = cache_miss + cache_read - total = sum( - _num(row.get(k)) - for k in ("input", "output", "reasoning", "cache_read", "cache_write") - ) - model_usage.append( - { - "model": row.get("model"), - "provider": row.get("provider"), - "sessions": int(_num(row.get("sessions"))), - "input": int(total_input), - "cache_miss": int(cache_miss), - "cache_read": int(cache_read), - "output": int(_num(row.get("output"))), - "reasoning": int(_num(row.get("reasoning"))), - "cache_write": int(_num(row.get("cache_write"))), - "total": int(total), - "cost": round(_num(row.get("cost")), 6), - "cache_hit_pct": _pct(cache_read, total_input), - } - ) - return { - "available": True, - "sessions": { - "root": int(_num(sessions.get("root"))), - "zombie": int(_num(sessions.get("zombie"))), - }, - "n_turns": int(_num(sessions.get("turns"))), - "n_toolcalls": int(_num(sessions.get("toolcalls"))), - "error_rate": { - "tool_calls": int(_num(sessions.get("toolcalls"))), - "tool_errors": None, - "rate_pct": None, - }, - "tool_schema": _tool_schema_summary(calls), - "model_usage": model_usage, - "source_distribution": [ - {"source": r.get("source"), "sessions": int(_num(r.get("sessions")))} - for r in source_rows - ], - "end_reason_distribution": [ - { - "end_reason": r.get("end_reason"), - "sessions": int(_num(r.get("sessions"))), - } - for r in end_reason_rows - ], - "hourly_sessions": [ - {"hour": r.get("hour"), "sessions": int(_num(r.get("sessions")))} - for r in hourly - ], - "tokens": { - "input": int(_num(sessions.get("input"))), - "output": int(_num(sessions.get("output"))), - "reasoning": int(_num(sessions.get("reasoning"))), - "cache_read": int(_num(sessions.get("cache_read"))), - "cache_write": int(_num(sessions.get("cache_write"))), - "total": int(total_tokens), - }, - "latency": { - "avg_session_sec": round(_num(sessions.get("avg_session_sec")), 2), - "max_session_sec": round(_num(sessions.get("max_session_sec")), 2), - }, - "cost": { - "recorded_usd": round(_num(sessions.get("cost")), 6), - "cost_per_session_usd": ( - round(_num(sessions.get("cost")) / _num(sessions.get("root")), 6) - if _num(sessions.get("root")) - else None - ), - }, - } diff --git a/src/kb/cli/usage_reports/render.py b/src/kb/cli/usage_reports/render.py deleted file mode 100644 index 11cf6f4..0000000 --- a/src/kb/cli/usage_reports/render.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Shared markdown helpers for deterministic usage reports.""" - -from __future__ import annotations - -from pathlib import PurePath -from typing import Any - - -def _num(value: Any) -> float: - return float(value or 0) - - -def _fmt(value: Any) -> str: - if value is None: - return "N/A" - if isinstance(value, float): - return f"{value:.2f}" if not value.is_integer() else str(int(value)) - return str(value) - - -def _int(value: Any) -> str: - return f"{int(_num(value)):,}" - - -def _pct(value: Any) -> str: - return "N/A" if value is None else f"{_fmt(value)}%" - - -def _schema_cell(system: dict[str, Any]) -> str: - s = system.get("tool_schema", {}) - examples = s.get("examples") or [] - suffix = f"; example: {examples[0]}" if examples else "" - return f"{_fmt(s.get('invalid_calls'))}/{_fmt(s.get('validated_calls'))} invalid, compliance {_fmt(s.get('compliance_rate_pct'))}%{suffix}" - - -def _model_table(rows: list[dict[str, Any]], include_sessions: bool = False) -> str: - if not rows: - return "- Model usage: N/A" - session_col = " | sessions" if include_sessions else "" - out = [ - f"| model | provider{session_col} | total | total_input | cache_miss | cache_read | output | cache_write | reasoning | cache_hit | cost USD |", - f"|---|---:{'|---:' if include_sessions else ''}|---:|---:|---:|---:|---:|---:|---:|---:|---:|", - ] - for r in rows: - model = r.get("model") or "unknown" - provider = r.get("provider") or "unknown" - sessions = f" | {_int(r.get('sessions'))}" if include_sessions else "" - out.append( - f"| {model} | {provider}{sessions} | {_int(r.get('total'))} | {_int(r.get('input'))} | {_int(r.get('cache_miss'))} | {_int(r.get('cache_read'))} | {_int(r.get('output'))} | {_int(r.get('cache_write'))} | {_int(r.get('reasoning'))} | {_pct(r.get('cache_hit_pct'))} | {_fmt(r.get('cost'))} |" - ) - return "\n".join(out) - - -def _tool_table(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Tool errors: N/A" - out = ["| tool | calls | errors | error_rate |", "|---|---:|---:|---:|"] - for r in rows: - out.append( - f"| {r.get('tool')} | {_int(r.get('calls'))} | {_int(r.get('errors'))} | {_pct(r.get('error_rate_pct'))} |" - ) - return "\n".join(out) - - -def _hourly_lines(rows: list[dict[str, Any]]) -> str: - if not rows: - return "- Hourly distribution: N/A" - lines = [] - for r in rows: - extra = "" - if "root" in r: - extra = ( - f" (root {_fmt(r.get('root'))} / subagent {_fmt(r.get('subagent'))})" - ) - lines.append(f" - {r.get('hour')}: {_fmt(r.get('sessions'))} sessions{extra}") - return "\n".join(lines) - - -def _kv_lines( - rows: list[dict[str, Any]], key: str, value: str = "sessions", limit: int = 8 -) -> str: - if not rows: - return " - N/A" - return "\n".join(f" - {r.get(key)}: {_fmt(r.get(value))}" for r in rows[:limit]) - - -def _hot_files(rows: list[dict[str, Any]]) -> str: - if not rows: - return " - N/A" - lines = [] - for r in rows[:8]: - name = PurePath(str(r.get("file"))).name - lines.append(f" - {name}: {_fmt(r.get('edits'))} edits ({r.get('file')})") - return "\n".join(lines) diff --git a/test/test_cli_consumers.py b/test/test_cli_consumers.py deleted file mode 100644 index f22c7d1..0000000 --- a/test/test_cli_consumers.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Integration tests for Task 8 CLI consumers — in-process service calls. - -All tests use the real Postgres DB (via `database_url` + `session` fixtures) and -a temp `data_dir`. The fixtures set DATABASE_URL and KB_DATA_DIR so that -session_scope() and service functions resolve them correctly. -""" - -from __future__ import annotations - -from pathlib import Path - -from sqlalchemy import select - -from kb.db.models import CronRun, MetricsRecord, Page -from kb.service import pages as service_pages - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_SUMMARY_MD = """\ ---- -type: summary -subtype: daily -date: "2026-06-04" -created: "2026-06-04" -updated: "2026-06-04" -sources: [] -tags: [agent-usage, deterministic-report, opencode] ---- - -# OpenCode Daily Report - 2026-06-04 - -## Summary - -- Sessions: total 1 - -## Problems / Improvement Candidates - -- No notable signals. Current values can be used as baseline. -""" - - -# --------------------------------------------------------------------------- -# submit_page_and_metrics -# --------------------------------------------------------------------------- - - -def test_submit_page_and_metrics_creates_page_and_metrics( - database_url: str, data_dir: Path -) -> None: - from kb.cli._submit import submit_page_and_metrics - - submit_page_and_metrics( - report=_SUMMARY_MD, - export_path="wiki/summaries/2026/06/2026-06-04-x-usage.md", - slug="2026-06-04-x-usage", - report_date="2026-06-04", - report_type="opencode", - metrics={"k": 1}, - token_total=5, - ) - - # Assert via a fresh session that the DB rows exist. - from kb.db import make_engine, make_session_factory - - engine = make_engine() - factory = make_session_factory(engine) - sess = factory() - try: - page = sess.execute( - select(Page).where(Page.slug == "2026-06-04-x-usage") - ).scalar_one() - assert page.slug == "2026-06-04-x-usage" - - metrics_row = sess.execute( - select(MetricsRecord).where( - MetricsRecord.report_date == "2026-06-04", - MetricsRecord.report_type == "opencode", - ) - ).scalar_one() - assert metrics_row.token_total == 5 - finally: - sess.close() - engine.dispose() - - # Assert the export file exists on disk. - export_file = data_dir / "wiki/summaries/2026/06/2026-06-04-x-usage.md" - assert export_file.exists(), f"Export file not found: {export_file}" - - -# --------------------------------------------------------------------------- -# db_ttl_sweep.main -# --------------------------------------------------------------------------- - - -def test_db_ttl_sweep_main_rejects_stale_page( - database_url: str, data_dir: Path -) -> None: - import kb.cli.db_ttl_sweep as ttl_mod - - # Create a stale not_processed page via the service layer (same session scope). - from kb.db import make_engine, make_session_factory - - engine = make_engine() - factory = make_session_factory(engine) - sess = factory() - try: - stale_fm = { - "type": "concept", - "review_status": "not_processed", - "created": "2026-01-01", - "updated": "2026-01-01", - "sources": [], - "tags": ["stale"], - } - service_pages.upsert_page( - sess, - data_dir, - slug="stale-sweep-page", - type="concept", - body_md="\n# Stale Sweep Page\n\nA stale not_processed concept for CLI sweep test.\n", - frontmatter=stale_fm, - export_path="wiki/concepts/stale-sweep-page.md", - ) - finally: - sess.close() - engine.dispose() - - # Call the CLI main — it uses session_scope() which reads DATABASE_URL / KB_DATA_DIR. - rc = ttl_mod.main(["--days", "7"]) - assert rc == 0 - - # Verify the page is now rejected in DB. - engine2 = make_engine() - factory2 = make_session_factory(engine2) - sess2 = factory2() - try: - page = sess2.execute( - select(Page).where(Page.slug == "stale-sweep-page") - ).scalar_one() - assert page.review_status == "rejected" - finally: - sess2.close() - engine2.dispose() - - -# --------------------------------------------------------------------------- -# submit_cron_run.main -# --------------------------------------------------------------------------- - - -def test_submit_cron_run_main_creates_cron_run_row( - database_url: str, data_dir: Path, tmp_path: Path -) -> None: - import kb.cli.submit_cron_run as scr_mod - - logfile = tmp_path / "test.log" - logfile.write_text("log line 1\nlog line 2\n", encoding="utf-8") - - rc = scr_mod.main( - [ - "--job-name", - "x", - "--target", - "y", - "--status", - "success", - "--exit-code", - "0", - "--log-file", - str(logfile), - ] - ) - assert rc == 0 - - from kb.db import make_engine, make_session_factory - - engine = make_engine() - factory = make_session_factory(engine) - sess = factory() - try: - row = sess.execute(select(CronRun).where(CronRun.job_name == "x")).scalar_one() - assert row.job_name == "x" - assert row.target == "y" - assert row.status == "success" - assert row.exit_code == 0 - finally: - sess.close() - engine.dispose() diff --git a/test/test_payloads.py b/test/test_payloads.py deleted file mode 100644 index 0460019..0000000 --- a/test/test_payloads.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Tests for kb.cli._payloads — pure payload parsers extracted from db_api.py.""" - -from __future__ import annotations - -import pytest - -from kb.cli._payloads import ( - PayloadError, - _split_frontmatter, - markdown_page_payload, - raw_source_payload, -) - -# --------------------------------------------------------------------------- -# _split_frontmatter error cases -# --------------------------------------------------------------------------- - - -def test_split_frontmatter_missing_opening_delimiter() -> None: - """Markdown without leading '---\\n' raises PayloadError.""" - with pytest.raises(PayloadError, match="frontmatter"): - _split_frontmatter("# No frontmatter here\n\nBody.\n") - - -def test_split_frontmatter_missing_closing_delimiter() -> None: - """Markdown with only an opening delimiter but no closing '---' raises PayloadError.""" - with pytest.raises(PayloadError, match="closing delimiter"): - _split_frontmatter("---\ntype: summary\nno closing delimiter\n") - - -# --------------------------------------------------------------------------- -# markdown_page_payload -# --------------------------------------------------------------------------- - - -_FULL_MD = """\ ---- -type: summary -subtype: daily -date: "2026-06-04" -sources: [] -tags: [agent-usage] ---- - -# Daily Report - -Body. -""" - - -def test_markdown_page_payload_preserves_frontmatter_and_body() -> None: - """Ported from test_db_canonical_api.py — import now comes from _payloads.""" - payload = markdown_page_payload( - markdown=_FULL_MD, - export_path="wiki/summaries/2026/06/2026-06-04-usage.md", - slug="2026-06-04-usage", - origin="ingested", - source="test", - ) - assert payload["frontmatter"] == { - "type": "summary", - "subtype": "daily", - "date": "2026-06-04", - "sources": [], - "tags": ["agent-usage"], - } - assert payload["body_md"].startswith("\n\n# Daily Report") - assert payload["export_path"] == "wiki/summaries/2026/06/2026-06-04-usage.md" - assert payload["slug"] == "2026-06-04-usage" - assert payload["type"] == "summary" - assert payload["origin"] == "ingested" - assert payload["source"] == "test" - - -def test_markdown_page_payload_missing_type_raises() -> None: - md = """\ ---- -subtype: daily -sources: [] ---- - -# No type field -""" - with pytest.raises(PayloadError, match="type"): - markdown_page_payload( - markdown=md, - export_path="wiki/x.md", - slug="x", - origin="ingested", - source="cli", - ) - - -# --------------------------------------------------------------------------- -# raw_source_payload -# --------------------------------------------------------------------------- - - -_RAW_MD = """\ ---- -type: web_article -source_url: https://example.com/paper -captured_at: "2026-06-04T00:00:00Z" -contributor: cli ---- - -# Example Paper - -Abstract text here. -""" - - -def test_raw_source_payload_returns_expected_keys() -> None: - payload = raw_source_payload( - markdown=_RAW_MD, - source_key="raw/web/huggingface/daily_papers_2026-06-04.md", - source_type="web_article", - ) - assert payload["source_key"] == "raw/web/huggingface/daily_papers_2026-06-04.md" - assert payload["source_type"] == "web_article" - assert payload["source_url"] == "https://example.com/paper" - assert payload["captured_at"] == "2026-06-04T00:00:00Z" - assert payload["title"] == "Example Paper" - assert "content_md" in payload - assert "frontmatter" in payload - - -def test_raw_source_payload_falls_back_to_frontmatter_type() -> None: - md = """\ ---- -type: github_issue -source_url: https://github.com/org/repo/issues/1 -captured_at: "2026-06-04T00:00:00Z" -contributor: cli ---- - -# Some Issue -""" - payload = raw_source_payload( - markdown=md, - source_key="raw/github/issues/repo_1.md", - ) - assert payload["source_type"] == "github_issue" - - -def test_raw_source_payload_missing_frontmatter_raises() -> None: - with pytest.raises(PayloadError): - raw_source_payload( - markdown="# No frontmatter\n\nJust body.\n", - source_key="raw/x.md", - ) diff --git a/test/test_usage_daily_report.py b/test/test_usage_daily_report.py deleted file mode 100644 index 5b60d78..0000000 --- a/test/test_usage_daily_report.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import annotations - -import json -import sqlite3 -from pathlib import Path - -from kb.cli.hermes_daily_report import ( - collect_metrics as collect_hermes_metrics, - render_report as render_hermes_report, -) -from kb.cli.opencode_daily_report import ( - collect_metrics as collect_opencode_metrics, - render_report as render_opencode_report, -) - - -def make_opencode_db(path: Path) -> None: - con = sqlite3.connect(path) - con.executescript(""" - CREATE TABLE project (id text PRIMARY KEY, worktree text, name text, time_created integer, time_updated integer); - CREATE TABLE session ( - id text PRIMARY KEY, - project_id text NOT NULL, - parent_id text, - title text NOT NULL, - directory text NOT NULL, - time_created integer NOT NULL, - time_updated integer NOT NULL, - summary_files INTEGER DEFAULT 0, - time_compacting INTEGER, - model TEXT - ); - CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL); - CREATE TABLE part (id text PRIMARY KEY, message_id text NOT NULL, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL); - """) - # 2026-05-14 01:00 KST == 2026-05-13 16:00 UTC in ms. - t = 1778688000000 - con.execute("INSERT INTO project VALUES ('p1', '/repo', 'repo', ?, ?)", (t, t)) - con.execute( - "INSERT INTO session VALUES ('s1', 'p1', NULL, 'root', '/repo', ?, ?, 1, NULL, 'claude-sonnet-4-6')", - (t, t + 60000), - ) - con.execute( - "INSERT INTO session VALUES ('s2', 'p1', 's1', 'sub', '/repo', ?, ?, 0, NULL, 'claude-sonnet-4-6')", - (t + 1000, t + 31000), - ) - con.execute( - "INSERT INTO message VALUES ('m1', 's1', ?, ?, ?)", - ( - t, - t, - json.dumps( - { - "role": "assistant", - "modelID": "claude-sonnet-4-6", - "providerID": "anthropic", - "tokens": { - "input": 10, - "output": 5, - "reasoning": 2, - "cache": {"read": 90, "write": 3}, - }, - "cost": 0.12, - } - ), - ), - ) - con.execute( - "INSERT INTO message VALUES ('m2', 's1', ?, ?, ?)", - (t, t, json.dumps({"role": "user"})), - ) - con.execute( - "INSERT INTO part VALUES ('pt1', 'm1', 's1', ?, ?, ?)", - ( - t, - t, - json.dumps( - { - "type": "tool", - "tool": "bash", - "state": {"status": "completed", "input": {"command": "pwd"}}, - } - ), - ), - ) - con.execute( - "INSERT INTO part VALUES ('pt2', 'm1', 's1', ?, ?, ?)", - ( - t, - t, - json.dumps( - { - "type": "tool", - "tool": "read", - "state": { - "status": "error", - "input": {"filePath": "x", "bogus": True}, - }, - } - ), - ), - ) - con.commit() - con.close() - - -def make_hermes_db(path: Path) -> None: - con = sqlite3.connect(path) - con.executescript(""" - CREATE TABLE sessions ( - id TEXT PRIMARY KEY, - source TEXT NOT NULL, - model TEXT, - parent_session_id TEXT, - started_at REAL NOT NULL, - ended_at REAL, - end_reason TEXT, - message_count INTEGER DEFAULT 0, - tool_call_count INTEGER DEFAULT 0, - input_tokens INTEGER DEFAULT 0, - output_tokens INTEGER DEFAULT 0, - cache_read_tokens INTEGER DEFAULT 0, - cache_write_tokens INTEGER DEFAULT 0, - reasoning_tokens INTEGER DEFAULT 0, - actual_cost_usd REAL, - estimated_cost_usd REAL, - billing_provider TEXT - ); - CREATE TABLE messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - content TEXT, - tool_calls TEXT, - tool_name TEXT, - timestamp REAL NOT NULL - ); - """) - # 2026-05-14 01:00 KST in seconds. - t = 1778688000 - con.execute( - "INSERT INTO sessions VALUES ('h1', 'cron', 'gpt-5.5', NULL, ?, ?, 'cron_complete', 4, 2, 100, 20, 300, 0, 5, 0.5, NULL, 'custom')", - (t, t + 120), - ) - con.execute( - "INSERT INTO messages(session_id, role, content, tool_calls, tool_name, timestamp) VALUES (?, ?, ?, ?, ?, ?)", - ( - "h1", - "assistant", - "", - json.dumps( - [ - { - "type": "function", - "function": { - "name": "terminal", - "arguments": json.dumps({"command": "date", "extra": 1}), - }, - } - ] - ), - None, - t, - ), - ) - con.commit() - con.close() - - -def test_collect_split_usage_metrics_counts_non_deferred_metrics(tmp_path): - opencode_db = tmp_path / "opencode.db" - hermes_db = tmp_path / "state.db" - make_opencode_db(opencode_db) - make_hermes_db(hermes_db) - - opencode_metrics = collect_opencode_metrics("2026-05-14", opencode_db) - hermes_metrics = collect_hermes_metrics("2026-05-14", hermes_db) - - assert opencode_metrics["deferred_metrics"] == [ - "Task Completion Rate", - "pass@k", - "pass^k", - ] - assert opencode_metrics["opencode"]["sessions"]["total"] == 2 - assert opencode_metrics["opencode"]["sessions"]["root"] == 1 - assert opencode_metrics["opencode"]["error_rate"]["tool_errors"] == 1 - assert opencode_metrics["opencode"]["error_rate"]["tool_calls"] == 2 - assert opencode_metrics["opencode"]["tool_schema"]["invalid_calls"] == 1 - assert opencode_metrics["opencode"]["n_turns"] == 2 - assert opencode_metrics["opencode"]["tokens"]["total"] == 110 - assert opencode_metrics["opencode"]["cost"]["recorded_usd"] == 0.12 - assert hermes_metrics["hermes"]["sessions"]["root"] == 1 - assert hermes_metrics["hermes"]["tool_schema"]["invalid_calls"] == 1 - assert hermes_metrics["hermes"]["latency"]["avg_session_sec"] == 120 - - -def test_render_split_daily_reports_use_fixed_evaluation_layout(tmp_path): - opencode_db = tmp_path / "opencode.db" - hermes_db = tmp_path / "state.db" - make_opencode_db(opencode_db) - make_hermes_db(hermes_db) - opencode_metrics = collect_opencode_metrics("2026-05-14", opencode_db) - hermes_metrics = collect_hermes_metrics("2026-05-14", hermes_db) - - opencode_report = render_opencode_report(opencode_metrics) - hermes_report = render_hermes_report(hermes_metrics) - - for report in (opencode_report, hermes_report): - assert "## Development/Evaluation Metrics" in report - assert "| Error Rate |" in report - assert "| Hallucinated Parameters |" in report - assert "| n_toolcalls / n_turns |" in report - assert "| Policy Compliance Rate |" in report - assert "Task Completion Rate" in report - assert report.count("## Development/Evaluation Metrics") == 1 From b105149e66433c0525039051ac0705f0efe32dd5 Mon Sep 17 00:00:00 2001 From: yw0nam Date: Fri, 10 Jul 2026 11:02:35 +0900 Subject: [PATCH 2/9] refactor: remove cron/metrics/review_status from source and tests Co-Authored-By: Claude Opus 4.8 (1M context) --- src/kb/db/__init__.py | 4 - src/kb/db/models.py | 35 +----- src/kb/lint/common.py | 15 +-- src/kb/lint/wiki.py | 18 --- src/kb/mcp/tools_read.py | 5 +- src/kb/mcp/tools_write.py | 185 +------------------------------ src/kb/service/_helpers.py | 1 - src/kb/service/export.py | 37 +------ src/kb/service/ops.py | 77 +------------ src/kb/service/pages.py | 176 +---------------------------- test/test_db_init.py | 5 +- test/test_mcp_read_tools.py | 3 +- test/test_mcp_write_tools.py | 52 +-------- test/test_service_ops.py | 208 +---------------------------------- test/test_service_pages.py | 101 +---------------- test/test_wiki_note_skill.py | 1 - 16 files changed, 31 insertions(+), 892 deletions(-) diff --git a/src/kb/db/__init__.py b/src/kb/db/__init__.py index 6b84a24..8c6358a 100644 --- a/src/kb/db/__init__.py +++ b/src/kb/db/__init__.py @@ -15,11 +15,9 @@ from kb.db.models import ( Base, - CronRun, Dispatch, ExportRecord, Handoff, - MetricsRecord, OperationLog, Page, PageRevision, @@ -29,11 +27,9 @@ __all__ = [ "Base", - "CronRun", "Dispatch", "ExportRecord", "Handoff", - "MetricsRecord", "OperationLog", "Page", "PageRevision", diff --git a/src/kb/db/models.py b/src/kb/db/models.py index bfd3515..9b13211 100644 --- a/src/kb/db/models.py +++ b/src/kb/db/models.py @@ -6,7 +6,7 @@ from __future__ import annotations -from sqlalchemy import JSON, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy import JSON, ForeignKey, Integer, String, Text from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -39,7 +39,6 @@ class Page(Base): title: Mapped[str] = mapped_column(String, nullable=False) type: Mapped[str] = mapped_column(String, nullable=False) category: Mapped[str | None] = mapped_column(String, nullable=True) - review_status: Mapped[str | None] = mapped_column(String, nullable=True) origin: Mapped[str] = mapped_column(String, nullable=False, default="ingested") body_md: Mapped[str] = mapped_column(Text, nullable=False) frontmatter: Mapped[dict] = mapped_column(JSON, nullable=False) @@ -116,21 +115,6 @@ class OperationLog(Base): created_at: Mapped[str] = mapped_column(String, nullable=False) -class CronRun(Base): - __tablename__ = "cron_runs" - - id: Mapped[int] = mapped_column(Integer, primary_key=True) - job_name: Mapped[str] = mapped_column(String, nullable=False) - target: Mapped[str] = mapped_column(String, nullable=False) - status: Mapped[str] = mapped_column(String, nullable=False) - exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True) - log_body: Mapped[str] = mapped_column(Text, nullable=False) - log_path: Mapped[str | None] = mapped_column(String, nullable=True) - started_at: Mapped[str | None] = mapped_column(String, nullable=True) - finished_at: Mapped[str | None] = mapped_column(String, nullable=True) - created_at: Mapped[str] = mapped_column(String, nullable=False) - - class ExportRecord(Base): __tablename__ = "exports" @@ -139,20 +123,3 @@ class ExportRecord(Base): status: Mapped[str] = mapped_column(String, nullable=False) message: Mapped[str | None] = mapped_column(Text, nullable=True) exported_at: Mapped[str] = mapped_column(String, nullable=False) - - -class MetricsRecord(Base): - __tablename__ = "metrics" - __table_args__ = ( - UniqueConstraint("report_date", "report_type", name="uq_metrics_date_type"), - ) - - id: Mapped[int] = mapped_column(Integer, primary_key=True) - report_date: Mapped[str] = mapped_column(String, nullable=False) - report_type: Mapped[str] = mapped_column(String, nullable=False) - session_count: Mapped[int | None] = mapped_column(Integer, nullable=True) - token_total: Mapped[int | None] = mapped_column(Integer, nullable=True) - cost_usd: Mapped[float | None] = mapped_column(nullable=True) - tool_error_count: Mapped[int | None] = mapped_column(Integer, nullable=True) - metrics_json: Mapped[dict] = mapped_column(JSON, nullable=False) - created_at: Mapped[str] = mapped_column(String, nullable=False) diff --git a/src/kb/lint/common.py b/src/kb/lint/common.py index b171522..fc59086 100644 --- a/src/kb/lint/common.py +++ b/src/kb/lint/common.py @@ -3,10 +3,6 @@ from __future__ import annotations import re -REVIEW_STATUS_VALUES = frozenset({"not_processed", "pending_for_approve", "approved"}) -REVIEW_STATUS_TYPES = frozenset( - {"entity", "concept", "decision", "improvement", "checklist", "question"} -) IMPROVEMENT_KIND_VALUES = frozenset({"improvement", "issue", "proposal"}) IMPROVEMENT_DOMAIN_VALUES = frozenset({"cost", "correctness", "perf", "dx", "security"}) IMPROVEMENT_SEVERITY_VALUES = frozenset({"low", "med", "high"}) @@ -17,12 +13,11 @@ ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") REQUIRED_FM_FIELDS = { - "entity": ["type", "review_status", "created", "updated", "sources", "tags"], - "concept": ["type", "review_status", "created", "updated", "sources", "tags"], - "decision": ["type", "review_status", "created", "updated", "sources", "tags"], + "entity": ["type", "created", "updated", "sources", "tags"], + "concept": ["type", "created", "updated", "sources", "tags"], + "decision": ["type", "created", "updated", "sources", "tags"], "improvement": [ "type", - "review_status", "kind", "observed_at", "domain", @@ -34,9 +29,9 @@ "sources", "tags", ], - "checklist": ["type", "review_status", "created", "updated", "sources", "tags"], + "checklist": ["type", "created", "updated", "sources", "tags"], "summary": ["type", "created", "updated", "sources", "tags"], - "question": ["type", "review_status", "created", "updated", "sources", "tags"], + "question": ["type", "created", "updated", "sources", "tags"], } STUB_THRESHOLD_CHARS = 100 diff --git a/src/kb/lint/wiki.py b/src/kb/lint/wiki.py index 95fc114..4a66c9a 100644 --- a/src/kb/lint/wiki.py +++ b/src/kb/lint/wiki.py @@ -14,8 +14,6 @@ IMPROVEMENT_SEVERITY_VALUES, ISO_DATE_RE, REQUIRED_FM_FIELDS, - REVIEW_STATUS_TYPES, - REVIEW_STATUS_VALUES, STUB_THRESHOLD_CHARS, LintResult, extract_wikilinks, @@ -65,9 +63,6 @@ def validate_page_create( if page_type == "checklist": _validate_checklist_items(result, identifier, body_md) - if page_type in REVIEW_STATUS_TYPES: - _validate_review_status(result, identifier, fm) - _validate_wikilinks(result, identifier, body_md, session, slug) _validate_stale_sources(result, identifier, fm, session) @@ -120,17 +115,6 @@ def validate_page_full(session: Session) -> LintResult: return result -def _validate_review_status(result: LintResult, identifier: str, fm: dict) -> None: - rs = fm.get("review_status") - if rs is None: - return - if rs not in REVIEW_STATUS_VALUES: - result.error( - identifier, - f"invalid review_status: {rs!r} (must be one of {sorted(REVIEW_STATUS_VALUES)})", - ) - - def _validate_improvement_fm( result: LintResult, identifier: str, @@ -325,6 +309,4 @@ def _should_skip_orphan(page) -> bool: return True if page.type == "summary": return True - if page.review_status and page.review_status != "approved": - return True return False diff --git a/src/kb/mcp/tools_read.py b/src/kb/mcp/tools_read.py index 06e515d..8aaa5b4 100644 --- a/src/kb/mcp/tools_read.py +++ b/src/kb/mcp/tools_read.py @@ -46,7 +46,7 @@ def query_sql(ctx: Context, sql: str | None = None, limit: int = 100) -> dict: """Run a READ-ONLY SQL query (SELECT / WITH only) against the KnowledgeBase DB. Use this to inspect the canonical Postgres store: pages, raw_sources, - handoffs, operation_logs, cron_runs, metrics, etc. Call ``get_schema`` first + handoffs, operation_logs, etc. Call ``get_schema`` first to learn the tables and columns. Rules: @@ -144,8 +144,7 @@ def get_schema(ctx: Context) -> dict: return { "tables": tables, "examples": [ - "SELECT slug, type, review_status FROM pages " - "WHERE review_status='pending_for_approve';", + "SELECT slug, type FROM pages ORDER BY updated_at DESC LIMIT 20;", "SELECT source_key, source_type, captured_at FROM raw_sources " "ORDER BY created_at DESC LIMIT 20;", "SELECT handoff_id, task_slug, status FROM handoffs " diff --git a/src/kb/mcp/tools_write.py b/src/kb/mcp/tools_write.py index 62d85c6..07b26c6 100644 --- a/src/kb/mcp/tools_write.py +++ b/src/kb/mcp/tools_write.py @@ -95,7 +95,6 @@ def upsert_page( export_path: NullableStr = None, title: NullableStr = None, category: NullableStr = None, - review_status: NullableStr = None, origin: str = "ingested", created_at: NullableStr = None, updated_at: NullableStr = None, @@ -106,7 +105,7 @@ def upsert_page( Required: slug (unique page identifier), type (e.g. "concept", "summary"), body_md (Markdown content), frontmatter (dict with at minimum type, created, updated, sources, tags fields), export_path (relative path under data/). - Optional: title, category, review_status, origin, created_at, updated_at, source. + Optional: title, category, origin, created_at, updated_at, source. Returns export status dict; raises lint_failed or conflict on validation errors. """ missing = require( @@ -130,7 +129,6 @@ def upsert_page( export_path=export_path, title=title, category=category, - review_status=review_status, origin=origin, created_at=created_at, updated_at=updated_at, @@ -148,14 +146,13 @@ def patch_page( body_md: NullableStr = None, frontmatter: dict | None = None, category: NullableStr = None, - review_status: NullableStr = None, source: str = "agent", note: NullableStr = None, ) -> dict: """Partially update an existing wiki page (only provided fields are changed). Required: slug (page to update). - Optional: title, body_md, frontmatter, category, review_status, source, note. + Optional: title, body_md, frontmatter, category, source, note. Returns export status dict; raises not_found if slug doesn't exist. """ missing = require(slug=slug) @@ -171,7 +168,6 @@ def patch_page( body_md=body_md, frontmatter=frontmatter, category=category, - review_status=review_status, source=source, note=note, ) @@ -179,98 +175,6 @@ def patch_page( return {"error": str(exc), "code": exc.code, "detail": exc.detail} -@mcp.tool -def promote_page( - ctx: Context, - slug: NullableStr = None, - feedback: str = "", - source: str = "console", -) -> dict: - """Advance a wiki page from not_processed → pending_for_approve. - - Required: slug. Optional: feedback (review notes), source. - Returns updated page dict; raises conflict if page is not in not_processed state. - """ - missing = require(slug=slug) - if missing: - return missing - try: - with tool_session(ctx) as (session, data_dir): - return service_pages.promote_page( - session, data_dir, slug=slug, feedback=feedback, source=source - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - -@mcp.tool -def approve_page( - ctx: Context, - slug: NullableStr = None, - feedback: str = "", - source: str = "console", -) -> dict: - """Approve a wiki page (pending_for_approve → approved). - - Required: slug. Optional: feedback (review notes), source. - Returns updated page dict; raises conflict if page is not in pending_for_approve state. - """ - missing = require(slug=slug) - if missing: - return missing - try: - with tool_session(ctx) as (session, data_dir): - return service_pages.approve_page( - session, data_dir, slug=slug, feedback=feedback, source=source - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - -@mcp.tool -def reject_page( - ctx: Context, - slug: NullableStr = None, - feedback: str = "", - source: str = "console", -) -> dict: - """Reject a wiki page (pending_for_approve or not_processed → rejected). - - Required: slug. Optional: feedback (rejection reason), source. - Returns the updated page dict with export_path rewritten under rejected/. - Raises a conflict error if the page is already approved or rejected. - """ - missing = require(slug=slug) - if missing: - return missing - try: - with tool_session(ctx) as (session, data_dir): - return service_pages.reject_page( - session, data_dir, slug=slug, feedback=feedback, source=source - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - -@mcp.tool -def ttl_sweep_pages( - ctx: Context, - days: NullableInt = None, -) -> dict: - """Reject all not_processed pages older than `days` days (default 7). - - Pass days=None or omit to use the 7-day default. - Returns ``{"swept": int}`` with the number of pages rejected. - """ - try: - with tool_session(ctx) as (session, data_dir): - return service_pages.ttl_sweep( - session, data_dir, days=days if days is not None else 7 - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - # --------------------------------------------------------------------------- # Handoffs # --------------------------------------------------------------------------- @@ -368,91 +272,6 @@ def create_operation_log( return {"error": str(exc), "code": exc.code, "detail": exc.detail} -@mcp.tool -def create_cron_run( - ctx: Context, - job_name: NullableStr = None, - target: NullableStr = None, - status: NullableStr = None, - log_body: NullableStr = None, - exit_code: NullableInt = None, - log_path: NullableStr = None, - started_at: NullableStr = None, - finished_at: NullableStr = None, - created_at: NullableStr = None, -) -> dict: - """Record a cron job execution and export to Markdown. - - Required: job_name (e.g. "kb-memory-daily"), target (date or identifier, - e.g. "2026-06-12"), status (e.g. "success", "failed"), log_body (log content). - Optional: exit_code, log_path, started_at, finished_at (ISO), created_at. - """ - missing = require( - job_name=job_name, target=target, status=status, log_body=log_body - ) - if missing: - return missing - try: - with tool_session(ctx) as (session, data_dir): - return service_ops.create_cron_run( - session, - data_dir, - job_name=job_name, - target=target, - status=status, - log_body=log_body, - exit_code=exit_code, - log_path=log_path, - started_at=started_at, - finished_at=finished_at, - created_at=created_at, - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - -@mcp.tool -def upsert_metrics( - ctx: Context, - report_date: NullableStr = None, - report_type: NullableStr = None, - metrics_json: dict | None = None, - session_count: NullableInt = None, - token_total: NullableInt = None, - cost_usd: float | None = None, - tool_error_count: NullableInt = None, -) -> dict: - """Insert or update a MetricsRecord for (report_date, report_type). - - Required: report_date (ISO date, e.g. "2026-06-12"), report_type - (e.g. "opencode", "hermes", "claude_code"), metrics_json (raw metrics dict). - Optional: session_count, token_total, cost_usd, tool_error_count. - If a row already exists for the (report_date, report_type) pair it is updated. - """ - missing = require( - report_date=report_date, - report_type=report_type, - metrics_json=metrics_json, - ) - if missing: - return missing - try: - with tool_session(ctx) as (session, data_dir): - return service_ops.upsert_metrics( - session, - data_dir, - report_date=report_date, - report_type=report_type, - metrics_json=metrics_json, - session_count=session_count, - token_total=token_total, - cost_usd=cost_usd, - tool_error_count=tool_error_count, - ) - except ServiceError as exc: - return {"error": str(exc), "code": exc.code, "detail": exc.detail} - - @mcp.tool def export_markdown(ctx: Context) -> dict: """Export all canonical DB rows to Markdown/JSON files under data_dir. diff --git a/src/kb/service/_helpers.py b/src/kb/service/_helpers.py index efab0f6..d16ecef 100644 --- a/src/kb/service/_helpers.py +++ b/src/kb/service/_helpers.py @@ -32,7 +32,6 @@ def _page_payload(row: Page) -> dict: "title": row.title, "type": row.type, "category": row.category, - "review_status": row.review_status, "origin": row.origin, "frontmatter": row.frontmatter, "body": row.body_md, diff --git a/src/kb/service/export.py b/src/kb/service/export.py index ed539d9..24544c8 100644 --- a/src/kb/service/export.py +++ b/src/kb/service/export.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path import yaml @@ -10,10 +9,8 @@ from sqlalchemy.orm import Session from kb.db.models import ( - CronRun, ExportRecord, Handoff, - MetricsRecord, OperationLog, Page, RawSource, @@ -38,7 +35,7 @@ def _write(path: Path, text: str) -> None: def _clean_stale_markdown(data_dir: Path, expected: set[str]) -> int: """Remove .md files under wiki/ and handoffs/ not present in expected paths.""" removed = 0 - for top in ("wiki", "rejected", "handoffs"): + for top in ("wiki", "handoffs"): root = data_dir / top if not root.is_dir(): continue @@ -48,7 +45,7 @@ def _clean_stale_markdown(data_dir: Path, expected: set[str]) -> int: continue md.unlink() removed += 1 - for top in ("wiki", "rejected", "handoffs"): + for top in ("wiki", "handoffs"): root = data_dir / top if not root.is_dir(): continue @@ -62,7 +59,7 @@ def export_all(session: Session, data_dir: Path) -> int: """Export canonical DB rows to Markdown/JSON files. Export is derived output. Stale files that have no corresponding DB row - are cleaned up automatically under wiki/, rejected/, and handoffs/. + are cleaned up automatically under wiki/ and handoffs/. """ written = 0 expected: set[str] = set() @@ -97,26 +94,6 @@ def export_all(session: Session, data_dir: Path) -> int: expected.add(handoff.export_path) written += 1 - for metrics in session.execute( - select(MetricsRecord).order_by( - MetricsRecord.report_date, MetricsRecord.report_type - ) - ).scalars(): - year, month, _ = metrics.report_date.split("-") - path_str = ( - f"ops/reports/{year}/{month}" - f"/{metrics.report_date}-{metrics.report_type}-usage.metrics.json" - ) - _write( - data_dir / path_str, - json.dumps( - metrics.metrics_json, ensure_ascii=False, indent=2, sort_keys=True - ) - + "\n", - ) - expected.add(path_str) - written += 1 - logs = list( session.execute( select(OperationLog).order_by(OperationLog.log_date, OperationLog.id) @@ -130,14 +107,6 @@ def export_all(session: Session, data_dir: Path) -> int: expected.add("log.md") written += 1 - for run in session.execute( - select(CronRun).where(CronRun.log_path.is_not(None)) - ).scalars(): - if run.log_path: - _write(data_dir / run.log_path, run.log_body) - expected.add(run.log_path) - written += 1 - cleaned = _clean_stale_markdown(data_dir, expected) msg = f"wrote {written} files, cleaned {cleaned} stale" diff --git a/src/kb/service/ops.py b/src/kb/service/ops.py index 89fde17..4aa0ad1 100644 --- a/src/kb/service/ops.py +++ b/src/kb/service/ops.py @@ -1,4 +1,4 @@ -"""Service functions for operational records (OperationLog, CronRun, MetricsRecord). +"""Service functions for operational records (OperationLog). Extracted from ``kb.web.routes.db_canonical`` route handlers with HTTP specifics removed. HTTP exceptions become ``ServiceError``; ``data_dir`` @@ -9,10 +9,9 @@ from pathlib import Path -from sqlalchemy import select from sqlalchemy.orm import Session -from kb.db.models import CronRun, MetricsRecord, OperationLog +from kb.db.models import OperationLog from kb.service._helpers import commit_and_export from kb.service._time import now_iso_kst from kb.service.errors import ServiceError @@ -40,78 +39,6 @@ def create_operation_log( return commit_and_export(session, data_dir, {"id": row.id}) -def create_cron_run( - session: Session, - data_dir: Path, - *, - job_name: str, - target: str, - status: str, - log_body: str, - exit_code: int | None = None, - log_path: str | None = None, - started_at: str | None = None, - finished_at: str | None = None, - created_at: str | None = None, -) -> dict: - """Insert a CronRun row and export.""" - row = CronRun( - job_name=job_name, - target=target, - status=status, - exit_code=exit_code, - log_body=log_body, - log_path=log_path, - started_at=started_at, - finished_at=finished_at, - created_at=created_at or now_iso_kst(), - ) - session.add(row) - session.flush() - return commit_and_export(session, data_dir, {"id": row.id}) - - -def upsert_metrics( - session: Session, - data_dir: Path, - *, - report_date: str, - report_type: str, - metrics_json: dict, - session_count: int | None = None, - token_total: int | None = None, - cost_usd: float | None = None, - tool_error_count: int | None = None, -) -> dict: - """Insert or update a MetricsRecord for (report_date, report_type). - - If a row already exists for the given (report_date, report_type) pair it is - updated in place; otherwise a new row is created. Latest values always win. - On update, all scalar columns are overwritten — passing None for a field - clears it. - """ - row = session.execute( - select(MetricsRecord).where( - MetricsRecord.report_date == report_date, - MetricsRecord.report_type == report_type, - ) - ).scalar_one_or_none() - if row is None: - row = MetricsRecord( - report_date=report_date, - report_type=report_type, - created_at=now_iso_kst(), - ) - session.add(row) - row.session_count = session_count - row.token_total = token_total - row.cost_usd = cost_usd - row.tool_error_count = tool_error_count - row.metrics_json = metrics_json - session.flush() - return commit_and_export(session, data_dir, {"id": row.id}) - - def export_markdown(session: Session, data_dir: Path) -> dict: """Export all canonical DB rows to Markdown/JSON files. diff --git a/src/kb/service/pages.py b/src/kb/service/pages.py index 7fc297a..9562bbb 100644 --- a/src/kb/service/pages.py +++ b/src/kb/service/pages.py @@ -24,7 +24,7 @@ _sync_page_frontmatter, commit_and_export, ) -from kb.service._time import date_from_iso, now_iso_kst, today_kst +from kb.service._time import now_iso_kst from kb.service.errors import ServiceError @@ -38,7 +38,6 @@ def upsert_page( frontmatter: dict, title: str | None = None, category: str | None = None, - review_status: str | None = None, origin: str = "ingested", export_path: str, created_at: str | None = None, @@ -47,9 +46,6 @@ def upsert_page( ) -> dict: """Insert or update a Page by slug; run lint before writing.""" now = now_iso_kst() - review_status = ( - review_status if review_status is not None else frontmatter.get("review_status") - ) title = title or frontmatter.get("title") or _first_heading(body_md, slug) category = category or frontmatter.get("category") @@ -63,7 +59,6 @@ def upsert_page( "title": title, "type": type, "category": category, - "review_status": review_status, "origin": origin, "body_md": body_md, "frontmatter": frontmatter, @@ -118,7 +113,6 @@ def patch_page( body_md: str | None = None, frontmatter: dict | None = None, category: str | None = None, - review_status: str | None = None, source: str = "agent", note: str | None = None, ) -> dict: @@ -134,7 +128,6 @@ def patch_page( old_body = page.body_md old_frontmatter = dict(page.frontmatter) old_category = page.category - old_review_status = page.review_status changed: dict[str, dict] = {} @@ -151,19 +144,12 @@ def patch_page( page.frontmatter = frontmatter if "category" in frontmatter: page.category = frontmatter.get("category") - if "review_status" in frontmatter: - page.review_status = frontmatter.get("review_status") if category is not None and category != old_category: changed["category"] = {"old": old_category, "new": category} page.category = category _sync_page_frontmatter(page, {"category": category}) - if review_status is not None and review_status != old_review_status: - changed["review_status"] = {"old": old_review_status, "new": review_status} - page.review_status = review_status - _sync_page_frontmatter(page, {"review_status": review_status}) - if not changed: return {"page": _page_payload(page), "export": {"status": "skipped"}} @@ -184,163 +170,3 @@ def patch_page( note=note, ) return commit_and_export(session, data_dir, {"page": _page_payload(page)}) - - -def _transition_page( - session: Session, - data_dir: Path, - *, - slug: str, - expected: str, - new_status: str, - change_kind: str, - source: str, - feedback: str = "", - extra_fm: dict | None = None, -) -> dict: - """Shared status-transition logic for promote/approve.""" - page = session.execute(select(Page).where(Page.slug == slug)).scalar_one_or_none() - if page is None: - raise ServiceError("not_found", "page not found") - if page.review_status != expected: - raise ServiceError( - "conflict", f"expected {expected}, current {page.review_status!r}" - ) - - changed = {"review_status": {"old": page.review_status, "new": new_status}} - page.review_status = new_status - fm_fields = {"review_status": new_status} - if extra_fm: - fm_fields.update(extra_fm) - _sync_page_frontmatter(page, fm_fields) - page.updated_at = now_iso_kst() - _append_revision( - session, - page, - change_kind=change_kind, - changed_fields=changed, - source=source, - note=feedback or None, - ) - return commit_and_export(session, data_dir, {"page": _page_payload(page)}) - - -def promote_page( - session: Session, - data_dir: Path, - *, - slug: str, - feedback: str = "", - source: str = "console", -) -> dict: - """Transition not_processed → pending_for_approve.""" - return _transition_page( - session, - data_dir, - slug=slug, - expected="not_processed", - new_status="pending_for_approve", - change_kind="update", - source=source, - feedback=feedback, - ) - - -def approve_page( - session: Session, - data_dir: Path, - *, - slug: str, - feedback: str = "", - source: str = "console", -) -> dict: - """Transition pending_for_approve → approved.""" - return _transition_page( - session, - data_dir, - slug=slug, - expected="pending_for_approve", - new_status="approved", - change_kind="approve", - source=source, - feedback=feedback, - extra_fm={"approved_at": now_iso_kst()}, - ) - - -def reject_page( - session: Session, - data_dir: Path, - *, - slug: str, - feedback: str = "", - source: str = "console", -) -> dict: - """Reject a page from pending_for_approve or not_processed.""" - page = session.execute(select(Page).where(Page.slug == slug)).scalar_one_or_none() - if page is None: - raise ServiceError("not_found", "page not found") - if page.review_status not in {"pending_for_approve", "not_processed"}: - raise ServiceError( - "conflict", f"reject not allowed from {page.review_status!r}" - ) - - old_status = page.review_status - page.review_status = "rejected" - if page.export_path and page.export_path.startswith("wiki/"): - page.export_path = "rejected/" + page.export_path.removeprefix("wiki/") - _sync_page_frontmatter( - page, - { - "review_status": "rejected", - "rejected_at": now_iso_kst(), - "rejected_by": source, - }, - ) - page.updated_at = now_iso_kst() - _append_revision( - session, - page, - change_kind="reject", - changed_fields={"review_status": {"old": old_status, "new": "rejected"}}, - source=source, - note=feedback or None, - ) - return commit_and_export(session, data_dir, {"page": _page_payload(page)}) - - -def ttl_sweep(session: Session, data_dir: Path, *, days: int = 7) -> dict: - """Reject all not_processed pages older than ``days`` days.""" - today = today_kst() - swept = 0 - for page in session.execute( - select(Page).where(Page.review_status == "not_processed") - ).scalars(): - created = str(page.frontmatter.get("created") or "")[:10] - if not created: - continue - age = (date_from_iso(today) - date_from_iso(created)).days - if age < days: - continue - old_status = page.review_status - page.review_status = "rejected" - if page.export_path and page.export_path.startswith("wiki/"): - page.export_path = "rejected/" + page.export_path.removeprefix("wiki/") - _sync_page_frontmatter( - page, - { - "review_status": "rejected", - "rejected_at": now_iso_kst(), - "rejected_by": "auto_ttl", - }, - ) - _append_revision( - session, - page, - change_kind="reject", - changed_fields={"review_status": {"old": old_status, "new": "rejected"}}, - source="system", - note=f"No promotion within {days}d window.", - ) - swept += 1 - return commit_and_export(session, data_dir, {"swept": swept}) diff --git a/test/test_db_init.py b/test/test_db_init.py index b98adec..e7fcaab 100644 --- a/test/test_db_init.py +++ b/test/test_db_init.py @@ -71,16 +71,15 @@ def test_canonical_append_only_triggers_raise(database_url): conn.execute( text( "INSERT INTO pages " - "(slug, title, type, review_status, body_md, frontmatter, " + "(slug, title, type, body_md, frontmatter, " "created_at, updated_at) " - "VALUES (:slug, :title, :type, :status, :body, (:frontmatter)::jsonb, " + "VALUES (:slug, :title, :type, :body, (:frontmatter)::jsonb, " ":created_at, :updated_at)" ), { "slug": "demo-page", "title": "Demo Page", "type": "decision", - "status": "pending_for_approve", "body": "page body", "frontmatter": "{}", "created_at": "2026-06-04T10:31:00+09:00", diff --git a/test/test_mcp_read_tools.py b/test/test_mcp_read_tools.py index 2ec6afa..a3ed19e 100644 --- a/test/test_mcp_read_tools.py +++ b/test/test_mcp_read_tools.py @@ -52,7 +52,6 @@ def _get_tool_fn(tool_name: str): _CONCEPT_FM = { "type": "concept", - "review_status": "not_processed", "created": "2026-06-04", "updated": "2026-06-04", "sources": [], @@ -212,7 +211,7 @@ def test_get_schema(database_url: str, tool_ctx: SimpleNamespace) -> None: fn = _get_tool_fn("get_schema") result = fn(tool_ctx) tables = result["tables"] - for name in ("pages", "raw_sources", "handoffs", "metrics"): + for name in ("pages", "raw_sources", "handoffs"): assert name in tables, f"{name} missing from schema: {list(tables)}" page_cols = tables["pages"]["columns"] diff --git a/test/test_mcp_write_tools.py b/test/test_mcp_write_tools.py index f4f53ea..6d0f4b0 100644 --- a/test/test_mcp_write_tools.py +++ b/test/test_mcp_write_tools.py @@ -1,11 +1,10 @@ """Tests for kb.mcp.tools_write — MCP tool wrappers over the service layer. -Covers the 5 minimum scenarios required by Task 6: +Covers these scenarios: 1. upsert_page happy path → export success, file on disk, DB row. 2. upsert_page missing required arg (slug) → require() error dict, no DB touch. -3. ServiceError surfaces as a dict: approve_page on not_processed slug → conflict. -4. create_raw_source round-trip → row + file. -5. Smoke: all 12 expected tool names are registered on mcp. +3. create_raw_source round-trip → row + file. +4. Smoke: all 6 expected tool names are registered on mcp. Tool invocation: FunctionTool.fn is the original Python function; call it directly with a fake context whose lifespan_context mimics the real server_lifespan dict. @@ -34,14 +33,8 @@ "create_raw_source", "upsert_page", "patch_page", - "promote_page", - "approve_page", - "reject_page", - "ttl_sweep_pages", "create_handoff", "create_operation_log", - "create_cron_run", - "upsert_metrics", "export_markdown", } @@ -81,7 +74,6 @@ def _get_tool_fn(tool_name: str): _CONCEPT_FM = { "type": "concept", - "review_status": "not_processed", "created": "2026-06-04", "updated": "2026-06-04", "sources": [], @@ -171,39 +163,7 @@ def test_upsert_page_missing_slug_returns_error( # --------------------------------------------------------------------------- -# 3. ServiceError surfaces as a dict — approve_page on not_processed slug -# --------------------------------------------------------------------------- - - -def test_approve_page_on_not_processed_returns_conflict_dict( - database_url: str, tool_ctx: SimpleNamespace -) -> None: - """approve_page on a not_processed page → ServiceError → dict with code='conflict'.""" - # First, create the page using the upsert tool so it's in not_processed state - ctx = tool_ctx - upsert_fn = _get_tool_fn("upsert_page") - upsert_fn( - ctx, - slug="approve-direct", - type="concept", - body_md="\n# Approve Direct\n\nCreated via tool to test conflict path.\n", - frontmatter=dict(_CONCEPT_FM), - export_path="wiki/concepts/approve-direct.md", - ) - - # Now try to approve directly (skipping promote) — must get conflict - approve_fn = _get_tool_fn("approve_page") - result = approve_fn(ctx, slug="approve-direct") - - assert "error" in result, f"Expected error key, got: {result}" - assert result.get("code") == "conflict", f"Expected code='conflict', got: {result}" - assert "detail" in result, f"Expected detail key, got: {result}" - - # Ensure no exception was raised — tool must return a dict, not raise - - -# --------------------------------------------------------------------------- -# 4. create_raw_source round-trip +# 3. create_raw_source round-trip # --------------------------------------------------------------------------- @@ -246,12 +206,12 @@ def test_create_raw_source_round_trip( # --------------------------------------------------------------------------- -# 5. Smoke — all 12 expected tool names registered on mcp +# 4. Smoke — all 6 expected tool names registered on mcp # --------------------------------------------------------------------------- def test_all_expected_tool_names_registered() -> None: - """All 12 write tool names must be a subset of the mcp registry. + """All 6 write tool names must be a subset of the mcp registry. Subset (not exact-set) so the assertion stays correct as later tasks add read tools (query_sql, get_schema) onto the same global ``mcp`` — pytest diff --git a/test/test_service_ops.py b/test/test_service_ops.py index 0ec14a4..41d92af 100644 --- a/test/test_service_ops.py +++ b/test/test_service_ops.py @@ -2,13 +2,12 @@ from __future__ import annotations -import json from pathlib import Path import pytest from sqlalchemy import select -from kb.db.models import CronRun, ExportRecord, MetricsRecord, OperationLog +from kb.db.models import ExportRecord, OperationLog from kb.service.errors import ServiceError # --------------------------------------------------------------------------- # @@ -59,211 +58,6 @@ def test_create_operation_log_with_created_at(data_dir: Path, session) -> None: assert row.created_at == "2026-06-04T00:00:00+09:00" -# --------------------------------------------------------------------------- # -# CronRun tests # -# --------------------------------------------------------------------------- # - - -def test_create_cron_run_success(data_dir: Path, session) -> None: - """create_cron_run writes DB row.""" - from kb.service.ops import create_cron_run - - result = create_cron_run( - session, - data_dir, - job_name="kb-memory-daily", - target="2026-06-04", - status="success", - log_body="All done.\n", - exit_code=0, - ) - - assert result["export"]["status"] == "success" - assert "id" in result - - row = session.execute(select(CronRun)).scalar_one() - assert row.job_name == "kb-memory-daily" - assert row.status == "success" - assert row.exit_code == 0 - - -def test_create_cron_run_with_log_path(data_dir: Path, session) -> None: - """When log_path is set the log file is written under data_dir.""" - from kb.service.ops import create_cron_run - - log_path = "raw/ops/cron/2026/06/2026-06-04_kb-memory-daily.log" - result = create_cron_run( - session, - data_dir, - job_name="kb-memory-daily", - target="2026-06-04", - status="success", - log_body="Log content here.\n", - exit_code=0, - log_path=log_path, - ) - - assert result["export"]["status"] == "success" - - log_file = data_dir / log_path - assert log_file.exists() - assert "Log content here." in log_file.read_text() - - -def test_create_cron_run_no_log_path(data_dir: Path, session) -> None: - """When log_path is None the export still succeeds (no file written).""" - from kb.service.ops import create_cron_run - - result = create_cron_run( - session, - data_dir, - job_name="kb-wiki-promote", - target="2026-06-04", - status="success", - log_body="Done.\n", - log_path=None, - ) - - assert result["export"]["status"] == "success" - row = session.execute(select(CronRun)).scalar_one() - assert row.log_path is None - - -# --------------------------------------------------------------------------- # -# MetricsRecord tests # -# --------------------------------------------------------------------------- # - - -def test_upsert_metrics_creates_row(data_dir: Path, session) -> None: - """First upsert creates a MetricsRecord and exports JSON file.""" - from kb.service.ops import upsert_metrics - - result = upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - token_total=100, - metrics_json={"token_total": 100}, - ) - - assert result["export"]["status"] == "success" - assert "id" in result - - rows = ( - session.execute( - select(MetricsRecord).where( - MetricsRecord.report_date == "2026-06-04", - MetricsRecord.report_type == "opencode", - ) - ) - .scalars() - .all() - ) - assert len(rows) == 1 - assert rows[0].token_total == 100 - - exported = data_dir / "ops/reports/2026/06/2026-06-04-opencode-usage.metrics.json" - assert json.loads(exported.read_text()) == {"token_total": 100} - - -def test_upsert_metrics_keeps_single_row_per_date_type(data_dir: Path, session) -> None: - """Second upsert for same (date,type) updates values; still one DB row.""" - from kb.service.ops import upsert_metrics - - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - token_total=100, - metrics_json={"token_total": 100}, - ) - - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - token_total=250, - metrics_json={"token_total": 250}, - ) - - rows = ( - session.execute( - select(MetricsRecord).where( - MetricsRecord.report_date == "2026-06-04", - MetricsRecord.report_type == "opencode", - ) - ) - .scalars() - .all() - ) - assert len(rows) == 1 - assert rows[0].token_total == 250 - - exported = data_dir / "ops/reports/2026/06/2026-06-04-opencode-usage.metrics.json" - assert json.loads(exported.read_text()) == {"token_total": 250} - - -def test_upsert_metrics_update_nulls_omitted_field(data_dir: Path, session) -> None: - """On update, omitting a scalar field (default None) clears it in the DB. - - Documents the deliberate full-overwrite semantics: callers send a complete - payload, so an omitted field is interpreted as a NULL write. - """ - from kb.service.ops import upsert_metrics - - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - token_total=100, - metrics_json={"token_total": 100}, - ) - - # Second upsert omits token_total (defaults to None) → clears the column. - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - metrics_json={"note": "no token total this time"}, - ) - - row = session.execute( - select(MetricsRecord).where( - MetricsRecord.report_date == "2026-06-04", - MetricsRecord.report_type == "opencode", - ) - ).scalar_one() - assert row.token_total is None - - -def test_upsert_metrics_different_types_separate_rows(data_dir: Path, session) -> None: - """Different report_type values create separate rows.""" - from kb.service.ops import upsert_metrics - - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="opencode", - metrics_json={"token_total": 10}, - ) - upsert_metrics( - session, - data_dir, - report_date="2026-06-04", - report_type="hermes", - metrics_json={"token_total": 20}, - ) - - rows = session.execute(select(MetricsRecord)).scalars().all() - assert len(rows) == 2 - - # --------------------------------------------------------------------------- # # export_markdown tests # # --------------------------------------------------------------------------- # diff --git a/test/test_service_pages.py b/test/test_service_pages.py index 7f6bf14..3907b0d 100644 --- a/test/test_service_pages.py +++ b/test/test_service_pages.py @@ -1,33 +1,21 @@ """Tests for kb.service.pages — plain-function service layer for Page operations. -Covers the 7 minimum scenarios required by Task 3: +Covers these scenarios: 1. create → export success, file on disk, DB row, create revision 2. source passes through to revision.source 3. upsert replaces slug → single Page, revisions [create, update], body_md in changed_fields, file updated -4. patch preserves category/review_status when frontmatter omits them -5. promote/approve happy path + guard (approve from not_processed → conflict) -6. reject from not_processed → rejected, export_path rewritten -7. ttl_sweep rejects stale not_processed page +4. patch preserves category when frontmatter omits it """ from __future__ import annotations from pathlib import Path -import pytest import yaml from sqlalchemy import select from kb.db.models import Page, PageRevision -from kb.service.pages import ( - approve_page, - patch_page, - promote_page, - reject_page, - ttl_sweep, - upsert_page, -) -from kb.service.errors import ServiceError +from kb.service.pages import patch_page, upsert_page # --------------------------------------------------------------------------- # Helpers / shared payloads @@ -35,7 +23,6 @@ _CONCEPT_FM = { "type": "concept", - "review_status": "not_processed", "created": "2026-06-04", "updated": "2026-06-04", "sources": [], @@ -53,7 +40,6 @@ _SUMMARY_FM_WITH_META = { "type": "summary", "category": "usage", - "review_status": "approved", "created": "2026-06-04", "updated": "2026-06-04", "sources": [], @@ -81,7 +67,6 @@ def _upsert_concept(session, data_dir, *, slug="test-concept", fm=None, body_md= def test_create_exports_file_and_creates_revision(data_dir: Path, session) -> None: fm = { "type": "concept", - "review_status": "not_processed", "created": "2026-06-04", "updated": "2026-06-04", "sources": [], @@ -201,7 +186,7 @@ def test_upsert_replaces_slug(data_dir: Path, session) -> None: # --------------------------------------------------------------------------- -# 4. patch preserves category/review_status when frontmatter omits them +# 4. patch preserves category when frontmatter omits it # --------------------------------------------------------------------------- @@ -214,7 +199,7 @@ def test_patch_preserves_columns_when_frontmatter_omits_keys( data_dir, slug="patch-target", type="summary", - body_md="\n# Patch Target\n\nA summary page with category and review_status set.\n", + body_md="\n# Patch Target\n\nA summary page with category set.\n", frontmatter=fm, export_path="wiki/summaries/2026/06/patch-target.md", ) @@ -235,7 +220,6 @@ def test_patch_preserves_columns_when_frontmatter_omits_keys( page = session.execute(select(Page).where(Page.slug == "patch-target")).scalar_one() assert page.category == "usage" - assert page.review_status == "approved" def test_patch_records_true_old_values(data_dir: Path, session) -> None: @@ -292,78 +276,3 @@ def test_patch_no_change_returns_skipped(data_dir: Path, session) -> None: result = patch_page(session, data_dir, slug="no-change") assert result["export"]["status"] == "skipped" - - -# --------------------------------------------------------------------------- -# 5. promote/approve happy path + guard -# --------------------------------------------------------------------------- - - -def test_promote_and_approve_happy_path(data_dir: Path, session) -> None: - _upsert_concept(session, data_dir, slug="promote-me") - - result = promote_page(session, data_dir, slug="promote-me") - page = session.execute(select(Page).where(Page.slug == "promote-me")).scalar_one() - assert page.review_status == "pending_for_approve" - assert result["page"]["review_status"] == "pending_for_approve" - - result = approve_page(session, data_dir, slug="promote-me") - page = session.execute(select(Page).where(Page.slug == "promote-me")).scalar_one() - assert page.review_status == "approved" - assert "approved_at" in page.frontmatter - - -def test_approve_from_not_processed_raises_conflict(data_dir: Path, session) -> None: - _upsert_concept(session, data_dir, slug="skip-promote") - - with pytest.raises(ServiceError) as exc_info: - approve_page(session, data_dir, slug="skip-promote") - assert exc_info.value.code == "conflict" - - -# --------------------------------------------------------------------------- -# 6. reject from not_processed → rejected, export_path rewritten -# --------------------------------------------------------------------------- - - -def test_reject_from_not_processed(data_dir: Path, session) -> None: - _upsert_concept(session, data_dir, slug="reject-me") - - result = reject_page(session, data_dir, slug="reject-me") - - page = session.execute(select(Page).where(Page.slug == "reject-me")).scalar_one() - assert page.review_status == "rejected" - assert page.export_path == "rejected/concepts/reject-me.md" - assert result["page"]["review_status"] == "rejected" - - -# --------------------------------------------------------------------------- -# 7. ttl_sweep rejects stale not_processed page -# --------------------------------------------------------------------------- - - -def test_ttl_sweep_rejects_stale_page(data_dir: Path, session) -> None: - fm = { - "type": "concept", - "review_status": "not_processed", - "created": "2026-01-01", - "updated": "2026-01-01", - "sources": [], - "tags": ["stale"], - } - upsert_page( - session, - data_dir, - slug="stale-page", - type="concept", - body_md="\n# Stale Page\n\nA not_processed concept old enough to be swept by TTL.\n", - frontmatter=fm, - export_path="wiki/concepts/stale-page.md", - ) - - result = ttl_sweep(session, data_dir, days=7) - assert result["swept"] == 1 - - page = session.execute(select(Page).where(Page.slug == "stale-page")).scalar_one() - assert page.review_status == "rejected" - assert page.export_path == "rejected/concepts/stale-page.md" diff --git a/test/test_wiki_note_skill.py b/test/test_wiki_note_skill.py index e39ab43..e1c55d4 100644 --- a/test/test_wiki_note_skill.py +++ b/test/test_wiki_note_skill.py @@ -26,7 +26,6 @@ AUTHORED_FM = { "type": "concept", "origin": "authored", - "review_status": "approved", "created": "2026-06-02", "updated": "2026-06-02", "sources": [], From 603090fc239eaa48209868215e5bc82a82f84127 Mon Sep 17 00:00:00 2001 From: yw0nam Date: Fri, 10 Jul 2026 11:12:47 +0900 Subject: [PATCH 3/9] feat: drop review_status column and cron_runs/metrics tables (migration) Co-Authored-By: Claude Opus 4.8 (1M context) --- ...3d4e5f6_drop_cron_metrics_review_status.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 alembic/versions/a1b2c3d4e5f6_drop_cron_metrics_review_status.py diff --git a/alembic/versions/a1b2c3d4e5f6_drop_cron_metrics_review_status.py b/alembic/versions/a1b2c3d4e5f6_drop_cron_metrics_review_status.py new file mode 100644 index 0000000..43137f6 --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_drop_cron_metrics_review_status.py @@ -0,0 +1,75 @@ +"""drop review_status column and cron_runs/metrics tables + +The ORM models and service layer no longer read or write +``pages.review_status``, the ``cron_runs`` table, or the ``metrics`` table +(references were removed in a prior change). Drop them from the schema. + +Revision ID: a1b2c3d4e5f6 +Revises: c2d3e4f5a6b7 +Create Date: 2026-07-10 00:00:00.000000 + +""" + +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, Sequence[str], None] = "c2d3e4f5a6b7" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Drop pages.review_status and the cron_runs/metrics tables.""" + op.drop_index("ix_pages_review_status_updated_at", table_name="pages") + op.drop_column("pages", "review_status") + op.drop_table("cron_runs") + op.drop_table("metrics") + + +def downgrade() -> None: + """Recreate pages.review_status and the cron_runs/metrics tables.""" + op.add_column("pages", sa.Column("review_status", sa.Text(), nullable=True)) + op.create_index( + "ix_pages_review_status_updated_at", + "pages", + ["review_status", sa.text("updated_at DESC")], + ) + + op.create_table( + "cron_runs", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("job_name", sa.Text(), nullable=False), + sa.Column("target", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("exit_code", sa.Integer(), nullable=True), + sa.Column("log_body", sa.Text(), nullable=False), + sa.Column("log_path", sa.Text(), nullable=True), + sa.Column("started_at", sa.Text(), nullable=True), + sa.Column("finished_at", sa.Text(), nullable=True), + sa.Column("created_at", sa.Text(), nullable=False), + ) + op.create_index("ix_cron_runs_target_job", "cron_runs", ["target", "job_name"]) + + op.create_table( + "metrics", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("report_date", sa.Text(), nullable=False), + sa.Column("report_type", sa.Text(), nullable=False), + sa.Column("session_count", sa.Integer(), nullable=True), + sa.Column("token_total", sa.Integer(), nullable=True), + sa.Column("cost_usd", sa.Float(), nullable=True), + sa.Column("tool_error_count", sa.Integer(), nullable=True), + sa.Column("metrics_json", sa.JSON(), nullable=False), + sa.Column("created_at", sa.Text(), nullable=False), + ) + op.create_index( + "uq_metrics_date_type", + "metrics", + ["report_date", "report_type"], + unique=True, + ) From 7963fdb02394135642d6cb93e75e823845301a24 Mon Sep 17 00:00:00 2001 From: yw0nam Date: Fri, 10 Jul 2026 11:28:50 +0900 Subject: [PATCH 4/9] docs: remove cron/report/promote references from docs and skills Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/knowledgebase-initialize/SKILL.md | 92 ++----------------- .claude/skills/wiki-authoring/SKILL.md | 13 +-- .../reference/templates/checklist.md | 1 - .../reference/templates/concept.md | 1 - .../reference/templates/decision.md | 1 - .../reference/templates/entity.md | 1 - .../reference/templates/improvement.md | 1 - .../reference/templates/question.md | 1 - .claude/skills/wiki-note/SKILL.md | 20 ++-- .../reference/templates/checklist.md | 1 - .../wiki-note/reference/templates/concept.md | 1 - .../wiki-note/reference/templates/decision.md | 1 - .../wiki-note/reference/templates/entity.md | 1 - .../reference/templates/improvement.md | 1 - CHANGELOG.md | 4 + CLAUDE.md | 64 ++----------- README.md | 10 +- docs/README.md | 17 +--- docs/architecture.md | 7 +- docs/db-canonical.md | 5 +- .../state-db-schema-reference.md | 65 ++----------- docs/reference/commands.md | 56 +---------- docs/reference/frontmatter.md | 2 - 23 files changed, 49 insertions(+), 317 deletions(-) diff --git a/.claude/skills/knowledgebase-initialize/SKILL.md b/.claude/skills/knowledgebase-initialize/SKILL.md index 5daced3..7b85ffd 100644 --- a/.claude/skills/knowledgebase-initialize/SKILL.md +++ b/.claude/skills/knowledgebase-initialize/SKILL.md @@ -1,6 +1,6 @@ --- name: knowledgebase-initialize -description: Use on a fresh clone, new machine, or new profile to create or repair the local data repository — verifying CLI tooling, choosing usage report mode, proposing cron jobs, and writing initialization handoff/log output. +description: Use on a fresh clone, new machine, or new profile to create or repair the local data repository — verifying CLI tooling and writing initialization handoff/log output. --- # KnowledgeBase Initialize @@ -15,10 +15,8 @@ Use this skill as the runtime contract for repository setup. Do not load `docs/` - `data/` is a generated export directory. Postgres (reached via `DATABASE_URL`) is the canonical store. Never add `data/` to the outer repo. - Never modify existing files under `data/raw/`. - Preserve existing `data/` contents; create only missing directories/files. -- Do not install or edit crontab until the user approves the exact entries. -- Memory cron jobs run lint and leave changes uncommitted for manual review. -- Wiki promotion writes through the kb-mcp tools; it does not commit or push from the AI session. -- Prefer relative paths from repo root. Do not hard-code machine-specific absolute paths except when showing final crontab examples. +- Wiki writes go through the kb-mcp tools; the skill does not commit or push from the AI session. +- Prefer relative paths from repo root. Do not hard-code machine-specific absolute paths. ## Required Data Layout @@ -33,7 +31,6 @@ data/ web/ manual/ ops/ - cron/YYYY/MM/ runs/YYYY/MM/ decisions/YYYY/MM/ traces/YYYY/MM/ @@ -47,14 +44,9 @@ data/ improvements/YYYY-MM/ checklists/ summaries/YYYY/MM/ - rejected/ - ops/ - reports/YYYY/MM/ log.md ``` -`data/rejected/` may be empty on a fresh install. It is populated by wiki rejection. - ## Phase 1: Inspect Check: @@ -65,8 +57,6 @@ psql "${DATABASE_URL/+psycopg/}" -tAc "SELECT 1" >/dev/null && echo "DB reachabl test -f data/log.md uv --version uv run kb-lint --help -uv run kb-db-ttl-sweep --help -find scripts/cron -maxdepth 1 -type f -name 'kb-*.sh' | sort ``` If a command fails because dependencies are missing, run: @@ -150,61 +140,7 @@ bash .claude/skills/knowledgebase-initialize/scripts/install-global-skills.sh - Set `CLAUDE_SKILLS_DIR` to override the destination. Edit the `GLOBAL_SKILLS` array in the script to expose more skills (currently: `handoff-document` and `wiki-note`). -## Phase 4: Usage Report Mode - -Use the `usage-report-setup` skill if the user asks for detailed setup. For initialization, ask which source-specific reports to enable: - -| Mode | Cron wrapper | -|---|---| -| none | no usage report job | -| OpenCode | `scripts/cron/kb-opencode-daily-report.sh` | -| Hermes | `scripts/cron/kb-hermes-daily-report.sh` | -| Claude Code | `scripts/cron/kb-claude-code-daily-report.sh` | -| multiple separate | selected wrappers only | - -Default recommendation: enable only sources the user actually runs. Do not create combined usage reports. - -## Phase 5: Propose Cron Jobs - -KnowledgeBase owns the expected job contract and portable wrapper scripts under `scripts/cron/`; it does not require a specific scheduler backend. - -Scheduler backend guidance: - -- **Tested scheduler**: Hermes cron, using scheduler-local dispatcher scripts that call the repo wrappers. -- **Compatible but not yet tested here**: OpenClaw cron, native Unix crontab, systemd timers, or any equivalent scheduler that can run the wrapper scripts on the documented schedule. -- Actual job registration state belongs to the chosen scheduler backend. Run evidence belongs in DB tables (`cron_runs`, `handoffs`, `operation_logs`, and summary `pages`); `data/raw/ops/cron/{YYYY}/{MM}/`, `data/handoffs/`, `data/log.md`, and wrap-up markdown are generated export/debug copies. - -Show the exact list before making cron changes: - -```text -KnowledgeBase jobs: -- daily memory build: 03:30 every day -- wiki promote: 04:00 every day -- weekly memory build: 04:15 every Monday -- monthly memory maintenance:04:45 on day 1 of each month -- wiki TTL sweep: 00:30 every day -- cron wrap-up: 05:00 every day - -Optional usage report jobs: -- OpenCode daily usage: 03:10 every day -- Hermes daily usage: 03:15 every day -- Claude Code daily usage: 03:20 every day - -Optional global digest job: -- morning Slack digest: 09:00 every day (optional, recommended) -``` - -If the user wants the optional global digest, read `reference/optional-global-digest.md` and show the setup separately from the required KB cron jobs. - -Wrapper prompt policy: - -- Memory wrappers import `.claude/skills/memory-report/SKILL.md`. -- Wiki promotion wrapper imports `.claude/skills/wiki-approval/SKILL.md`. -- Cron wrap-up wrapper imports `.claude/skills/cron-wrapup/SKILL.md` and writes a Slack-digest-stable `wiki/summaries/.../{date}-cron-wrapup.md` plus run handoff. -- Usage setup/import prompts use `.claude/skills/usage-report-setup/SKILL.md`. -- TTL sweep runs `uv run kb-db-ttl-sweep --days 7`, applying status changes in-process through the `kb.service` layer. - -### Register kb-mcp in agent runtimes (ask first) +## Phase 4: Register kb-mcp in Agent Runtimes LLM/agent runtimes reach the write surface by connecting to the `kb-mcp` daemon over http. Before registering anything, **ask the user two things**: @@ -225,17 +161,9 @@ Recipes (use the confirmed ``): - **Claude Code** — `claude mcp add --transport http --scope user kb-mcp ` - **Other runtimes (e.g. Hermes)** — ask the user where their MCP config lives; add an http entry pointing at ``. -Deterministic jobs (`kb-db-ttl-sweep`, daily reports, ingest) need no registration — they call `kb.service` in-process. The daemon has no auth: if it must be reachable from other hosts, publish it on the appropriate interface (compose default `0.0.0.0:8765`); on a single host, host-loopback (`127.0.0.1:8765:8765`) is safer (see #41). - -Only after approval: - -1. Create or update wrapper scripts under `scripts/cron/`. -2. Make wrappers executable. -3. Each wrapper writes its run log to `data/raw/ops/cron/{YYYY}/{MM}/{TARGET}_kb-.log` (where YYYY/MM derives from TARGET_DATE). The `kb-cron-wrapup` wrapper writes to `.cron/logs/cron-wrapup.log` during the session, then commits it to `data/raw/ops/cron/` in a follow-up commit after the session exits. -4. Keep locks in `.cron/locks/`. -5. Show scheduler entries for the selected backend, or install/register them only if the user explicitly asks. +The daemon has no auth: if it must be reachable from other hosts, publish it on the appropriate interface (compose default `0.0.0.0:8765`); on a single host, host-loopback (`127.0.0.1:8765:8765`) is safer (see #41). -## Phase 6: Initialization Handoff +## Phase 5: Initialization Handoff Write: @@ -285,7 +213,7 @@ Run: # Handoff validation runs inside the kb-mcp create_handoff tool (returns code: lint_failed on failure) ``` -## Phase 7: Log +## Phase 6: Log Write the setup note through the kb-mcp `create_operation_log` tool (the generated export may update `data/log.md`): @@ -297,8 +225,6 @@ Write the setup note through the kb-mcp `create_operation_log` tool (the generat - **directories**: created / already present - **tooling**: - **global skills**: symlinked / skipped -- **usage reports**: -- **cron**: proposed / approved / skipped - **handoff**: handoffs/YYYY/MM/kb-initialize/.md ``` @@ -309,14 +235,10 @@ Write the setup note through the kb-mcp `create_operation_log` tool (the generat - CLI smoke tests ran or blockers are documented. - Global skills are symlinked into `~/.claude/skills/` or explicitly skipped. - kb-mcp is registered in the agent runtimes the user approved (or explicitly skipped). -- Usage report mode is selected or explicitly skipped. -- Cron entries are proposed or explicitly skipped. - Initialization handoff exists and passes lint. - No existing `data/raw/` file was modified. ## Red Flags - About to write private data into the outer repo. -- About to install crontab without explicit approval. -- About to add auto-commit to daily/weekly/monthly memory wrappers. - About to rewrite existing user data to satisfy lint without instruction. diff --git a/.claude/skills/wiki-authoring/SKILL.md b/.claude/skills/wiki-authoring/SKILL.md index 7dfb93b..f2ca8a8 100644 --- a/.claude/skills/wiki-authoring/SKILL.md +++ b/.claude/skills/wiki-authoring/SKILL.md @@ -9,7 +9,7 @@ Use this skill as the runtime contract for raw-to-wiki authoring. DB rows are th source of truth; Markdown under `data/` is generated export. Do not look for a workflow doc during execution; this skill is the complete operating surface. -> **Evidence-derived only.** This skill is for pages grounded in a `data/raw` source — including all LLM/cron authoring, which MUST cite real sources. For a **first-party human note** with no external source, use the `wiki-note` skill (`origin: authored`, `sources: []`); do not fake a citation here. +> **Evidence-derived only.** This skill is for pages grounded in a `data/raw` source — including all LLM/automated authoring, which MUST cite real sources. For a **first-party human note** with no external source, use the `wiki-note` skill (`origin: authored`, `sources: []`); do not fake a citation here. ## Rules @@ -18,8 +18,6 @@ workflow doc during execution; this skill is the complete operating surface. - Never use `data/raw/...` or absolute paths in `sources:`. - Use wikilinks only to existing file stems. `aliases:` do not satisfy lint. - Do not use the reserved `## User Feedback` heading in authored content. -- New `entity`, `concept`, `decision`, `improvement`, `checklist`, and `question` pages start with `review_status: not_processed`. -- If semantically editing an `approved` page, reset `review_status: not_processed`; keep `approved` only for typo/format-only edits. ## Bundled Templates @@ -71,7 +69,6 @@ Use stable ASCII slugs unless an existing local convention requires otherwise. ```yaml --- type: entity -review_status: not_processed created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: @@ -88,7 +85,6 @@ Use `type: concept` or `type: question` as appropriate. `aliases:` is required f ```yaml --- type: decision -review_status: not_processed created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: @@ -104,7 +100,6 @@ Filename must carry the decision date: `YYYY-MM-DD-.md`. ```yaml --- type: improvement -review_status: not_processed kind: improvement observed_at: "YYYY-MM-DD" domain: dx @@ -131,7 +126,6 @@ Enums: ```yaml --- type: checklist -review_status: not_processed created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: @@ -163,7 +157,7 @@ tags: [] --- ``` -Weekly/monthly summaries use `period_start` and `period_end`; weekly also has `week: "YYYY-WNN"` and monthly `month: "YYYY-MM"`. Summaries do not use `review_status`. +Weekly/monthly summaries use `period_start` and `period_end`; weekly also has `week: "YYYY-WNN"` and monthly `month: "YYYY-MM"`. ## Promotion Criteria @@ -186,7 +180,7 @@ Writes go through kb-mcp tools (registered in the `opencode run` session). For a - `body_md` — the body **without** the `---` fence - `slug`, `type`, `export_path` -Optional: `title`, `category`, `review_status`, `origin`, `created_at`, `updated_at`, `source`. For a partial update to an existing page, call the kb-mcp `patch_page` tool with `slug` plus only the fields to change (`title`, `body_md`, `frontmatter`, `category`, `review_status`, `source`, `note`). +Optional: `title`, `category`, `origin`, `created_at`, `updated_at`, `source`. For a partial update to an existing page, call the kb-mcp `patch_page` tool with `slug` plus only the fields to change (`title`, `body_md`, `frontmatter`, `category`, `source`, `note`). The tool appends a revision and exports Markdown immediately. Do not manually regenerate `INDEX.md`; it is generated export from the DB. @@ -214,7 +208,6 @@ Call the kb-mcp `create_operation_log` tool with this `body_md`: - **sources**: raw/... - **created/updated**: wiki/... -- **review_status**: not_processed - **db_write**: page export succeeded ``` diff --git a/.claude/skills/wiki-authoring/reference/templates/checklist.md b/.claude/skills/wiki-authoring/reference/templates/checklist.md index 3c6e791..422ddb0 100644 --- a/.claude/skills/wiki-authoring/reference/templates/checklist.md +++ b/.claude/skills/wiki-authoring/reference/templates/checklist.md @@ -1,6 +1,5 @@ --- type: checklist -review_status: not_processed created: "" updated: "" sources: [] diff --git a/.claude/skills/wiki-authoring/reference/templates/concept.md b/.claude/skills/wiki-authoring/reference/templates/concept.md index 5823623..8f63a26 100644 --- a/.claude/skills/wiki-authoring/reference/templates/concept.md +++ b/.claude/skills/wiki-authoring/reference/templates/concept.md @@ -1,6 +1,5 @@ --- type: concept -review_status: not_processed created: "" updated: "" sources: [] diff --git a/.claude/skills/wiki-authoring/reference/templates/decision.md b/.claude/skills/wiki-authoring/reference/templates/decision.md index 59aa025..93e65eb 100644 --- a/.claude/skills/wiki-authoring/reference/templates/decision.md +++ b/.claude/skills/wiki-authoring/reference/templates/decision.md @@ -1,6 +1,5 @@ --- type: decision -review_status: not_processed created: "" updated: "" sources: [] diff --git a/.claude/skills/wiki-authoring/reference/templates/entity.md b/.claude/skills/wiki-authoring/reference/templates/entity.md index 1a07b84..f24afca 100644 --- a/.claude/skills/wiki-authoring/reference/templates/entity.md +++ b/.claude/skills/wiki-authoring/reference/templates/entity.md @@ -1,6 +1,5 @@ --- type: entity -review_status: not_processed created: "" updated: "" sources: [] diff --git a/.claude/skills/wiki-authoring/reference/templates/improvement.md b/.claude/skills/wiki-authoring/reference/templates/improvement.md index 24865d9..9b62b5b 100644 --- a/.claude/skills/wiki-authoring/reference/templates/improvement.md +++ b/.claude/skills/wiki-authoring/reference/templates/improvement.md @@ -1,6 +1,5 @@ --- type: improvement -review_status: not_processed kind: improvement observed_at: "" domain: "" diff --git a/.claude/skills/wiki-authoring/reference/templates/question.md b/.claude/skills/wiki-authoring/reference/templates/question.md index 9c30f3e..9dd1f2a 100644 --- a/.claude/skills/wiki-authoring/reference/templates/question.md +++ b/.claude/skills/wiki-authoring/reference/templates/question.md @@ -1,6 +1,5 @@ --- type: question -review_status: not_processed created: "" updated: "" sources: [] diff --git a/.claude/skills/wiki-note/SKILL.md b/.claude/skills/wiki-note/SKILL.md index 9da2297..99f93e6 100644 --- a/.claude/skills/wiki-note/SKILL.md +++ b/.claude/skills/wiki-note/SKILL.md @@ -1,6 +1,6 @@ --- name: wiki-note -description: Use when you (a human) want to capture a first-party note, insight, or decision directly into the KnowledgeBase wiki — original thinking that does NOT come from a data/raw source. Works from any repo. Not for LLM/cron authoring and not for evidence-derived pages. +description: Use when you (a human) want to capture a first-party note, insight, or decision directly into the KnowledgeBase wiki — original thinking that does NOT come from a data/raw source. Works from any repo. Not for LLM/automated authoring and not for evidence-derived pages. --- # wiki-note @@ -22,12 +22,12 @@ external `data/raw` evidence behind it — into the DB-backed KnowledgeBase wiki Normal wiki pages must cite `sources:`. That rule exists to **ground LLM-generated content in evidence**. A note you author yourself has different, valid provenance: *you wrote it*. This skill is the sanctioned exception — pages are marked -`origin: authored` with `sources: []` and may be born `approved`. +`origin: authored` with `sources: []`. **Core split — never blur it:** - **You, a human, thinking** → `origin: authored`, no source. **This skill.** -- **An LLM / cron summarizing captured evidence** → MUST cite real `data/raw` sources. +- **An LLM / automated process summarizing captured evidence** → MUST cite real `data/raw` sources. Use `wiki-authoring`. Automation must NEVER set `origin: authored`. ## When to Use @@ -45,7 +45,7 @@ content in evidence**. A note you author yourself has different, valid provenanc Supported types here: **`concept`, `decision`, `entity`, `checklist`, `improvement`** — one bundled template per type in `reference/templates/`, each pre-set with the -first-party frontmatter (`origin: authored`, `review_status: approved`, `sources: []`). +first-party frontmatter (`origin: authored`, `sources: []`). ## Step 1 — Resolve the KB root (works from any repo) @@ -90,7 +90,6 @@ keep the rest: --- type: concept # matches the template you copied origin: authored # first-party authorship — keep this; it's why sources can be empty -review_status: approved # born approved (you author and approve); not_processed also valid created: "YYYY-MM-DD" # today updated: "YYYY-MM-DD" sources: [] # empty is allowed BECAUSE origin: authored @@ -125,11 +124,6 @@ object/dict (not YAML text) and the body **without** the `---` fence: `tags`, and any type-specific fields) - `body_md` — the page body with no frontmatter fence - `origin` — `"authored"` (first-party authorship; keep it — it's why `sources` can be empty) -- `review_status` — `"approved"` if born approved, else `"not_processed"` - -If you author the page born approved, you may instead pass `review_status="not_processed"` -and then call the kb-mcp `approve_page` tool with the same `slug` — keep whichever -lifecycle you chose in Step 3. On success the tool returns the page plus `"export": {"status": "success", "written": N}`, confirming Markdown was exported immediately. On failure it returns @@ -147,17 +141,15 @@ Write through the kb-mcp `create_operation_log` tool: - **db_write**: page export succeeded ``` -## Commit & approval +## Commit - Do not commit `data/`; it is generated export. -- `wiki-promote` (cron) will not touch an already-`approved` page, so first-party notes - bypass the promotion ladder by design. ## Common Mistakes | Mistake | Fix | |---|---| -| `origin: authored` on an LLM/cron page | Automation must cite real sources — use `wiki-authoring` | +| `origin: authored` on an LLM/automated page | Automation must cite real sources — use `wiki-authoring` | | `sources: []` but you actually have a source | Cite it via `wiki-authoring`; don't mark authored | | `improvement` enum left blank/invalid | Fill `kind`/`domain`/`severity`/`issue_status` per template comments | | links scattered inline instead of `## Related` | Collect them as `- [[stem]] — why` in `## Related` | diff --git a/.claude/skills/wiki-note/reference/templates/checklist.md b/.claude/skills/wiki-note/reference/templates/checklist.md index 40bed0d..53a2121 100644 --- a/.claude/skills/wiki-note/reference/templates/checklist.md +++ b/.claude/skills/wiki-note/reference/templates/checklist.md @@ -1,7 +1,6 @@ --- type: checklist origin: authored -review_status: approved created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: [] diff --git a/.claude/skills/wiki-note/reference/templates/concept.md b/.claude/skills/wiki-note/reference/templates/concept.md index 3f4383f..43b4cee 100644 --- a/.claude/skills/wiki-note/reference/templates/concept.md +++ b/.claude/skills/wiki-note/reference/templates/concept.md @@ -1,7 +1,6 @@ --- type: concept origin: authored -review_status: approved created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: [] diff --git a/.claude/skills/wiki-note/reference/templates/decision.md b/.claude/skills/wiki-note/reference/templates/decision.md index 4fdcca2..8b480d4 100644 --- a/.claude/skills/wiki-note/reference/templates/decision.md +++ b/.claude/skills/wiki-note/reference/templates/decision.md @@ -1,7 +1,6 @@ --- type: decision origin: authored -review_status: approved created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: [] diff --git a/.claude/skills/wiki-note/reference/templates/entity.md b/.claude/skills/wiki-note/reference/templates/entity.md index 0ad19c7..5c01074 100644 --- a/.claude/skills/wiki-note/reference/templates/entity.md +++ b/.claude/skills/wiki-note/reference/templates/entity.md @@ -1,7 +1,6 @@ --- type: entity origin: authored -review_status: approved created: "YYYY-MM-DD" updated: "YYYY-MM-DD" sources: [] diff --git a/.claude/skills/wiki-note/reference/templates/improvement.md b/.claude/skills/wiki-note/reference/templates/improvement.md index 165ded9..97ea251 100644 --- a/.claude/skills/wiki-note/reference/templates/improvement.md +++ b/.claude/skills/wiki-note/reference/templates/improvement.md @@ -1,7 +1,6 @@ --- type: improvement origin: authored -review_status: approved kind: improvement # improvement | issue | proposal observed_at: "YYYY-MM-DD" domain: dx # cost | correctness | perf | dx | security diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f90f6b..7156442 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ Keep entries concise and user/operator-facing. Avoid tool traces, lint output, h ## Unreleased +### Removed + +- Removed the cron pipeline, usage/daily reports (metrics), and the wiki promote/review_status lifecycle — including related CLIs, MCP tools, skills, docs, and a migration dropping the review_status column and cron_runs/metrics tables. + ### Added - Added the MCP read tools `query_sql` and `get_schema` (`src/kb/mcp/tools_read.py`), registered on the `kb-mcp` server alongside the write tools. `query_sql` is **strictly read-only** with three layers of defense: a statement guard (rejects multi-statement input and any body not starting with `SELECT`/`WITH`), a `SET TRANSACTION READ ONLY` Postgres transaction issued as the first statement of the autobegun transaction (the real control — rejects every write, including data-modifying CTEs that pass the prefix guard), and a `fetchmany(limit)` row cap (default 100, `truncated` flag). DB errors (e.g. a write blocked by the read-only transaction) surface as `{"error", "code": "query_error"}` dicts rather than exceptions, so an agent loop continues. `get_schema` introspects the static ORM metadata (no DB connection) and returns every table's columns plus example queries; agents call it before `query_sql`. The 12-write-tool registration smoke test was relaxed from exact-set to subset so it stays correct as tools are added to the shared `mcp` instance. Read-only is verified against real Postgres in `test/test_mcp_read_tools.py` (10 tests: SELECT, real-data read, prefix-guard rejection of INSERT/UPDATE/DELETE/DROP, multi-statement rejection, the writing-CTE transaction-level block with a no-row assertion, row cap, and schema shape). diff --git a/CLAUDE.md b/CLAUDE.md index c261ff1..8e30950 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,16 +14,12 @@ KnowledgeBase is a memory-workflow system (v0) that captures knowledge from mult KnowledgeBase/ # Outer repo: code, lint, templates, docs ├── src/kb/ # CLI tools + FastMCP server + service layer │ ├── cli/ -│ │ ├── lint.py # kb-lint command (wiki + handoff validation) -│ │ ├── db_ttl_sweep.py # kb-db-ttl-sweep command -│ │ ├── opencode_daily_report.py # kb-opencode-daily-report command -│ │ ├── hermes_daily_report.py # kb-hermes-daily-report command -│ │ └── claude_code_daily_report.py # kb-claude-code-daily-report command +│ │ └── lint.py # kb-lint command (wiki + handoff validation) │ ├── service/ # DB-canonical write path (lint → DB → export) │ │ ├── pages.py # Wiki page operations │ │ ├── sources.py # Raw source ingest │ │ ├── handoffs.py # Handoff document operations -│ │ ├── ops.py # Cron runs, metrics, operation logs, export +│ │ ├── ops.py # Operation logs, export │ │ ├── export.py # Markdown export │ │ └── session.py # SQLAlchemy session helpers │ └── mcp/ # FastMCP server (kb-mcp) @@ -76,14 +72,14 @@ data/ # Generated Markdown export tree (KB_DATA_DIR) - Never modify files in `data/raw/`. They are immutable after creation. Use `kb-lint` to enforce (via the `kb.service` layer): git-status must not show modifications, `captured_at` must be ≤ file mtime (60s tolerance), and required raw frontmatter fields (`source_url`, `type`, `captured_at`, `contributor`) are always validated. - `data/wiki/` pages must always list their `sources:` in frontmatter. -- Keep `data/log.md` updated for `data/` repo operations only: raw ingest, wiki page creation/update, handoff creation/update, promotion/rejection, cron/report outputs, and data lint results. +- Keep `data/log.md` updated for `data/` repo operations only: raw ingest, wiki page creation/update, handoff creation/update, and data lint results. - Keep outer repo changes in `CHANGELOG.md`: source code, scripts, docs, skills, templates, config, schemas, and workflow contract changes. - Do not write outer repo implementation details to `data/log.md`; reference the data artefact or handoff only if the operation created one. - Use uv for package management ## Change Records -- `CHANGELOG.md` is for the outer repo. Update it when a change affects maintainers/operators or changes runtime behavior, CLI behavior, cron wrappers, skills, docs contracts, schemas, templates, or setup instructions. +- `CHANGELOG.md` is for the outer repo. Update it when a change affects maintainers/operators or changes runtime behavior, CLI behavior, skills, docs contracts, schemas, templates, or setup instructions. - Do not write outer repo implementation details to `data/log.md`; reference the data artefact or handoff only if the operation created one. - If outer-repo work produces or adjusts `data/` artefacts for verification, leave those `data/` changes uncommitted unless the user specifically asks to handle them. @@ -101,65 +97,17 @@ Never commit `data/` contents to the outer repository. Project skills live under `.claude/skills/`, auto-load by description match, and are the source of truth for workflow behavior. -- `knowledgebase-initialize` — bring up Postgres (compose `db`), run migrations, verify tooling, and propose cron jobs for approval. -- `wiki-approval` — promote, approve, reject, or TTL-sweep wiki pages through the `review_status` lifecycle; runtime contract for wiki-promote cron. +- `knowledgebase-initialize` — bring up Postgres (compose `db`), run migrations, and verify tooling. - `wiki-authoring` — create or update source-backed `data/wiki/` pages with valid schemas, paths, wikilinks, templates, and lint order. -- `usage-report-setup` — select and wire source-specific OpenCode/Hermes/Claude Code usage report jobs. - `handoff-document` — write or update handoff documents under `data/handoffs/` with lintable frontmatter, filename grammar, and canonical body sections. -- `memory-report` — daily, weekly, or monthly memory workflow (period dispatch inside the skill). Imports `wiki-authoring` for page edits and `handoff-document` for run handoffs. -- `cron-wrapup` — nightly KB cron wrap-up. Aggregates the previous day's usage reports, memory page, wiki-promote/TTL outcomes, and per-run cron evidence from `data/raw/ops/cron/{YYYY}/{MM}/{date}_kb-*.log` into a single Slack-digest-stable `wiki/summaries/.../{date}-cron-wrapup.md` plus run handoff. Runtime contract for the 05:00 cron job. - -## Cron Jobs - -The cron job runs KB cron jobs directly. The script delegates LLM work to `opencode run` where reasoning is needed, or calls deterministic `uv run` CLIs for pure data plumbing. - -### Execution patterns - -| Pattern | Used by | Mechanism | -|---------|---------|-----------| -| **LLM-Driven** | memory-daily/weekly/monthly, wiki-promote, cron-wrapup | skill-driven + opencode run `opencode run --model anthropic/claude-sonnet-4-6` | -| **Deterministic** (no LLM) | opencode/hermes/claude-code daily reports, wiki-ttl-sweep, ingest-papers | `uv run kb-*-daily-report --date --lint`, `uv run kb-db-ttl-sweep --days 7`, `uv run python scripts/ingest-daily-papers.py` | - -The deterministic CLIs call the `kb.service` layer **in-process** — no server needed. The LLM-driven cron jobs connect to the `kb-mcp` http daemon (compose `mcp` service, `127.0.0.1:8765`); see `knowledgebase-initialize` for registering `kb-mcp` in each agent runtime. - -### Pipeline schedule (KST) - -``` -00:30 kb-db-ttl-sweep deterministic -03:10 kb-opencode-daily-report deterministic -03:15 kb-hermes-daily-report deterministic -03:20 kb-claude-code-daily-report deterministic -03:30 kb-memory-daily LLM-Driven -04:00 kb-wiki-promote LLM-Driven -04:15 kb-memory-weekly (Mon) LLM-Driven -04:45 kb-memory-monthly (1st) LLM-Driven -05:00 kb-cron-wrapup LLM-Driven -10:05 kb-ingest-daily-papers deterministic -``` - -`morning-slack-digest` (09:00, Hermes agent) reads the wrapup artefact and delivers to Slack — it is **not** a KB cron job and runs with the agent. - -### Proxy scripts - -Cron jobs reference short script names (e.g. `kb-memory-daily.sh`) that resolve to `~/.hermes/scripts/`. Each is a thin proxy that `cd`s to KB root and `exec`s the real script under `scripts/cron/`. When adding a new cron script, create both the real script and the `~/.hermes/scripts/` proxy. - -### Runtime contracts - -OpenCode-powered jobs load their behavior from `.claude/skills//SKILL.md`: -- memory-* → `memory-report` (+ `wiki-authoring`, `handoff-document`) -- wiki-promote → `wiki-approval` (+ `wiki-authoring` if fixes needed) -- cron-wrapup → `cron-wrapup` (+ `handoff-document`) - -All run logs go to `data/raw/ops/cron/{YYYY}/{MM}/{date}_{job}.log`. ## Documentation - [Documentation Index](docs/README.md) — design/reference document map - [Architecture](docs/architecture.md) — repository layout and memory layers -- [Workflows](docs/workflows.md) — at-a-glance diagram map (nightly pipeline, review lifecycle); skills own the execution detail - [Frontmatter Conventions](docs/reference/frontmatter.md) — schema reference; `wiki-authoring` and `handoff-document` carry runtime rules - [Wiki Categories](docs/reference/wiki-categories.md) — category reference; use `wiki-authoring` at runtime -- [Commands](docs/reference/commands.md) — kb-lint, kb-db-ttl-sweep, kb-submit-cron-run, daily report CLIs, kb-mcp +- [Commands](docs/reference/commands.md) — kb-lint, kb-mcp ## Linting diff --git a/README.md b/README.md index ce513ad..bc541a7 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,7 @@ KnowledgeBase/ │ └── ingest-github.sh GitHub source collection ├── .claude/skills/ Runtime workflow contracts + bundled templates │ ├── wiki-authoring/ Wiki page templates and authoring rules -│ ├── wiki-approval/ review_status lifecycle workflow -│ ├── knowledgebase-initialize/ Setup workflow -│ └── usage-report-setup/ Usage report mode workflow +│ └── knowledgebase-initialize/ Setup workflow ├── docs/raw/ Raw source frontmatter templates ├── CLAUDE.md LLM entry point and project skill map ├── README.md This file @@ -40,13 +38,12 @@ data/ Generated Markdown export tree (Postgres is ca │ ├── improvements/ Open-ended improvements │ ├── checklists/ Operational checklists │ └── summaries/ Time/subject rollups -├── rejected/ Rejected wiki pages (created on reject via kb-mcp/service layer) └── log.md Operation record ``` ## Workflows -Project workflows live in `.claude/skills/`. Use `wiki-authoring` for source-backed wiki edits, `wiki-approval` for review lifecycle work, `memory-report` for daily/weekly/monthly synthesis, and `handoff-document` for handoffs. +Project workflows live in `.claude/skills/`. Use `wiki-authoring` for source-backed wiki edits and `handoff-document` for handoffs. ## Privacy @@ -83,7 +80,7 @@ kb-lint ### Commit -Writes go through the `kb-mcp` MCP server tools (or the in-process `kb.service` layer used by cron CLIs) — Markdown files under `data/` are generated exports. +Writes go through the `kb-mcp` MCP server tools (or the in-process `kb.service` layer) — Markdown files under `data/` are generated exports. Changes to the outer repo (code, docs, skills) are committed as usual. ## Files @@ -93,7 +90,6 @@ Changes to the outer repo (code, docs, skills) are committed as usual. | `CLAUDE.md` | LLM entry point and project skill map | | `scripts/ingest-github.sh` | GitHub source collection | | `src/kb/cli/lint.py` | Wiki + handoff validation | -| `src/kb/cli/db_ttl_sweep.py` | Auto-reject stale unprocessed pages | | `src/kb/mcp/` | DB-canonical MCP server (`kb-mcp`) | | `src/kb/service/` | In-process write path (lint → DB → export) | | `data/log.md` | Operation record | diff --git a/docs/README.md b/docs/README.md index 89a6715..572ab2f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,9 +18,6 @@ Agents should execute core workflows from `.claude/skills/`, not by loading work | Initialize a clone/profile | `.claude/skills/knowledgebase-initialize/SKILL.md` | | Write or fix wiki pages (evidence-derived) | `.claude/skills/wiki-authoring/SKILL.md` | | Capture a first-party note (human, no source) | `.claude/skills/wiki-note/SKILL.md` | -| Promote/approve/reject wiki pages | `.claude/skills/wiki-approval/SKILL.md` | -| Configure usage reports | `.claude/skills/usage-report-setup/SKILL.md` | -| Daily/weekly/monthly memory build | `.claude/skills/memory-report/SKILL.md` | | Write handoff documents | `.claude/skills/handoff-document/SKILL.md` | For human onboarding, read: @@ -35,10 +32,9 @@ For human onboarding, read: |---|---| | `docs/architecture.md` | System layout and data boundaries | | `docs/reference/` | Human-readable schemas, categories, commands, and lookup tables | -| `docs/db_informations/` | Database and reporting references | +| `docs/db_informations/` | Database references | | `docs/db_informations/state-db-schema-reference.md` | Postgres state DB schema + `psql` read recipes (reads go direct to the DB) | | `docs/db-canonical.md` | DB-as-source-of-truth memory architecture decision | -| `docs/workflows.md` | At-a-glance diagram map: nightly pipeline, review lifecycle (overview only — skills own execution) | | `docs/CLAUDE.md` | Document authoring rules | ### Document Types @@ -50,19 +46,14 @@ Use project skills for ordered execution steps. Use `docs/reference/` for human- | Need | Read | |---|---| | Understand the repo | `docs/architecture.md` | -| See the whole picture at a glance | `docs/workflows.md` | | Run raw-to-wiki pipeline | `.claude/skills/wiki-authoring/SKILL.md` | | Jot a first-party note (no source, from any repo) | `.claude/skills/wiki-note/SKILL.md` | -| Configure cron jobs | `.claude/skills/knowledgebase-initialize/SKILL.md` | -| Configure usage reports | `.claude/skills/usage-report-setup/SKILL.md` | -| Run daily/weekly/monthly memory build | `.claude/skills/memory-report/SKILL.md` | +| Initialize a clone/profile | `.claude/skills/knowledgebase-initialize/SKILL.md` | | Handle handoff lifecycle | `.claude/skills/handoff-document/SKILL.md` | | Write valid frontmatter | `.claude/skills/wiki-authoring/SKILL.md`; reference: `docs/reference/frontmatter.md` | | Choose wiki category/path | `.claude/skills/wiki-authoring/SKILL.md`; reference: `docs/reference/wiki-categories.md` | -| Approve/reject wiki pages | `.claude/skills/wiki-approval/SKILL.md` | | Run CLI commands | `docs/reference/commands.md` | | Read the DB directly (queries) | `docs/db_informations/state-db-schema-reference.md` | -| Start review console (web UI) | `README.md` → "Review console" section; `scripts/dev-web.sh` | | Back up / move the DB across machines | `docs/db-canonical.md` | | Understand the DB-canonical migration | `docs/db-canonical.md` | @@ -72,11 +63,11 @@ Use project skills for ordered execution steps. Use `docs/reference/` for human- ### A. PatchNote -- 2026-06-04: Removed the legacy `data-sync` skill and `docs/data-sync.md` (git-based two-repo sync); DB-canonical replaces it. Dropped the sync flow from `docs/workflows.md` and its document-map/routing rows here. +- 2026-06-04: Removed the legacy `data-sync` skill and `docs/data-sync.md` (git-based two-repo sync); DB-canonical replaces it. Dropped the sync flow from the at-a-glance workflow overview doc (since removed) and its document-map/routing rows here. - 2026-06-04: Postgres became the sole source of truth (SQLite removed); added `docs/db_informations/state-db-schema-reference.md` for direct `psql` reads. - 2026-06-04: Added `docs/db-canonical.md` to the document map and usage table. - 2026-06-02: Added the `wiki-note` skill (first-party human-authored pages, `origin: authored`) to runtime routing + usage tables; it is exposed globally like `handoff-document`. -- 2026-06-02: Added `docs/workflows.md` — at-a-glance Mermaid map of the nightly pipeline, review lifecycle, and two-repo sync. +- 2026-06-02: Added an at-a-glance Mermaid workflow overview doc (since removed) mapping the nightly pipeline, review lifecycle, and two-repo sync. - 2026-05-29: Updated data-sync sync entry to point at the `data-sync` skill. - 2026-05-28: Added `docs/data-sync.md` for private remote sync of the nested `data/` repo. - 2026-05-20: Removed `docs/workflows/`; project skills are now the sole workflow surface. diff --git a/docs/architecture.md b/docs/architecture.md index c44897a..8c1f414 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -27,7 +27,6 @@ Never commit `data/` contents to the outer repository. The outer `.gitignore` ex | Raw | `data/raw/` | Captured source evidence | Create only; never edit existing files | | Handoffs | `data/handoffs/` | Operational state and agent handoff | Update during tasks and periodic runs | | Wiki | `data/wiki/` | Durable long-term knowledge | Update only with source-backed frontmatter | -| Rejected | `data/rejected/` | Wiki pages rejected during review (audit trail) | Populated through DB API reject endpoint; mirrors `wiki/` tree | | Log | `data/log.md` | Append-only operation history | Append every operation | | Skill templates | `.claude/skills/*/reference/templates` | Runtime file skeletons bundled with skills | Update in outer repo | | Raw templates | `docs/raw/` | Raw source frontmatter skeletons | Update in outer repo | @@ -48,7 +47,7 @@ data/wiki/ summaries/ # daily, weekly, monthly, migration rollups ``` -Six of the seven types (`entity`, `concept`, `decision`, `improvement`, `checklist`, `question`) carry a `review_status` field (`not_processed` → `pending_for_approve` → `approved`) managed through DB API (promote/approve/reject endpoints). Summaries are exempt. Only `approved` pages appear in `INDEX.md`. Runtime approval work uses `.claude/skills/wiki-approval/SKILL.md`. +`INDEX.md` is generated export from the DB. ### Operating Flow @@ -58,7 +57,7 @@ The DB-canonical flow is: data/raw/ -> kb-mcp tools / service layer -> Postgres write -> data/wiki/ (Markdown export) -> data/log.md -> lint ``` -Writes go through the `kb-mcp` MCP server tools or the in-process `kb.service` layer (used by cron CLIs); Markdown files under `data/wiki/` are generated exports +Writes go through the `kb-mcp` MCP server tools or the in-process `kb.service` layer; Markdown files under `data/wiki/` are generated exports exported from Postgres. Handoffs sit beside the flow as the operational state board. They record what was processed, what is blocked, and what the next agent should do. @@ -76,7 +75,7 @@ is a generated export, not a read surface. | Component | Path | Role | |---|---|---| | `kb-mcp` FastMCP server | `src/kb/mcp/` | 12 write tools + `query_sql`/`get_schema` read tools over Postgres | -| Service layer | `src/kb/service/` | In-process write path (lint → DB → export); called by cron CLIs without a running server | +| Service layer | `src/kb/service/` | In-process write path (lint → DB → export); no running server needed | ## 3. Usage diff --git a/docs/db-canonical.md b/docs/db-canonical.md index da0c14d..0ae416d 100644 --- a/docs/db-canonical.md +++ b/docs/db-canonical.md @@ -19,13 +19,12 @@ The DB owns durable memory: | Table | Role | |---|---| -| `pages` | Canonical wiki page record: slug, title, body markdown, frontmatter JSON, review state, type/category/tags payload. | +| `pages` | Canonical wiki page record: slug, title, body markdown, frontmatter JSON, type/category/tags payload. | | `raw_sources` | Immutable captured evidence, replacing `data/raw/` as the canonical source store. | | `page_sources` | Many-to-many citation links between pages and raw sources. | | `page_revisions` | Append-only page revision ledger for body/frontmatter changes. | | `dispatches` | Existing external workflow dispatch state. | | `wiki_edits` | Former legacy frontmatter edit audit, now removed. `page_revisions` is the sole audit table. | -| `metrics` | Operational metrics (promotions, rejections, TTL sweeps, page counts) for dashboard and reporting. | Markdown exports are generated from DB rows. They may still be committed, searched, or opened in external tools, but a Markdown file no longer wins over a @@ -39,7 +38,7 @@ DB row when the two disagree. 3. ✅ Reads go directly against Postgres (`psql`), not Markdown scans or read endpoints — the web app is write-only. See `docs/db_informations/state-db-schema-reference.md`. -4. ✅ Move mutation endpoints (create/approve/reject/frontmatter edits) into one +4. ✅ Move mutation endpoints (create/update/frontmatter edits) into one DB transaction that also appends `page_revisions`, with write-time lint. 5. ✅ Add Markdown export as an explicit step. Export is allowed to overwrite generated files because it is derived output. diff --git a/docs/db_informations/state-db-schema-reference.md b/docs/db_informations/state-db-schema-reference.md index 7fc8f30..c75bba3 100644 --- a/docs/db_informations/state-db-schema-reference.md +++ b/docs/db_informations/state-db-schema-reference.md @@ -6,7 +6,7 @@ Updated: 2026-06-04 - **Purpose**: Let agents/skills **read** the canonical KnowledgeBase state directly. Postgres is the sole source of truth; `data/` is a generated human-readable export. - **I/O**: `psql "$DATABASE_URL" -tAc " fetched from GET /api/kanban/boards │ -│ ├── direction