diff --git a/README.md b/README.md index 37537aa..3587eff 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,16 @@ python -m claude_prospector dashboard --track-mcp-calls python -m claude_prospector dashboard --track-mcp-call-sizes ``` +The Skills tab also reports manual Claude Code built-in slash-command usage, +including invocation counts and distinct sessions. Classification comes from a +dated snapshot of the official Claude Code command reference; bundled skills +and workflows (including `/doctor`) are excluded, while unknown command names +are shown separately for auditability. Only the `` value and its +timestamp are retained as parsed command records for aggregation. Command +arguments and surrounding prompt text are read transiently while parsing the +transcript JSON, but claude-prospector never retains them or writes them to the +dashboard. + `--track-mcp-calls` adds a per-session MCP tool-call collection pass, which costs additional full transcript reads: measured on the maintainer's real corpus (~1,800 transcript files, 796 MB), `dashboard --format json` took 4.62s @@ -741,7 +751,7 @@ The table below lists everything `claude-prospector` writes under that base dire | Path | Contents | Written by | Contains prompt text? | |---|---|---|---| -| `dashboard.html` | Aggregated token/cost stats | `dashboard` subcommand, or the opt-in `dashboard-regen` Stop hook | No | +| `dashboard.html` | Aggregated token/cost, skill, and command-name stats | `dashboard` subcommand, or the opt-in `dashboard-regen` Stop hook | No — command arguments and surrounding prompt text are read transiently from transcript JSON, but are never retained or written by claude-prospector or the dashboard | | `hook.log` | One diagnostic line, e.g. `skipped: no skills found in Agent prompt for `; truncated and overwritten on every hook run | All hooks | No — logs the target agent *name*, never prompt content | | `config.json` | User settings (`project_exclude_patterns`, legacy `autoregen`) | `config` subcommand / manual edit | No | | `skill-tracking/.jsonl` | Skill name, timestamp, session-id, and (for Agent dispatches) target agent name, for each `Skill`/`Agent` tool-use event | `skill-tracker` PreToolUse hook — runs automatically on every `Skill`/`Agent` tool call once setup is `VALID` | No — only the matched skill *name* is stored, never the surrounding prompt | diff --git a/pyproject.toml b/pyproject.toml index b4ea89b..fef92db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ where = ["src"] include = ["claude_prospector*"] [tool.setuptools.package-data] -claude_prospector = ["templates/*.html", "static/**/*"] +claude_prospector = ["templates/*.html", "static/**/*", "data/*.json"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/claude_prospector/aggregator.py b/src/claude_prospector/aggregator.py index 018c7b8..b90f66e 100644 --- a/src/claude_prospector/aggregator.py +++ b/src/claude_prospector/aggregator.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone +from claude_prospector.builtin_commands import load_command_catalog from claude_prospector.constants import AGENT_PATH_SEPARATOR as _AGENT_PATH_SEPARATOR from claude_prospector.mcp_names import normalize_mcp_tool_name from claude_prospector.models import ( @@ -59,6 +60,7 @@ class AggregateResult: sessions: list[dict] = field(default_factory=list) by_skill_adoption: dict[str, dict] = field(default_factory=dict) by_mcp_usage: dict[str, dict] = field(default_factory=dict) + by_command_usage: dict[str, dict] = field(default_factory=dict) def _add_tokens(bucket: dict, msg: MessageRecord) -> None: @@ -94,6 +96,66 @@ def _agent_activity(msg: MessageRecord) -> dict: } +def _compute_command_usage( + sessions: list[SessionRecord], + from_date: datetime | None, + to_date: datetime | None, +) -> dict[str, dict]: + """Aggregate manual built-in commands for the selected dashboard window. + + Args: + sessions: Parsed sessions containing command records. + from_date: Inclusive lower timestamp bound. + to_date: Exclusive upper timestamp bound. + + Returns: + Classification metadata plus built-in and unclassified command counts. + Bundled skills and workflows are excluded. + """ + catalog = load_command_catalog() + command_counts: Counter[str] = Counter() + command_sessions: dict[str, set[str]] = defaultdict(set) + unclassified_counts: Counter[str] = Counter() + unclassified_sessions: dict[str, set[str]] = defaultdict(set) + + for session in sessions: + for command in session.commands: + if from_date and command.timestamp < from_date: + continue + if to_date and command.timestamp >= to_date: + continue + kind = catalog.classify(command.name) + if kind == "builtin": + command_counts[command.name] += 1 + command_sessions[command.name].add(session.session_id) + elif kind == "unclassified": + unclassified_counts[command.name] += 1 + unclassified_sessions[command.name].add(session.session_id) + + def summarize( + counts: Counter[str], + used_sessions: dict[str, set[str]], + ) -> dict[str, dict[str, int]]: + """Convert counters and session sets into the public payload shape.""" + return { + name: { + "invocation_count": counts[name], + "sessions_used_in": len(used_sessions[name]), + } + for name in sorted(counts) + } + + return { + "classification": { + "available": catalog.available, + "source_url": catalog.source_url, + "retrieved_at": catalog.retrieved_at, + }, + "by_command": summarize(command_counts, command_sessions), + "unclassified": summarize(unclassified_counts, unclassified_sessions), + } + + def aggregate( sessions: list[SessionRecord], from_date: datetime | None = None, @@ -278,6 +340,7 @@ def aggregate( result.by_day[day]["by_model"][model] += msg.total_tokens result.sessions.sort(key=lambda s: s["start_time"], reverse=True) + result.by_command_usage = _compute_command_usage(sessions, from_date, to_date) return result diff --git a/src/claude_prospector/builtin_commands.py b/src/claude_prospector/builtin_commands.py new file mode 100644 index 0000000..b245e9c --- /dev/null +++ b/src/claude_prospector/builtin_commands.py @@ -0,0 +1,142 @@ +"""Classify Claude Code slash commands using a packaged catalog.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from datetime import date +from functools import lru_cache +from importlib import resources +from typing import Literal + + +CommandKind = Literal["builtin", "bundled_skill", "workflow", "unclassified"] +_COMMAND_NAME_RE = re.compile(r"/[^\s<>]+") + + +@dataclass(frozen=True, slots=True) +class CommandCatalog: + """An auditable snapshot of Claude Code command categories. + + Attributes: + available: Whether the packaged catalog loaded successfully. + source_url: Official documentation URL used to build the snapshot. + retrieved_at: ISO date when the source was retrieved. + builtins: Literal names classified as built-in commands. + bundled_skills: Literal names classified as bundled skills. + workflows: Literal names classified as bundled workflows. + """ + + available: bool = False + source_url: str | None = None + retrieved_at: str | None = None + builtins: frozenset[str] = frozenset() + bundled_skills: frozenset[str] = frozenset() + workflows: frozenset[str] = frozenset() + + def classify(self, command_name: str) -> CommandKind: + """Classify one literal slash-command name. + + Args: + command_name: Command name including its leading slash. + + Returns: + The catalog category, or ``"unclassified"`` when unknown. + """ + if command_name in self.builtins: + return "builtin" + if command_name in self.bundled_skills: + return "bundled_skill" + if command_name in self.workflows: + return "workflow" + return "unclassified" + + +def _validated_commands(payload: object, field_name: str) -> frozenset[str]: + """Validate and normalize one catalog category. + + Args: + payload: Decoded JSON value for the category. + field_name: Category name used in validation errors. + + Returns: + Validated unique command names. + + Raises: + TypeError: If the value is not a list of strings. + ValueError: If names are duplicated or malformed. + """ + if not isinstance(payload, list) or not all( + isinstance(name, str) for name in payload + ): + raise TypeError(f"{field_name} must be a list of strings") + names = frozenset(payload) + if len(names) != len(payload): + raise ValueError(f"{field_name} contains duplicate names") + if any(_COMMAND_NAME_RE.fullmatch(name) is None for name in names): + raise ValueError(f"{field_name} contains an invalid command name") + return names + + +def _catalog_from_payload(payload: object) -> CommandCatalog: + """Build a catalog only when its decoded JSON schema is valid. + + Args: + payload: Decoded catalog JSON. + + Returns: + An available, semantically validated command catalog. + + Raises: + KeyError: If a required field is absent. + TypeError: If a field has the wrong type. + ValueError: If provenance or command categories are invalid. + """ + if not isinstance(payload, dict): + raise TypeError("catalog must be an object") + source_url = payload["source_url"] + retrieved_at = payload["retrieved_at"] + if not isinstance(source_url, str) or not source_url.startswith("https://"): + raise ValueError("source_url must be an HTTPS URL") + if not isinstance(retrieved_at, str): + raise TypeError("retrieved_at must be a string") + date.fromisoformat(retrieved_at) + + builtins = _validated_commands(payload["builtins"], "builtins") + bundled_skills = _validated_commands( + payload["bundled_skills"], + "bundled_skills", + ) + workflows = _validated_commands(payload["workflows"], "workflows") + if builtins & bundled_skills or builtins & workflows or bundled_skills & workflows: + raise ValueError("command categories must be disjoint") + + return CommandCatalog( + available=True, + source_url=source_url, + retrieved_at=retrieved_at, + builtins=builtins, + bundled_skills=bundled_skills, + workflows=workflows, + ) + + +@lru_cache(maxsize=1) +def load_command_catalog() -> CommandCatalog: + """Load the packaged command catalog. + + Returns: + The packaged catalog, or an unavailable catalog when its resource is + missing or malformed. + """ + try: + catalog_text = ( + resources.files("claude_prospector") + .joinpath("data/claude-code-commands.json") + .read_text(encoding="utf-8") + ) + payload = json.loads(catalog_text) + return _catalog_from_payload(payload) + except (KeyError, OSError, TypeError, ValueError): + return CommandCatalog() diff --git a/src/claude_prospector/cli/dashboard.py b/src/claude_prospector/cli/dashboard.py index 26e5fc1..ac6f857 100644 --- a/src/claude_prospector/cli/dashboard.py +++ b/src/claude_prospector/cli/dashboard.py @@ -270,6 +270,7 @@ def run(args: argparse.Namespace) -> int: "by_model": result.by_model, "by_agent": result.by_agent, "by_skill": result.by_skill, + "by_command_usage": result.by_command_usage, "by_project": result.by_project, "by_day": result.by_day, "sessions": result.sessions, diff --git a/src/claude_prospector/data/claude-code-commands.json b/src/claude_prospector/data/claude-code-commands.json new file mode 100644 index 0000000..254479d --- /dev/null +++ b/src/claude_prospector/data/claude-code-commands.json @@ -0,0 +1,143 @@ +{ + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + "classification_note": "Commands marked Skill or Workflow in the official table are excluded from built-ins; documented aliases inherit the canonical command category.", + "builtins": [ + "/add-dir", + "/advisor", + "/agents", + "/allowed-tools", + "/android", + "/app", + "/artifacts", + "/auto-mode-setup", + "/autocompact", + "/autofix-pr", + "/background", + "/bashes", + "/bg", + "/branch", + "/btw", + "/bug", + "/cd", + "/checkpoint", + "/chrome", + "/clear", + "/color", + "/compact", + "/config", + "/context", + "/continue", + "/copy", + "/cost", + "/design-login", + "/desktop", + "/diff", + "/effort", + "/exit", + "/export", + "/fast", + "/feedback", + "/focus", + "/fork", + "/goal", + "/heapdump", + "/help", + "/hooks", + "/ide", + "/import", + "/init", + "/insights", + "/install-github-app", + "/install-slack-app", + "/ios", + "/keybindings", + "/list-agents", + "/login", + "/logout", + "/mcp", + "/memory", + "/mobile", + "/model", + "/new", + "/passes", + "/peers", + "/permissions", + "/plan", + "/plugin", + "/powerup", + "/pr-comments", + "/privacy-settings", + "/quit", + "/radio", + "/rate-limit-options", + "/rc", + "/recap", + "/release-notes", + "/reload-plugins", + "/reload-skills", + "/remote-control", + "/remote-env", + "/rename", + "/reset", + "/resume", + "/rewind", + "/routines", + "/sandbox", + "/schedule", + "/scroll-speed", + "/security-review", + "/settings", + "/setup-bedrock", + "/setup-vertex", + "/share", + "/skill-doctor", + "/skills", + "/stats", + "/status", + "/statusline", + "/stickers", + "/stop", + "/subtask", + "/tasks", + "/team-onboarding", + "/teleport", + "/terminal-setup", + "/theme", + "/tp", + "/tui", + "/ultraplan", + "/undo", + "/upgrade", + "/usage", + "/usage-credits", + "/vim", + "/voice", + "/web-setup", + "/workflows" + ], + "bundled_skills": [ + "/batch", + "/checkup", + "/claude-api", + "/code-review", + "/dataviz", + "/debug", + "/design", + "/design-sync", + "/doctor", + "/fewer-permission-prompts", + "/loop", + "/proactive", + "/review", + "/run", + "/run-skill-generator", + "/simplify", + "/ultrareview", + "/verify", + "/workflow-authoring" + ], + "workflows": [ + "/deep-research" + ] +} diff --git a/src/claude_prospector/models.py b/src/claude_prospector/models.py index d35e968..dcfd22d 100644 --- a/src/claude_prospector/models.py +++ b/src/claude_prospector/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime @@ -69,6 +69,19 @@ def model_short(self) -> str: return self.model +@dataclass(frozen=True, slots=True) +class CommandInvocationRecord: + """A manual slash-command invocation from an external user entry. + + Attributes: + name: Literal command name, including its leading slash. + timestamp: When the user invoked the command. + """ + + name: str + timestamp: datetime + + @dataclass(frozen=True, slots=True) class SessionRecord: """A parsed session with all its messages (including subagent messages). @@ -89,6 +102,8 @@ class SessionRecord: messages: All messages from this session and its subagents. subagent_types: Sorted, de-duplicated list of subagent type names encountered at any depth. + commands: Manual slash-command invocations. Records retain only the + command name and timestamp, never arguments or prompt content. """ session_id: str @@ -98,6 +113,7 @@ class SessionRecord: root_agent: str messages: list[MessageRecord] subagent_types: list[str] + commands: list[CommandInvocationRecord] = field(default_factory=list) @property def total_tokens(self) -> int: diff --git a/src/claude_prospector/parser.py b/src/claude_prospector/parser.py index b37a43f..c85fe45 100644 --- a/src/claude_prospector/parser.py +++ b/src/claude_prospector/parser.py @@ -8,10 +8,25 @@ from datetime import datetime, timezone from pathlib import Path -from claude_prospector.models import MessageRecord, SessionRecord +from claude_prospector.models import ( + CommandInvocationRecord, + MessageRecord, + SessionRecord, +) from claude_prospector.transcript_walker import walk_session +_COMMAND_ENVELOPE_RE = re.compile( + r"\A\s*(?:" + r"[^<>]*\s*" + r"(/[^\s<>]+)" + r"|" + r"(/[^\s<>]+)" + r"(?:\s*[^<>]*)?" + r")\s*\Z" +) + + def decode_project_hash(hash_name: str) -> str: """Decode a project hash directory name to a human-readable project name. @@ -313,14 +328,60 @@ def _extract_skill(content: list[dict]) -> str | None: return None -def _parse_jsonl_messages( +def _extract_manual_command(entry: dict) -> CommandInvocationRecord | None: + """Extract one privacy-safe manual command from an external user entry. + + Args: + entry: Decoded transcript entry. + + Returns: + A name-and-timestamp record when the canonical wrapper is valid; + otherwise ``None``. Content inside ```` is never scanned. + """ + message = entry.get("message") + if not isinstance(message, dict): + return None + content = message.get("content") + timestamp_raw = entry.get("timestamp") + if not isinstance(content, str) or not isinstance(timestamp_raw, str): + return None + + command_wrapper = content.partition(" list[MessageRecord]: - """Parse assistant messages from a JSONL file, attributing to agent.""" + *, + collect_commands: bool = True, +) -> tuple[list[MessageRecord], list[CommandInvocationRecord]]: + """Parse assistant messages and manual commands in one transcript pass. + + Args: + jsonl_path: Transcript JSONL file to parse. + agent_type: Leaf agent name assigned to assistant messages. + agent_path: Full agent ancestry assigned to assistant messages. + collect_commands: Whether to extract manual command records. This is + enabled only for a session's root transcript. + + Returns: + Assistant message records and manual command records. Command records + retain only the command-name tag and timestamp. + """ messages: list[MessageRecord] = [] + commands: list[CommandInvocationRecord] = [] message_indexes: dict[str, int] = {} + command_entry_ids: set[str] = set() with open(jsonl_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() @@ -331,6 +392,20 @@ def _parse_jsonl_messages( except json.JSONDecodeError: continue + if ( + collect_commands + and entry.get("type") == "user" + and entry.get("userType") == "external" + ): + entry_id = entry.get("uuid") + if isinstance(entry_id, str) and entry_id in command_entry_ids: + continue + command = _extract_manual_command(entry) + if command is not None: + commands.append(command) + if isinstance(entry_id, str): + command_entry_ids.add(entry_id) + if entry.get("type") != "assistant": continue @@ -370,6 +445,30 @@ def _parse_jsonl_messages( ) if message_id is not None: message_indexes[message_id] = len(messages) - 1 + return messages, commands + + +def _parse_jsonl_messages( + jsonl_path: Path, + agent_type: str, + agent_path: tuple[str, ...] = (), +) -> list[MessageRecord]: + """Parse assistant messages from a JSONL file, attributing to agent. + + Args: + jsonl_path: Transcript JSONL file to parse. + agent_type: Leaf agent name assigned to assistant messages. + agent_path: Full agent ancestry assigned to assistant messages. + + Returns: + Parsed assistant message records. + """ + messages, _ = _parse_jsonl_records( + jsonl_path, + agent_type, + agent_path, + collect_commands=False, + ) return messages @@ -440,14 +539,16 @@ def _parse_session( transcripts, subagent_types = walk_session(jsonl_path, root_agent) messages: list[MessageRecord] = [] + commands: list[CommandInvocationRecord] = [] for unit in transcripts: - messages.extend( - _parse_jsonl_messages( - unit.jsonl_path, - agent_type=unit.agent_type, - agent_path=unit.agent_path, - ) + unit_messages, unit_commands = _parse_jsonl_records( + unit.jsonl_path, + agent_type=unit.agent_type, + agent_path=unit.agent_path, + collect_commands=unit.jsonl_path == jsonl_path, ) + messages.extend(unit_messages) + commands.extend(unit_commands) if not messages: start_time = datetime.now(timezone.utc) @@ -462,6 +563,7 @@ def _parse_session( root_agent=root_agent, messages=messages, subagent_types=sorted(set(subagent_types)), + commands=commands, ) diff --git a/src/claude_prospector/renderer.py b/src/claude_prospector/renderer.py index a9d82bc..7c3e7aa 100644 --- a/src/claude_prospector/renderer.py +++ b/src/claude_prospector/renderer.py @@ -92,6 +92,7 @@ def render( "by_agent": result.by_agent, "by_skill": result.by_skill, "by_skill_adoption": result.by_skill_adoption, + "by_command_usage": result.by_command_usage, "by_project": result.by_project, "by_day": result.by_day, "sessions": result.sessions, diff --git a/src/claude_prospector/static/views/skills.js b/src/claude_prospector/static/views/skills.js index 6630cae..e4e129a 100644 --- a/src/claude_prospector/static/views/skills.js +++ b/src/claude_prospector/static/views/skills.js @@ -77,6 +77,29 @@ } .skills-style .skills-empty { padding: 28px 20px; text-align: center; color: #8b949e; } .skills-style .skills-empty strong { display: block; color: #f0f6fc; margin-bottom: 5px; } + .skills-style .command-report { + margin-top: 28px; padding-top: 24px; border-top: 1px solid #21262d; + } + .skills-style .command-heading { margin-bottom: 12px; } + .skills-style .command-heading h2 { + margin: 0; color: #f0f6fc; font-size: 17px; font-weight: 600; + } + .skills-style .command-heading p { + margin: 4px 0 0; color: #8b949e; font-size: 11px; line-height: 1.45; + } + .skills-style .command-table { min-width: 440px; } + .skills-style .command-name { + color: #79c0ff; font-family: ui-monospace, SFMono-Regular, Consolas, + 'Liberation Mono', monospace; font-weight: 600; + } + .skills-style .command-warning { + margin-top: 12px; padding: 10px 12px; color: #c9d1d9; + background: #161b22; border: 1px solid #21262d; + border-left: 3px solid #d29922; border-radius: 8px; font-size: 12px; + } + .skills-style .command-warning strong { color: #d29922; } + .skills-style .command-warning ul { margin: 7px 0 0; padding-left: 20px; } + .skills-style .command-warning li { margin: 3px 0; } @media (max-width: 600px) { .skills-style .skills-toolbar { align-items: stretch; } .skills-style input#skill-name-filter { width: 100%; min-width: 0; } @@ -251,6 +274,91 @@ ${table}`; } + function sortedCommandEntries(commands) { + return Object.entries(commands || {}).sort((left, right) => ( + (Number(right[1].invocation_count) || 0) + - (Number(left[1].invocation_count) || 0) + || String(left[0]).localeCompare(String(right[0])) + )); + } + + function commandRows(entries) { + return entries.map(([name, info]) => ` + + ${CP.esc(name)} + ${CP.esc(CP.fmtTokens(info.invocation_count || 0))} + ${CP.esc(CP.fmtTokens(info.sessions_used_in || 0))} + `).join(''); + } + + function renderCommandUsage(usage) { + usage = usage || {}; + const classification = usage.classification || {}; + const provenance = classification.retrieved_at + ? `Official Claude Code command reference · catalog retrieved ${CP.esc(classification.retrieved_at)}` + : 'Official Claude Code command reference'; + const heading = ` +
+

Built-in Commands

+

${provenance} · command names only; arguments are never retained.

+
`; + + if (!classification.available) { + return ` +
+ ${heading} +
+ Command classification unavailable + The packaged command catalog could not be loaded. +
+
`; + } + + const builtins = sortedCommandEntries(usage.by_command); + const unclassified = sortedCommandEntries(usage.unclassified); + if (builtins.length === 0 && unclassified.length === 0) { + return ` +
+ ${heading} +
+ No manual built-in command usage recorded + Invoke a built-in slash command to populate this report. +
+
`; + } + + const table = builtins.length === 0 ? ` +
No classified built-in commands recorded.
` : ` +
+ + + + + + + + + ${commandRows(builtins)} +
CommandInvocationsSessions
+
`; + const warning = unclassified.length === 0 ? '' : ` + `; + return ` +
+ ${heading} + ${table} + ${warning} +
`; + } + function renderSkills(root) { if (!document.getElementById('skills-css')) { const style = document.createElement('style'); @@ -275,6 +383,7 @@ aria-label="Search skills by name">
+ ${renderCommandUsage(window.DATA.by_command_usage)} `; const filterInput = root.querySelector('#skill-name-filter'); diff --git a/tests/fixtures/dashboard_snapshot_pre_refactor.json b/tests/fixtures/dashboard_snapshot_pre_refactor.json index d03ce29..4700d9d 100644 --- a/tests/fixtures/dashboard_snapshot_pre_refactor.json +++ b/tests/fixtures/dashboard_snapshot_pre_refactor.json @@ -26,6 +26,15 @@ } }, "by_skill": {}, + "by_command_usage": { + "classification": { + "available": true, + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06" + }, + "by_command": {}, + "unclassified": {} + }, "by_project": { "fake-project-abc123": { "total_tokens": 15, diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py index c05bc82..9cefe9a 100644 --- a/tests/test_aggregator.py +++ b/tests/test_aggregator.py @@ -2,7 +2,11 @@ from pathlib import Path from claude_prospector.aggregator import AGENT_PATH_SEPARATOR, aggregate -from claude_prospector.models import MessageRecord, SessionRecord +from claude_prospector.models import ( + CommandInvocationRecord, + MessageRecord, + SessionRecord, +) from claude_prospector.parser import parse_sessions @@ -36,6 +40,7 @@ def _session( project="proj", root_agent="general-purpose", subagent_types=None, + commands=None, ): start = ( min(m.timestamp for m in messages) @@ -50,6 +55,15 @@ def _session( root_agent=root_agent, messages=messages, subagent_types=subagent_types or [], + commands=commands or [], + ) + + +def _command(name: str, hour: int, minute: int = 0) -> CommandInvocationRecord: + """Build a command record at a hand-controlled UTC timestamp.""" + return CommandInvocationRecord( + name=name, + timestamp=datetime(2026, 4, 9, hour, minute, tzinfo=timezone.utc), ) @@ -261,6 +275,56 @@ def test_groups_by_skill(self): assert None not in result.by_skill +class TestAggregateCommandUsage: + def test_counts_builtins_by_invocation_and_session_within_window(self) -> None: + """Windowed totals exclude bundled skills and surface unknown names.""" + sessions = [ + _session( + [_msg()], + session_id="s1", + commands=[ + _command("/fork", 12, 5), + _command("/fork", 12, 10), + _command("/branch", 12, 15), + _command("/doctor", 12, 20), + _command("/project-review", 12, 25), + ], + ), + _session( + [_msg()], + session_id="s2", + commands=[ + _command("/fork", 12, 30), + _command("/fork", 14), + ], + ), + ] + + result = aggregate( + sessions, + from_date=datetime(2026, 4, 9, 12, tzinfo=timezone.utc), + to_date=datetime(2026, 4, 9, 13, tzinfo=timezone.utc), + ) + + assert result.by_command_usage == { + "classification": { + "available": True, + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + }, + "by_command": { + "/branch": {"invocation_count": 1, "sessions_used_in": 1}, + "/fork": {"invocation_count": 3, "sessions_used_in": 2}, + }, + "unclassified": { + "/project-review": { + "invocation_count": 1, + "sessions_used_in": 1, + } + }, + } + + class TestAggregateByProject: def test_groups_by_project(self): sessions = [ diff --git a/tests/test_builtin_commands.py b/tests/test_builtin_commands.py new file mode 100644 index 0000000..982ab16 --- /dev/null +++ b/tests/test_builtin_commands.py @@ -0,0 +1,115 @@ +"""Behavior tests for the packaged Claude Code command catalog.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from claude_prospector import builtin_commands +from claude_prospector.builtin_commands import load_command_catalog + + +def test_catalog_classifies_builtins_skills_workflows_and_unknowns() -> None: + """Catalog categories must keep non-built-ins out of usage totals.""" + catalog = load_command_catalog() + + assert catalog.available is True + assert catalog.classify("/compact") == "builtin" + assert catalog.classify("/fork") == "builtin" + assert catalog.classify("/batch") == "bundled_skill" + assert catalog.classify("/doctor") == "bundled_skill" + assert catalog.classify("/checkup") == "bundled_skill" + assert catalog.classify("/deep-research") == "workflow" + assert catalog.classify("/project-review") == "unclassified" + + +@pytest.mark.parametrize("alias", ["/peers", "/undo", "/bashes", "/tp"]) +def test_catalog_classifies_documented_builtin_aliases(alias: str) -> None: + """Documented aliases must contribute to built-in command usage.""" + catalog = load_command_catalog() + + assert catalog.classify(alias) == "builtin" + + +def test_catalog_exposes_a_dated_official_source() -> None: + """Users must be able to audit where the classification came from.""" + catalog = load_command_catalog() + + assert catalog.source_url == "https://code.claude.com/docs/en/commands" + assert catalog.retrieved_at == "2026-09-06" + + +@pytest.mark.parametrize( + "payload", + [ + { + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + "builtins": "/fork", + "bundled_skills": [], + "workflows": [], + }, + { + "source_url": None, + "retrieved_at": "2026-09-06", + "builtins": ["/fork"], + "bundled_skills": [], + "workflows": [], + }, + { + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + "builtins": ["/fork"], + "bundled_skills": ["/fork"], + "workflows": [], + }, + { + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + "builtins": ["fork without slash"], + "bundled_skills": [], + "workflows": [], + }, + ], +) +def test_malformed_catalogs_fall_back_to_unavailable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + payload: dict[str, object], +) -> None: + """Invalid catalog structure cannot produce misleading classifications.""" + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "claude-code-commands.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + monkeypatch.setattr(builtin_commands.resources, "files", lambda _: tmp_path) + load_command_catalog.cache_clear() + + try: + catalog = load_command_catalog() + finally: + load_command_catalog.cache_clear() + + assert catalog.available is False + assert catalog.source_url is None + assert catalog.classify("/fork") == "unclassified" + + +def test_missing_catalog_falls_back_to_unavailable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A missing package resource degrades without breaking the dashboard.""" + monkeypatch.setattr(builtin_commands.resources, "files", lambda _: tmp_path) + load_command_catalog.cache_clear() + + try: + catalog = load_command_catalog() + finally: + load_command_catalog.cache_clear() + + assert catalog.available is False diff --git a/tests/test_cli.py b/tests/test_cli.py index c7dfbfe..a3872f3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -29,6 +29,7 @@ def run_cli(args: list[str], cwd: Path) -> subprocess.CompletedProcess: "by_agent", "by_skill", "by_skill_adoption", + "by_command_usage", "by_project", "by_day", "sessions", @@ -90,6 +91,47 @@ def test_contains_aggregated_data(self, sample_session_dir: Path): assert "general-purpose" in data["by_agent"] assert len(data["sessions"]) == 1 + def test_contains_manual_builtin_command_usage( + self, sample_session_dir: Path + ) -> None: + """JSON output carries command names without command arguments.""" + session_file = next((sample_session_dir / "projects").glob("*/*.jsonl")) + command_entry = { + "type": "user", + "userType": "external", + "uuid": "command-1", + "timestamp": "2026-04-09T12:02:00.000Z", + "message": { + "role": "user", + "content": ( + "/fork" + "private follow-up prompt" + ), + }, + } + with session_file.open("a", encoding="utf-8") as stream: + stream.write("\n" + json.dumps(command_entry) + "\n") + + result = run_cli( + [ + "dashboard", + "--format", + "json", + "--no-open", + "--data-dir", + str(sample_session_dir), + ], + cwd=WORKTREE_ROOT, + ) + + assert result.returncode == 0, f"CLI failed:\n{result.stderr}" + data = json.loads(result.stdout) + assert data["by_command_usage"]["by_command"]["/fork"] == { + "invocation_count": 1, + "sessions_used_in": 1, + } + assert "private follow-up prompt" not in result.stdout + def test_generated_at_is_iso8601(self, sample_session_dir: Path): """generated_at must be a valid ISO-8601 datetime string.""" from datetime import datetime diff --git a/tests/test_parser.py b/tests/test_parser.py index 0a82f13..cf281bc 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -97,6 +97,243 @@ def test_no_projects_dir(self, tmp_path: Path): sessions = parse_sessions(tmp_path) assert sessions == [] + def test_extracts_only_manual_command_name(self, tmp_path: Path): + """A manual command retains its name and timestamp, not its arguments.""" + jsonl = tmp_path / "manual-command.jsonl" + entries = [ + { + "type": "user", + "userType": "external", + "uuid": "user-command-1", + "timestamp": "2026-04-09T12:00:01.000Z", + "message": { + "role": "user", + "content": ( + "/fork" + "keep " + "/secret-client private" + "" + ), + }, + }, + _make_assistant_line("manual-command"), + ] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert [(record.name, record.timestamp) for record in session.commands] == [ + ("/fork", datetime(2026, 4, 9, 12, 0, 1, tzinfo=timezone.utc)) + ] + assert "secret-client" not in repr(session.commands) + + def test_command_name_tag_after_ordinary_prose_is_not_manual_usage( + self, + tmp_path: Path, + ) -> None: + """Ordinary prompts containing a command tag cannot become usage.""" + jsonl = tmp_path / "ordinary-prose-command-tag.jsonl" + entries = [ + { + "type": "user", + "userType": "external", + "uuid": "ordinary-prose-command-tag", + "timestamp": "2026-04-09T12:00:01.000Z", + "message": { + "role": "user", + "content": "Explain /fork", + }, + }, + _make_assistant_line("ordinary-prose-command-tag"), + ] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert session.commands == [] + + @pytest.mark.parametrize( + "content", + [ + ( + "fork\n" + "/fork\n" + "private details" + ), + ( + "/fork\n" + "fork\n" + "private details" + ), + ], + ) + def test_canonical_command_message_orderings_are_manual_usage( + self, + tmp_path: Path, + content: str, + ) -> None: + """Observed command-message orderings retain one command name.""" + jsonl = tmp_path / "canonical-command-envelope.jsonl" + entries = [ + { + "type": "user", + "userType": "external", + "uuid": "canonical-command-envelope", + "timestamp": "2026-04-09T12:00:01.000Z", + "message": {"role": "user", "content": content}, + }, + _make_assistant_line("canonical-command-envelope"), + ] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert [record.name for record in session.commands] == ["/fork"] + + def test_command_collection_deduplicates_entries_and_ignores_automatic_events( + self, + tmp_path: Path, + ) -> None: + """Repeated fragments and automatic summaries cannot inflate usage.""" + jsonl = tmp_path / "command-boundaries.jsonl" + manual = { + "type": "user", + "userType": "external", + "uuid": "user-command-1", + "timestamp": "2026-04-09T12:00:01.000Z", + "message": { + "role": "user", + "content": "/compact", + }, + } + automatic = { + "type": "system", + "subtype": "compact_boundary", + "uuid": "automatic-compact-1", + "timestamp": "2026-04-09T12:00:02.000Z", + "message": { + "content": "/compact", + }, + } + entries = [manual, manual, automatic, _make_assistant_line("boundaries")] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert [record.name for record in session.commands] == ["/compact"] + + def test_command_name_tag_cannot_capture_whitespace_or_arguments( + self, + tmp_path: Path, + ) -> None: + """Malformed command-name content cannot retain private arguments.""" + jsonl = tmp_path / "command-name-privacy.jsonl" + entries = [ + { + "type": "user", + "userType": "external", + "uuid": "command-with-inline-args", + "timestamp": "2026-04-09T12:00:00.000Z", + "message": { + "role": "user", + "content": ("/fork private details"), + }, + }, + _make_assistant_line("command-name-privacy"), + ] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert session.commands == [] + + def test_subagent_command_tags_are_not_manual_usage( + self, + sample_session_dir: Path, + ) -> None: + """Synthetic subagent prompts cannot become manual command usage.""" + subagent_file = next( + (sample_session_dir / "projects").glob("*/*/subagents/*.jsonl") + ) + synthetic_prompt = { + "type": "user", + "userType": "external", + "uuid": "synthetic-subagent-command", + "timestamp": "2026-04-09T12:06:00.000Z", + "message": { + "role": "user", + "content": "/fork", + }, + } + with subagent_file.open("a", encoding="utf-8") as stream: + stream.write("\n" + json.dumps(synthetic_prompt) + "\n") + + session = parse_sessions(sample_session_dir)[0] + + assert session.commands == [] + + def test_malformed_external_command_entries_are_ignored( + self, + tmp_path: Path, + ) -> None: + """Malformed user records cannot terminate dashboard parsing.""" + jsonl = tmp_path / "malformed-command-entries.jsonl" + entries = [ + { + "type": "user", + "userType": "external", + "message": None, + }, + { + "type": "user", + "userType": "external", + "uuid": "missing-timestamp", + "message": { + "content": "/fork", + }, + }, + { + "type": "user", + "userType": "external", + "uuid": "invalid-timestamp", + "timestamp": "not-a-timestamp", + "message": { + "content": "/fork", + }, + }, + _make_assistant_line("malformed-command-entries"), + ] + jsonl.write_text( + "\n".join(json.dumps(entry) for entry in entries), + encoding="utf-8", + ) + + session = _parse_session(jsonl, "proj") + + assert session is not None + assert session.commands == [] + assert len(session.messages) == 1 + # --------------------------------------------------------------------------- # Helpers for agent-setting resolution tests diff --git a/tests/test_phase2_shell.py b/tests/test_phase2_shell.py index a98d436..b25bfb8 100644 --- a/tests/test_phase2_shell.py +++ b/tests/test_phase2_shell.py @@ -376,15 +376,15 @@ def test_phase3_placeholder_present(self, tmp_path: Path) -> None: class TestJsonPayloadUnchanged: - """Verify the dashboard --format json payload shape is unchanged.""" + """Verify the dashboard --format json payload shape is intentional.""" def test_json_output_has_expected_top_level_keys(self) -> None: - """dashboard --format json must have the same top-level keys as before. + """dashboard --format json must have the approved top-level keys. Phase 2 itself was template-only — no Python-side data contract changes. This test still pins the pre-#256 key set, but now includes - "by_skill_adoption", added by #256; the rest of the key set remains - unchanged from Phase 2. + "by_skill_adoption", added by #256, and "by_command_usage", added by + #298. """ import json import subprocess @@ -432,6 +432,7 @@ def test_json_output_has_expected_top_level_keys(self) -> None: "by_agent", "by_skill", "by_skill_adoption", + "by_command_usage", "by_project", "by_day", "sessions", diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 4de6195..0a353f8 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -305,6 +305,31 @@ def test_real_data_depth3_renders_correct_by_agent_values( ) +def test_command_usage_reaches_embedded_dashboard_data(tmp_path: Path) -> None: + """The renderer preserves command counts and catalog provenance.""" + usage = { + "classification": { + "available": True, + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + }, + "by_command": {"/fork": {"invocation_count": 2, "sessions_used_in": 1}}, + "unclassified": {}, + } + output = tmp_path / "dashboard-commands.html" + render( + AggregateResult(by_command_usage=usage), + output_path=output, + open_browser=False, + ) + html = output.read_text(encoding="utf-8") + marker = "window.DATA = " + data_start = html.index(marker) + len(marker) + data, _ = json.JSONDecoder().raw_decode(html, data_start) + + assert data["by_command_usage"] == usage + + def test_data_json_escapes_script_breakout_from_mcp_usage_keys( tmp_path: Path, ) -> None: diff --git a/tests/test_skills_view.py b/tests/test_skills_view.py index 7119bec..ba35cd6 100644 --- a/tests/test_skills_view.py +++ b/tests/test_skills_view.py @@ -195,6 +195,104 @@ def test_available_adoption_labels_recorded_pass_events(tmp_path: Path) -> None: assert "2 recorded pass events" in rendered +def test_builtin_command_report_renders_counts_and_unclassified_names( + tmp_path: Path, +) -> None: + """Built-ins and unknown commands remain visibly distinct.""" + rendered = _render_skills( + tmp_path, + { + "by_skill": {}, + "by_skill_adoption": {}, + "by_mcp_usage": {}, + "by_command_usage": { + "classification": { + "available": True, + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + }, + "by_command": { + "/fork": {"invocation_count": 3, "sessions_used_in": 2}, + }, + "unclassified": { + "/project-review": { + "invocation_count": 1, + "sessions_used_in": 1, + }, + }, + }, + }, + )["root"] + + assert "Built-in Commands" in rendered + assert "/fork" in rendered + assert ">3" in rendered + assert ">2" in rendered + assert "/project-review" in rendered + assert "not counted as built-ins" in rendered + assert "2026-09-06" in rendered + + +def test_builtin_command_report_has_empty_and_unavailable_states( + tmp_path: Path, +) -> None: + """No observations and no catalog are reported differently.""" + empty = _render_skills( + tmp_path, + { + "by_skill": {}, + "by_skill_adoption": {}, + "by_mcp_usage": {}, + "by_command_usage": { + "classification": { + "available": True, + "source_url": "https://code.claude.com/docs/en/commands", + "retrieved_at": "2026-09-06", + }, + "by_command": {}, + "unclassified": {}, + }, + }, + )["root"] + unavailable = _render_skills( + tmp_path, + { + "by_skill": {}, + "by_skill_adoption": {}, + "by_mcp_usage": {}, + "by_command_usage": {}, + }, + )["root"] + + assert "No manual built-in command usage recorded" in empty + assert "Command classification unavailable" in unavailable + + +def test_builtin_command_names_are_html_escaped(tmp_path: Path) -> None: + """Transcript-derived unknown names cannot inject dashboard markup.""" + rendered = _render_skills( + tmp_path, + { + "by_skill": {}, + "by_skill_adoption": {}, + "by_mcp_usage": {}, + "by_command_usage": { + "classification": {"available": True}, + "by_command": {}, + "unclassified": { + "/": { + "invocation_count": 1, + "sessions_used_in": 1, + }, + }, + }, + }, + )["root"] + + assert "" not in rendered + assert "<script>alert(1)</script>" in rendered + + def test_gap_detection_uses_the_actual_adoption_rate() -> None: source = _source() assert "row.adoptionRate === 0" in source