From 9f32c3a0d7f523cb52f5c113e5a43daae610e62b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 24 Aug 2026 02:32:39 +0900 Subject: [PATCH 1/3] Bound persisted status metadata (#5161) --- AGENT_GUIDE.md | 3 +- DEVELOPER_GUIDE.md | 36 +- README.md | 31 +- changelog.d/unreleased/5161.fixed.md | 19 + .../Cli/IndexCommandRunner.FullScan.cs | 2 +- .../Cli/QueryCommandRunner.Status.cs | 8 +- .../Database/DbReader.FilesStatus.cs | 328 ++++++++++++++++-- ...bReader.ReferenceExtractionCompleteness.cs | 2 +- src/CodeIndex/Database/DbReader.Status.cs | 10 +- src/CodeIndex/Diagnostics/BoundedJson.cs | 12 + .../Mcp/McpToolHandlers.Indexing.Execution.cs | 2 +- src/CodeIndex/Models/QueryResults.cs | 14 + src/CodeIndex/Models/StatusMetadataLimits.cs | 14 + .../DbReaderStatusMetadataIssue5161Tests.cs | 261 ++++++++++++++ .../StatusMetadataOutputIssue5161Tests.cs | 201 +++++++++++ 15 files changed, 897 insertions(+), 46 deletions(-) create mode 100644 changelog.d/unreleased/5161.fixed.md create mode 100644 src/CodeIndex/Models/StatusMetadataLimits.cs create mode 100644 tests/CodeIndex.Tests/DbReaderStatusMetadataIssue5161Tests.cs create mode 100644 tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index ce2d28a719..3ffc269eba 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -143,7 +143,8 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `status_metadata_diagnostics`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- Persisted status subdocuments for reference-extraction cap hits, rebuild reclaim, and failed-file diagnostics use a 512 KiB UTF-8 input limit and maximum JSON depth 16 before semantic validation. Preserve the shared 50-item writer/reader limits, nested reason and string budgets documented in `README.md` and `DEVELOPER_GUIDE.md`, omission of only the rejected subdocument, and `status_metadata_diagnostics` reasons `raw_size_exceeded`, `invalid_json`, and `semantic_validation_failed`. Human first-failure and recovery-hint fields must pass through the bounded control-flattening formatter while accepted structured JSON remains unchanged. - `status --check` repair actions are structured by `name`, `action`, `args`, `mutation_class`, `safety_class`, and `safety_notes`. Preserve compatibility `reason` as the first trigger and ordered `reasons` as the complete trigger set. Deduplicate only exact structured identities; different targets, options, actions, mutation classes, or safety semantics must remain distinct. JSON and human output must use the same deduplicated order. Human output must preserve platform-aware shell quoting, visibly escape control characters to keep each repair action on one diagnostic line, and leave structured JSON `args` unchanged. - `maintenance_guidance.fts_optimization` is the shared, read-only recommendation contract for status, explain, optimize preview, and optimize execution. Keep `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` synchronized; stale or unavailable snapshots must not recommend mutation. - A successful CLI or MCP `index --rebuild` evaluates the shared freelist warning threshold after the index transaction commits. Incremental-auto-vacuum databases may run bounded `PRAGMA incremental_vacuum`; legacy databases must skip automatic reclaim instead of running a full `VACUUM`. Preserve the immediate index-result and persisted `last_index_run.rebuild_reclaim` telemetry, stable states/reasons, before/after ratios and byte/page counts, and the rule that reclaim failure never reclassifies an already committed index run as failed. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index fdc0e07b80..5fbeeb6f48 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2330,13 +2330,29 @@ catalog and unknown-field errors are generated from that same schema. | Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`. | | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | -| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | +| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `status_metadata_diagnostics`. | | Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. | | Database size attribution | `database_size_attribution` (`available`, `measurement`, `unavailable_reason`, physical main/WAL/SHM sizes, logical/object/freelist/residual reconciliation, page-type and payload/overhead subtotals, and bounded `top_objects`). | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `mcp_session.audit_log`, `mcp.rate_limit.bucket_limit`, and `mcp.rate_limit.bucket_limit_rejection_count`. `mcp_session` is session-scoped diagnostics rather than persisted DB state. It contains `log_level`, bounded `roots`, optional `client_info`, bounded optional `client_capabilities`, an always-present `metrics` object, and `audit_log` when audit emission is enabled. When advertised roots are capped, `roots_truncated`, `root_count`, `root_limit`, and `root_uri_length_limit` describe the truncation. When client capabilities are capped, `client_capabilities_truncated`, `client_capabilities_truncation_reason`, `client_capabilities_serialized_bytes`, `client_capabilities_byte_limit`, and `client_capabilities_depth_limit` describe the retained diagnostic subset. `mcp_session.metrics` is `{"enabled":false}` when unconfigured. An enabled metrics sink contains `enabled`, `path`, `max_bytes`, `bytes_written`, `disposed`, `degraded`, `queue_capacity`, `queue_depth`, `queued_event_count`, `written_event_count`, `dropped_event_count`, `queue_full_drop_count`, `serialization_failure_count`, `write_failure_count`, `rotation_failure_count`, `batch_flush_count`, `consecutive_failure_count`, and `recovery_count`, plus optional `next_retry_at`, `last_recovery_at`, and `last_failure`. MCP ping always mirrors the metrics object as `metrics`; metrics degradation is intentionally excluded from its top-level liveness result. The audit status fields and their health semantics are defined in [MCP audit log emission](#mcp-audit-log-emission). `mcp.rate_limit.bucket_limit` is the configured process-local cap across normalized `(partition, caller)` buckets: every direct call uses one fixed caller-wide coarse partition, canonical known tools additionally use secondary per-tool partitions, and unknown `batch_query` slots share one fixed invalid-slot partition per caller. `mcp.rate_limit.bucket_limit_rejection_count` counts calls denied because creating a new bucket would exceed that cap. | | Documentation sync | Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. | +The persisted status subdocuments `last_index_run.reference_extraction_cap_hits`, +`last_index_run.rebuild_reclaim`, and +`last_failed_or_partial_index_run.file_errors` are selected through a bounded +SQLite accessor that returns no managed string when the stored UTF-8 value is +larger than 512 KiB. `BoundedJson` then parses them at maximum depth 16 before +the reader revalidates the persisted semantic contract: at most 50 file errors, +50 cap-hit files, and 16 nested reasons; at most 32,768 characters per path, +128 per category/phase/reason code, 4,096 per detail/rebuild reason, and 262,144 +decoded string characters per subdocument. A rejected subdocument is omitted +without failing the rest of status. `status_metadata_diagnostics[]` reports its +field, `max_utf8_bytes`, optional `observed_utf8_bytes`, and one stable reason: +`raw_size_exceeded`, `invalid_json`, or `semantic_validation_failed`. Human +first-failure and recovery-hint fields pass through `ConsoleUi.FormatBoundedValue` +so persisted controls cannot create extra lines and long values use the stable +truncation marker; structured JSON retains accepted values unchanged. + `status --explain` resolves top-level keys through the source-generated `StatusResult` `JsonTypeInfo` used by `status --json`; ignored properties are excluded, and a coverage test requires every serialized top-level property to @@ -6331,13 +6347,29 @@ total / cursor / truncation、partial-family metadata、definition body の pagi | readiness / graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`。 | | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | -| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | +| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `status_metadata_diagnostics`。 | | database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`。 | | database size attribution | `database_size_attribution`（`available`、`measurement`、`unavailable_reason`、物理 main/WAL/SHM size、論理/object/freelist/residual の再照合、page type と payload/overhead の小計、上限付き `top_objects`）。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`、`mcp_session.metrics`、`mcp_session.audit_log`、`mcp.rate_limit.bucket_limit`、`mcp.rate_limit.bucket_limit_rejection_count`。`mcp_session` は persisted DB state ではなく session-scoped diagnostics で、`log_level`、上限付きの `roots`、任意の `client_info`、上限付きの任意の `client_capabilities`、常設の `metrics` object、audit 出力が有効な場合の `audit_log` を含みます。advertised root が切り詰められた場合は `roots_truncated`、`root_count`、`root_limit`、`root_uri_length_limit` が切り詰め内容を示します。client capabilities が切り詰められた場合は `client_capabilities_truncated`、`client_capabilities_truncation_reason`、`client_capabilities_serialized_bytes`、`client_capabilities_byte_limit`、`client_capabilities_depth_limit` が保持された診断 subset を示します。未設定時の `mcp_session.metrics` は `{"enabled":false}` です。有効な metrics sink は `enabled`、`path`、`max_bytes`、`bytes_written`、`disposed`、`degraded`、`queue_capacity`、`queue_depth`、`queued_event_count`、`written_event_count`、`dropped_event_count`、`queue_full_drop_count`、`serialization_failure_count`、`write_failure_count`、`rotation_failure_count`、`batch_flush_count`、`consecutive_failure_count`、`recovery_count` に加え、任意の `next_retry_at`、`last_recovery_at`、`last_failure` を追加します。MCP ping は常に metrics object を `metrics` として返し、metrics の degradation は意図的に top-level liveness result へ反映しません。audit status field と health semantics は [MCP 監査ログの出力](#mcp-監査ログの出力) に定義します。`mcp.rate_limit.bucket_limit` は normalized な `(partition, caller)` bucket 全体に対する process-local 上限で、direct call はすべて caller-wide の固定 coarse partition、canonical な既知 tool は追加の secondary per-tool partition、unknown な `batch_query` slot は caller ごとの 1 つの固定 invalid-slot partition を使います。`mcp.rate_limit.bucket_limit_rejection_count` は新規 bucket 作成がその上限を超えるため拒否された呼び出し数です。 | | documentation sync | この一覧は `README.md` と `AGENT_GUIDE.md` と同期してください。必須 field がそれらの docs から欠けると `DocumentationStatusContractTests` が失敗します。 | +永続化された status subdocument の +`last_index_run.reference_extraction_cap_hits`、`last_index_run.rebuild_reclaim`、 +`last_failed_or_partial_index_run.file_errors` は bounded SQLite accessor で取得し、 +保存済み UTF-8 value が 512 KiB を超える場合は managed string を返しません。 +続いて `BoundedJson` が最大 depth 16 で parse し、reader が永続化 semantic 契約を +再検証します。file error は最大 50 件、cap-hit file は最大 50 件、入れ子の reason は +最大 16 件、path は 1 件 32,768 文字、category / phase / reason code は 128 文字、 +detail / rebuild reason は 4,096 文字、subdocument 内の decoded string 合計は +262,144 文字です。拒否した subdocument は status の残りを失敗させずに省略します。 +`status_metadata_diagnostics[]` は対象 field、`max_utf8_bytes`、任意の +`observed_utf8_bytes` と、安定した reason `raw_size_exceeded`、`invalid_json`、 +`semantic_validation_failed` のいずれかを返します。human の first failure と +recovery hint は `ConsoleUi.FormatBoundedValue` を通すため、永続化された control で +追加行を作れず、長い値には安定した truncation marker を使います。structured JSON は +受理した値を変更せずに維持します。 + `status --explain` の top-level key は `status --json` と同じ source-generated `StatusResult` `JsonTypeInfo` で解決します。ignored property は除外し、coverage test で serialized top-level property がすべて説明を返すことを固定します。主要な readiness、 diff --git a/README.md b/README.md index 4a5b5666a6..46d2618c22 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ visible here as a compact compatibility index. | Version compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | | Extension and extractor diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`. | | Runtime trust and permissions | `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`. | -| Check context and run diagnostics | `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_failed_or_partial_index_run`. | +| Check context and run diagnostics | `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_failed_or_partial_index_run`, `status_metadata_diagnostics`. | | Last-run detail | `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_index_run.rebuild_reclaim`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | | SQLite and maintenance | `sqlite_connection_policy`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings`, `prepared_command_cache`, `maintenance_guidance`, `maintenance_guidance.fts_optimization`, `threshold_writes`, `observed_writes`. | | WAL checkpoint diagnostics | `read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`. | @@ -179,6 +179,20 @@ visible here as a compact compatibility index. | Remediation | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `queue_capacity`, `queue_depth`, `queued_event_count`, `written_event_count`, `dropped_event_count`, `queue_full_drop_count`, `serialization_failure_count`, `write_failure_count`, `rotation_failure_count`, `batch_flush_count`, `consecutive_failure_count`, `recovery_count`, `next_retry_at`, `last_recovery_at`, `last_failure`, `mcp_session.audit_log`, `queued_record_count`, `written_record_count`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`. | +Persisted JSON subdocuments for `last_index_run.reference_extraction_cap_hits`, +`last_index_run.rebuild_reclaim`, and +`last_failed_or_partial_index_run.file_errors` have a 512 KiB UTF-8 input limit +and a maximum nesting depth of 16. File-error and cap-hit file lists accept at +most 50 entries; nested reason lists accept at most 16. Paths accept 32,768 +characters, category/phase/reason codes accept 128, detail/rebuild-reason text +accepts 4,096, and each subdocument accepts at most 262,144 decoded string +characters in aggregate. An unavailable subdocument does not fail the rest of +`status`; `status_metadata_diagnostics` identifies its field with the stable +reason `raw_size_exceeded`, `invalid_json`, or `semantic_validation_failed`, the +configured `max_utf8_bytes`, and the observed byte size when available. Human +status output flattens persisted control characters and uses the stable bounded +value truncation marker, while accepted JSON values remain unchanged. + Full index completion, the watch initial scan, and full-scan dry runs also report the unknown-language file count, the top 10 extension groups, explicit omission metadata, and remediation guidance. `status --compact` includes the persisted @@ -388,7 +402,7 @@ field group を表に残します。 | version compatibility | `index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`。 | | extension / extractor diagnostics | `unknown_extension_file_count`、`unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit`、`unknown_extension_extension_counts`、`unknown_extension_category_counts`、`unknown_extension_groups`、`unknown_extension_group_count`、`unknown_extension_groups_truncated`、`unknown_extension_group_limit`、`unknown_extension_group_omitted_count`、`unknown_extension_guidance`、`extractors`、`hooks`、`hook_diagnostics`。 | | runtime trust / permissions | `trust_overrides`、`git_executable`、`path_case_sensitive`、`data_dir_mode`、`db_file_mode`、`database_permission_policy`、`database_permission_diagnostics`、`mac_profile`、`mac_profile_diagnostics`。 | -| check context / run diagnostics | `stale_after_seconds`、`index_age_seconds`、`query_context.check_mode`、`query_context.stale_after_seconds`、`process`、`last_index_run`、`last_workspace_freshened_at`、`last_failed_or_partial_index_run`。 | +| check context / run diagnostics | `stale_after_seconds`、`index_age_seconds`、`query_context.check_mode`、`query_context.stale_after_seconds`、`process`、`last_index_run`、`last_workspace_freshened_at`、`last_failed_or_partial_index_run`、`status_metadata_diagnostics`。 | | last-run detail | `last_index_run.bytes_read_skipped_file_count`、`last_index_run.bytes_read_incomplete`、`last_index_run.diagnostics`、`last_index_run.diagnostic_count`、`last_index_run.diagnostics_truncated`、`last_index_run.reference_extraction_cap_hits`、`last_index_run.rebuild_reclaim`、`last_failed_or_partial_index_run.progress_persisted`、`last_failed_or_partial_index_run.recovery_hint`、`last_failed_or_partial_index_run.file_errors`。 | | SQLite / maintenance | `sqlite_connection_policy`、`db_size_bytes`、`wal_size_bytes`、`db_pragma_settings`、`prepared_command_cache`、`maintenance_guidance`、`maintenance_guidance.fts_optimization`、`threshold_writes`、`observed_writes`。 | | WAL checkpoint diagnostics | `read_only_fallback`、`wal_checkpoint_attempted`、`wal_checkpoint_succeeded`、`wal_checkpoint_skipped_reason`、`wal_checkpoint_failure_reason`、`wal_checkpoint_busy`、`wal_checkpoint_log_page_count`、`wal_checkpoint_checkpointed_page_count`、`wal_checkpoint_remaining_page_count`、`read_only_immutable_fallback`、`wal_stale_snapshot_risk`、`wal_stale_snapshot_reason`。 | @@ -396,6 +410,19 @@ field group を表に残します。 | remediation | `degraded_root_cause`、`degraded_reason`、`recommended_action`、`alternative_action`、`readiness_degradations`、`repair_commands`。 | | MCP-only session diagnostics | `mcp_session`、`mcp_session.metrics`、`queue_capacity`、`queue_depth`、`queued_event_count`、`written_event_count`、`dropped_event_count`、`queue_full_drop_count`、`serialization_failure_count`、`write_failure_count`、`rotation_failure_count`、`batch_flush_count`、`consecutive_failure_count`、`recovery_count`、`next_retry_at`、`last_recovery_at`、`last_failure`、`mcp_session.audit_log`、`queued_record_count`、`written_record_count`、`mcp.rate_limit.bucket_limit`、`mcp.rate_limit.bucket_limit_rejection_count`。 | +`last_index_run.reference_extraction_cap_hits`、`last_index_run.rebuild_reclaim`、 +`last_failed_or_partial_index_run.file_errors` の永続化 JSON subdocument には、 +UTF-8 で 512 KiB の入力上限と最大 depth 16 を適用します。file error と cap-hit +file の一覧は最大 50 件、入れ子の reason 一覧は最大 16 件です。path は 32,768 +文字、category / phase / reason code は 128 文字、detail / rebuild reason は +4,096 文字、subdocument 内の decoded string 合計は 262,144 文字まで受け付けます。 +利用不能な subdocument があっても `status` の残りは失敗せず、 +`status_metadata_diagnostics` が対象 field、安定した reason +（`raw_size_exceeded`、`invalid_json`、`semantic_validation_failed`）、設定済みの +`max_utf8_bytes`、取得できる場合は実測 byte size を返します。human status 出力は +永続化された control character を平坦化し、安定した bounded-value truncation marker +を使いますが、契約内で受理した JSON 値は変更しません。 + 全体 index の完了時、watch の初回 scan、全体 scan の dry-run でも、言語未対応 ファイル数、上位 10 個の拡張子 group、明示的な省略 metadata、対処 guidance を 返します。`status --compact` には、直近に成功した全体 scan で永続化された同等の diff --git a/changelog.d/unreleased/5161.fixed.md b/changelog.d/unreleased/5161.fixed.md new file mode 100644 index 0000000000..df6cab2c58 --- /dev/null +++ b/changelog.d/unreleased/5161.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 5161 +affected: + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Cli/QueryCommandRunner.Status.cs + - README.md + - DEVELOPER_GUIDE.md + - AGENT_GUIDE.md +--- + +## English + +- **Persisted status diagnostics are now bounded at read and display boundaries (#5161)** — `status` rejects oversized, over-depth, or semantically invalid structured metadata without losing the rest of the response, reports a stable bounded diagnostic, and flattens and truncates persisted first-failure and recovery-hint fields in human output while preserving accepted JSON values. + +## 日本語 + +- **永続化された status 診断を読み取り境界と表示境界で制限するようになりました (#5161)** — `status` は oversized、over-depth、または意味的に不正な構造化 metadata を、response の残りを失わずに拒否して安定した上限付き診断を返します。また、受理した JSON 値は維持しながら、human 出力の first failure と recovery hint の永続化 field を平坦化・切り詰めします。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 5973dc8542..a54fa58678 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -19,7 +19,7 @@ public static partial class IndexCommandRunner { internal static Action? FullScanInputSnapshotBarrierForTesting { get; set; } - private const int PartialIndexFileErrorLimit = 50; + private const int PartialIndexFileErrorLimit = StatusMetadataLimits.MaxFileErrors; internal static bool ShouldUseFreshReferenceResolutionDefaults( bool startedWithNoIndexedFiles, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index 36ba49d33f..322323bb20 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -333,10 +333,12 @@ public static int RunStatus( var firstFailure = status.LastFailedOrPartialIndexRun?.FileErrors?.FirstOrDefault(); var failureSuffix = firstFailure == null ? string.Empty - : $" First failure: {firstFailure.File} ({firstFailure.Category}, {firstFailure.Phase}): {firstFailure.Detail}"; + : $" First failure: {ConsoleUi.FormatBoundedValue(firstFailure.File)} ({ConsoleUi.FormatBoundedValue(firstFailure.Category)}, {ConsoleUi.FormatBoundedValue(firstFailure.Phase)}): {ConsoleUi.FormatBoundedValue(firstFailure.Detail)}"; Console.WriteLine(ConsoleUi.FormatSummaryLine("WARN", $"index generation is incomplete; successful files and graph edges remain queryable.{failureSuffix}")); - Console.WriteLine(ConsoleUi.FormatSummaryLine("Hint", status.LastFailedOrPartialIndexRun?.RecoveryHint - ?? "fix the reported file/extractor failure, then rerun the same index command; a rebuild is not required.")); + var persistedRecoveryHint = status.LastFailedOrPartialIndexRun?.RecoveryHint; + Console.WriteLine(ConsoleUi.FormatSummaryLine("Hint", persistedRecoveryHint == null + ? "fix the reported file/extractor failure, then rerun the same index command; a rebuild is not required." + : ConsoleUi.FormatBoundedValue(persistedRecoveryHint))); } if (!status.ReferenceGraphComplete) { diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 978e5cb963..cab60021bc 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -1,10 +1,12 @@ using System.Buffers; +using CodeIndex.Diagnostics; using CodeIndex.Indexer; using CodeIndex.Models; using Microsoft.Data.Sqlite; using System.Globalization; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; using System.Text.RegularExpressions; using System.Threading; @@ -36,6 +38,12 @@ public partial class DbReader internal const int MaxLegacyResourceReadSqliteVmSteps = 250_000; private const string EmptyIndexedContentChecksum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + internal const string StatusMetadataRawSizeExceededReason = "raw_size_exceeded"; + internal const string StatusMetadataInvalidJsonReason = "invalid_json"; + internal const string StatusMetadataSemanticValidationFailedReason = "semantic_validation_failed"; + private const string LastIndexRunReferenceCapHitsField = "last_index_run.reference_extraction_cap_hits"; + private const string LastIndexRunRebuildReclaimField = "last_index_run.rebuild_reclaim"; + private const string LastFailedRunFileErrorsField = "last_failed_or_partial_index_run.file_errors"; private static readonly AsyncLocal FindRegexMatchTimeoutOverride = new(); private static readonly AsyncLocal FindLineScannedOverride = new(); @@ -688,7 +696,7 @@ private long ExecuteScalar(string sql) } } - private StatusLastIndexRun? GetLastIndexRun() + private StatusLastIndexRun? GetLastIndexRun(List metadataDiagnostics) { var mode = TryGetMetaStringInternal(DbContext.LastIndexRunModeMetaKey); var startedAt = ParseMetaDateTime(TryGetMetaStringInternal(DbContext.LastIndexRunStartedAtMetaKey)); @@ -705,10 +713,18 @@ private long ExecuteScalar(string sql) var diagnostics = ParseMetaStringList(TryGetMetaStringInternal(DbContext.LastIndexRunDiagnosticsMetaKey)); var diagnosticCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunDiagnosticCountMetaKey)); var diagnosticsTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.LastIndexRunDiagnosticsTruncatedMetaKey)); - var referenceExtractionCapHits = ParseReferenceExtractionCapHits( - TryGetMetaStringInternal(DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey)); - var rebuildReclaim = ParseRebuildReclaim( - TryGetMetaStringInternal(DbContext.LastIndexRunRebuildReclaimMetaKey)); + var referenceExtractionCapHits = ReadStructuredStatusMetadata( + DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, + LastIndexRunReferenceCapHitsField, + StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary, + ValidateReferenceExtractionCapHits, + metadataDiagnostics); + var rebuildReclaim = ReadStructuredStatusMetadata( + DbContext.LastIndexRunRebuildReclaimMetaKey, + LastIndexRunRebuildReclaimField, + StatusMetadataJsonContext.Default.StatusRebuildReclaim, + ValidateRebuildReclaim, + metadataDiagnostics); if (mode == null && startedAt == null && durationMs == null && filesScanned == null && filesSkipped == null && parseErrors == null && bytesRead == null && bytesReadSkippedFileCount == null && bytesReadIncomplete == null && rowsUpserted == null && rowsDeleted == null && peakMemoryMb == null @@ -740,35 +756,80 @@ private long ExecuteScalar(string sql) }; } - private static StatusRebuildReclaim? ParseRebuildReclaim(string? json) + private T? ReadStructuredStatusMetadata( + string key, + string field, + JsonTypeInfo jsonTypeInfo, + Func validate, + List metadataDiagnostics) + where T : class { - if (string.IsNullOrWhiteSpace(json)) + var read = TryGetBoundedStatusMetaStringInternal(key); + if (!read.Exists) return null; - try + + if (read.Utf8ByteCount > StatusMetadataLimits.MaxRawUtf8Bytes) { - return JsonSerializer.Deserialize(json, StatusMetadataJsonContext.Default.StatusRebuildReclaim); + AddStatusMetadataDiagnostic( + metadataDiagnostics, + field, + StatusMetadataRawSizeExceededReason, + read.Utf8ByteCount); + return null; } - catch (JsonException) + + if (!read.IsText || string.IsNullOrWhiteSpace(read.Value)) { + AddStatusMetadataDiagnostic( + metadataDiagnostics, + field, + StatusMetadataSemanticValidationFailedReason, + read.Utf8ByteCount); return null; } - } - private static ReferenceExtractionCapHitSummary? ParseReferenceExtractionCapHits(string? json) - { - if (string.IsNullOrWhiteSpace(json)) - return null; try { - return JsonSerializer.Deserialize(json, StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary); + var value = BoundedJson.Deserialize( + read.Value, + StatusMetadataLimits.MaxRawUtf8Bytes, + StatusMetadataLimits.MaxJsonDepth, + jsonTypeInfo); + if (value == null || !validate(value)) + { + AddStatusMetadataDiagnostic( + metadataDiagnostics, + field, + StatusMetadataSemanticValidationFailedReason, + read.Utf8ByteCount); + return null; + } + + return value; } catch (JsonException) { + AddStatusMetadataDiagnostic( + metadataDiagnostics, + field, + StatusMetadataInvalidJsonReason, + read.Utf8ByteCount); + return null; + } + catch (InvalidDataException) + { + AddStatusMetadataDiagnostic( + metadataDiagnostics, + field, + StatusMetadataRawSizeExceededReason, + read.Utf8ByteCount); return null; } } - private StatusFailedOrPartialIndexRun? GetLastFailedOrPartialIndexRun(bool batchInProgress) + private StatusFailedOrPartialIndexRun? GetLastFailedOrPartialIndexRun( + bool batchInProgress, + List metadataDiagnostics) { var status = TryGetMetaStringInternal(DbContext.LastFailedIndexRunStatusMetaKey); var mode = TryGetMetaStringInternal(DbContext.LastFailedIndexRunModeMetaKey); @@ -780,7 +841,12 @@ private long ExecuteScalar(string sql) var reason = TryGetMetaStringInternal(DbContext.LastFailedIndexRunReasonMetaKey); var progressPersisted = ParseMetaBool(TryGetMetaStringInternal(DbContext.LastFailedIndexRunProgressPersistedMetaKey)); var recoveryHint = TryGetMetaStringInternal(DbContext.LastFailedIndexRunRecoveryHintMetaKey); - var fileErrors = ParseStatusIndexFileErrors(TryGetMetaStringInternal(DbContext.LastFailedIndexRunFileErrorsMetaKey)); + var fileErrors = ReadStructuredStatusMetadata( + DbContext.LastFailedIndexRunFileErrorsMetaKey, + LastFailedRunFileErrorsField, + StatusMetadataJsonContext.Default.ListStatusIndexFileError, + ValidateStatusIndexFileErrors, + metadataDiagnostics); if (status == null && mode == null && startedAt == null && durationMs == null && filesProcessed == null && filesTotal == null && errorCode == null && reason == null && progressPersisted == null && recoveryHint == null && fileErrors == null) @@ -968,6 +1034,217 @@ private StringComparer GetIndexedPathComparer() } } + private readonly record struct BoundedStatusMetaRead( + bool Exists, + string? Value, + long? Utf8ByteCount, + bool IsText); + + private BoundedStatusMetaRead TryGetBoundedStatusMetaStringInternal(string key) + { + try + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = """ + SELECT + CASE + WHEN typeof(value) = 'text' + AND length(CAST(value AS BLOB)) <= @maxUtf8Bytes + THEN value + ELSE NULL + END, + length(CAST(value AS BLOB)), + typeof(value) + FROM codeindex_meta + WHERE key = @key + """; + SqliteCommandPolicy.Add(cmd, "@key", key); + SqliteCommandPolicy.Add(cmd, "@maxUtf8Bytes", StatusMetadataLimits.MaxRawUtf8Bytes); + using var reader = cmd.ExecuteTrackedReader(); + if (!reader.TrackedRead()) + return default; + + var valueType = reader.GetString(2); + if (string.Equals(valueType, "null", StringComparison.Ordinal)) + return default; + + var utf8ByteCount = reader.IsDBNull(1) ? (long?)null : reader.GetInt64(1); + var isText = string.Equals(valueType, "text", StringComparison.Ordinal); + var value = reader.IsDBNull(0) ? null : reader.GetString(0); + return new BoundedStatusMetaRead( + Exists: true, + value, + utf8ByteCount, + isText); + } + catch (SqliteException) + { + return default; + } + } + + private static void AddStatusMetadataDiagnostic( + List diagnostics, + string field, + string reason, + long? observedUtf8Bytes) + { + diagnostics.Add(new StatusMetadataDiagnostic + { + Field = field, + Reason = reason, + MaxUtf8Bytes = StatusMetadataLimits.MaxRawUtf8Bytes, + ObservedUtf8Bytes = observedUtf8Bytes, + }); + } + + private static bool ValidateStatusIndexFileErrors(List fileErrors) + { + if (fileErrors.Count > StatusMetadataLimits.MaxFileErrors) + return false; + + var decodedCharacters = 0; + foreach (var fileError in fileErrors) + { + if (fileError == null + || !TryAcceptStatusMetadataString( + fileError.File, + StatusMetadataLimits.MaxPathCharacters, + ref decodedCharacters) + || !TryAcceptStatusMetadataString( + fileError.Category, + StatusMetadataLimits.MaxCodeCharacters, + ref decodedCharacters) + || !TryAcceptStatusMetadataString( + fileError.Phase, + StatusMetadataLimits.MaxCodeCharacters, + ref decodedCharacters) + || !TryAcceptStatusMetadataString( + fileError.Detail, + StatusMetadataLimits.MaxDetailCharacters, + ref decodedCharacters) + || fileError.Line < 0 + || fileError.Column < 0) + { + return false; + } + } + + return true; + } + + private static bool ValidateReferenceExtractionCapHits(ReferenceExtractionCapHitSummary summary) + { + if (summary.HitCount < 0 + || summary.AffectedFileCount < 0 + || summary.FileLimit != StatusMetadataLimits.MaxReferenceCapHitFiles + || summary.Reasons == null + || summary.Files == null + || summary.Reasons.Count > StatusMetadataLimits.MaxReferenceReasons + || summary.Files.Count > StatusMetadataLimits.MaxReferenceCapHitFiles + || summary.Files.Count > summary.AffectedFileCount + || summary.FilesTruncated != (summary.AffectedFileCount > summary.Files.Count)) + { + return false; + } + + var decodedCharacters = 0; + foreach (var reason in summary.Reasons) + { + if (!TryAcceptStatusMetadataString( + reason, + StatusMetadataLimits.MaxCodeCharacters, + ref decodedCharacters)) + { + return false; + } + } + + long sampledHitCount = 0; + foreach (var file in summary.Files) + { + if (file == null + || file.HitCount < 0 + || file.Reasons == null + || file.Reasons.Count > StatusMetadataLimits.MaxReferenceReasons + || !TryAcceptStatusMetadataString( + file.File, + StatusMetadataLimits.MaxPathCharacters, + ref decodedCharacters)) + { + return false; + } + + if (sampledHitCount > long.MaxValue - file.HitCount) + return false; + sampledHitCount += file.HitCount; + foreach (var reason in file.Reasons) + { + if (!TryAcceptStatusMetadataString( + reason, + StatusMetadataLimits.MaxCodeCharacters, + ref decodedCharacters)) + { + return false; + } + } + } + + return summary.HitCount >= sampledHitCount; + } + + private static bool ValidateRebuildReclaim(StatusRebuildReclaim reclaim) + { + var decodedCharacters = 0; + return TryAcceptStatusMetadataString( + reclaim.State, + StatusMetadataLimits.MaxCodeCharacters, + ref decodedCharacters) + && TryAcceptStatusMetadataString( + reclaim.Reason, + StatusMetadataLimits.MaxDetailCharacters, + ref decodedCharacters) + && reclaim.DurationMs >= 0 + && IsNonNegative(reclaim.PageSizeBytes) + && IsNonNegative(reclaim.PageCountBefore) + && IsNonNegative(reclaim.FreelistCountBefore) + && IsRatio(reclaim.FreelistRatioBefore) + && IsRatio(reclaim.FreelistThresholdRatio) + && IsNonNegative(reclaim.EstimatedBytesReclaimableBefore) + && IsNonNegative(reclaim.PageCountAfter) + && IsNonNegative(reclaim.FreelistCountAfter) + && IsRatio(reclaim.FreelistRatioAfter) + && IsNonNegative(reclaim.PagesReclaimed) + && IsNonNegative(reclaim.BytesReclaimed) + && IsNonNegative(reclaim.LogicalDatabaseBytesBefore) + && IsNonNegative(reclaim.LogicalDatabaseBytesAfter) + && IsNonNegative(reclaim.DbSizeBytesBefore) + && IsNonNegative(reclaim.DbSizeBytesAfter) + && reclaim.AutoVacuumMode is null or >= 0 and <= 2; + } + + private static bool TryAcceptStatusMetadataString( + string? value, + int maxCharacters, + ref int decodedCharacters) + { + if (string.IsNullOrEmpty(value) + || value.Length > maxCharacters + || value.Length > StatusMetadataLimits.MaxDecodedStringCharacters - decodedCharacters) + { + return false; + } + + decodedCharacters += value.Length; + return true; + } + + private static bool IsNonNegative(long? value) + => value is null or >= 0; + + private static bool IsRatio(double? value) + => value is null || (double.IsFinite(value.Value) && value.Value >= 0 && value.Value <= 1); + // Parse an ISO-8601 timestamp persisted via SetMeta. Offsetless legacy values are // treated as UTC, while explicit offsets are honored before normalizing to UTC JSON. // SetMeta で保存された ISO-8601 timestamp を読む。offset の無い legacy 値は UTC 扱いし、 @@ -1008,21 +1285,6 @@ private StringComparer GetIndexedPathComparer() return JsonStringListCodec.Deserialize(raw); } - private static List? ParseStatusIndexFileErrors(string? raw) - { - if (string.IsNullOrWhiteSpace(raw)) - return null; - - try - { - return JsonSerializer.Deserialize(raw, StatusMetadataJsonContext.Default.ListStatusIndexFileError); - } - catch (JsonException) - { - return null; - } - } - private static long? ParseMetaLong(string? raw) => string.IsNullOrWhiteSpace(raw) || !long.TryParse( diff --git a/src/CodeIndex/Database/DbReader.ReferenceExtractionCompleteness.cs b/src/CodeIndex/Database/DbReader.ReferenceExtractionCompleteness.cs index 9e7abc9735..889dd0feee 100644 --- a/src/CodeIndex/Database/DbReader.ReferenceExtractionCompleteness.cs +++ b/src/CodeIndex/Database/DbReader.ReferenceExtractionCompleteness.cs @@ -6,7 +6,7 @@ namespace CodeIndex.Database; public partial class DbReader { - public const int ReferenceExtractionCapHitFileLimit = 50; + public const int ReferenceExtractionCapHitFileLimit = StatusMetadataLimits.MaxReferenceCapHitFiles; public const string ReferenceExtractionCapStateUnavailableReason = "reference_extraction_cap_state_unavailable"; private ReferenceExtractionCapHitSummary? _referenceExtractionCapHits; diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 7e7d4c2af9..1fcccc1160 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -220,7 +220,8 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind dbSizeBytes, dbPragmaSettings.AutoVacuum), ftsOptimization: ftsOptimization); - var lastIndexRun = GetLastIndexRun(); + var statusMetadataDiagnostics = new List(); + var lastIndexRun = GetLastIndexRun(statusMetadataDiagnostics); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); var persistedReadiness = GetPersistedIndexGenerationReadiness( referenceExtractionCapHits, @@ -228,7 +229,9 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind hdlGraphContractReady, txn); var batchInProgress = persistedReadiness.MigrationInProgress; - var lastFailedOrPartialIndexRun = GetLastFailedOrPartialIndexRun(batchInProgress); + var lastFailedOrPartialIndexRun = GetLastFailedOrPartialIndexRun( + batchInProgress, + statusMetadataDiagnostics); var result = new StatusResult { @@ -305,6 +308,9 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind Process = StatusProcessMetrics.Capture(), LastIndexRun = lastIndexRun, LastFailedOrPartialIndexRun = lastFailedOrPartialIndexRun, + StatusMetadataDiagnostics = statusMetadataDiagnostics.Count > 0 + ? statusMetadataDiagnostics + : null, ReadOnlyFallback = _readOnlyFallback, WalCheckpointAttempted = _walCheckpointAttempted, WalCheckpointSucceeded = _walCheckpointSucceeded, diff --git a/src/CodeIndex/Diagnostics/BoundedJson.cs b/src/CodeIndex/Diagnostics/BoundedJson.cs index 1a67f8d9db..51ea1ab2f3 100644 --- a/src/CodeIndex/Diagnostics/BoundedJson.cs +++ b/src/CodeIndex/Diagnostics/BoundedJson.cs @@ -1,6 +1,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; namespace CodeIndex.Diagnostics; @@ -73,6 +74,17 @@ internal static JsonDocument ParseDocument( return JsonSerializer.Deserialize(utf8Json, options); } + internal static T? Deserialize( + string json, + int maxUtf8Bytes, + int maxDepth, + JsonTypeInfo jsonTypeInfo) + { + ArgumentNullException.ThrowIfNull(jsonTypeInfo); + using var document = ParseDocument(json, maxUtf8Bytes, maxDepth); + return document.RootElement.Deserialize(jsonTypeInfo); + } + internal static string FormatExceptionDetail(JsonException ex) => JsonFrameParser.FormatExceptionDetail(ex); diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index f99ef0d65b..b7fa88b2bf 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -1081,7 +1081,7 @@ await EmitProgressNotificationAsync( if (errors > 0) { var statusFileErrors = failures - .Take(50) + .Take(StatusMetadataLimits.MaxFileErrors) .Select(failure => new StatusIndexFileError { File = failure.Path, diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index a66fa82ca8..20845a485a 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1973,6 +1973,9 @@ public class StatusResult [JsonPropertyName("last_failed_or_partial_index_run")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public StatusFailedOrPartialIndexRun? LastFailedOrPartialIndexRun { get; set; } + [JsonPropertyName("status_metadata_diagnostics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? StatusMetadataDiagnostics { get; set; } } public sealed class StatusHeadFreshness @@ -2302,6 +2305,17 @@ public sealed class StatusLastIndexRun public StatusRebuildReclaim? RebuildReclaim { get; set; } } +public sealed class StatusMetadataDiagnostic +{ + public string Field { get; init; } = string.Empty; + public string Reason { get; init; } = string.Empty; + [JsonPropertyName("max_utf8_bytes")] + public int MaxUtf8Bytes { get; init; } + [JsonPropertyName("observed_utf8_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? ObservedUtf8Bytes { get; init; } +} + /// /// Bounded telemetry for the thresholded free-page reclaim that follows a successful rebuild. /// 成功した rebuild 後にしきい値付きで行う free-page 回収の bounded telemetry。 diff --git a/src/CodeIndex/Models/StatusMetadataLimits.cs b/src/CodeIndex/Models/StatusMetadataLimits.cs new file mode 100644 index 0000000000..f4a7336724 --- /dev/null +++ b/src/CodeIndex/Models/StatusMetadataLimits.cs @@ -0,0 +1,14 @@ +namespace CodeIndex.Models; + +internal static class StatusMetadataLimits +{ + internal const int MaxRawUtf8Bytes = 512 * 1024; + internal const int MaxJsonDepth = 16; + internal const int MaxFileErrors = 50; + internal const int MaxReferenceCapHitFiles = 50; + internal const int MaxReferenceReasons = 16; + internal const int MaxPathCharacters = 32 * 1024; + internal const int MaxCodeCharacters = 128; + internal const int MaxDetailCharacters = 4 * 1024; + internal const int MaxDecodedStringCharacters = 256 * 1024; +} diff --git a/tests/CodeIndex.Tests/DbReaderStatusMetadataIssue5161Tests.cs b/tests/CodeIndex.Tests/DbReaderStatusMetadataIssue5161Tests.cs new file mode 100644 index 0000000000..f613270ef6 --- /dev/null +++ b/tests/CodeIndex.Tests/DbReaderStatusMetadataIssue5161Tests.cs @@ -0,0 +1,261 @@ +using System.Text; +using System.Text.Json; +using CodeIndex.Database; +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +public partial class DbReaderTests +{ + [Fact] + public void GetStatus_TreatsClearedStructuredMetadataAsAbsent_Issue5161() + { + _writer.SetMeta(DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, null); + _writer.SetMeta(DbContext.LastIndexRunRebuildReclaimMetaKey, null); + _writer.SetMeta(DbContext.LastFailedIndexRunFileErrorsMetaKey, null); + + var status = _reader.GetStatus(); + + Assert.Null(status.LastIndexRun); + Assert.Null(status.LastFailedOrPartialIndexRun); + Assert.Null(status.StatusMetadataDiagnostics); + } + + [Fact] + public void GetStatus_BoundsStructuredMetadataBeforeMaterialization_Issue5161() + { + _writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "rebuild"); + var validJson = JsonSerializer.Serialize( + new StatusRebuildReclaim + { + State = "not_needed", + Reason = "freelist_below_threshold", + DurationMs = 0, + }, + StatusMetadataJsonContext.Default.StatusRebuildReclaim); + var exactLimitJson = validJson + new string( + ' ', + StatusMetadataLimits.MaxRawUtf8Bytes - Encoding.UTF8.GetByteCount(validJson)); + Assert.Equal(StatusMetadataLimits.MaxRawUtf8Bytes, Encoding.UTF8.GetByteCount(exactLimitJson)); + + _writer.SetMeta(DbContext.LastIndexRunRebuildReclaimMetaKey, exactLimitJson); + var exactLimitStatus = _reader.GetStatus(); + Assert.Equal("not_needed", exactLimitStatus.LastIndexRun?.RebuildReclaim?.State); + Assert.Null(exactLimitStatus.StatusMetadataDiagnostics); + + _writer.SetMeta(DbContext.LastIndexRunRebuildReclaimMetaKey, exactLimitJson + " "); + AssertMetadataDiagnostic( + _reader.GetStatus(), + "last_index_run.rebuild_reclaim", + DbReader.StatusMetadataRawSizeExceededReason, + StatusMetadataLimits.MaxRawUtf8Bytes + 1L); + + var overDepthJson = "{\"state\":\"not_needed\",\"reason\":\"ok\",\"duration_ms\":0,\"extra\":" + + new string('[', StatusMetadataLimits.MaxJsonDepth + 1) + + "0" + + new string(']', StatusMetadataLimits.MaxJsonDepth + 1) + + "}"; + _writer.SetMeta(DbContext.LastIndexRunRebuildReclaimMetaKey, overDepthJson); + AssertMetadataDiagnostic( + _reader.GetStatus(), + "last_index_run.rebuild_reclaim", + DbReader.StatusMetadataInvalidJsonReason, + Encoding.UTF8.GetByteCount(overDepthJson)); + + _writer.SetMeta(DbContext.LastIndexRunRebuildReclaimMetaKey, "{"); + AssertMetadataDiagnostic( + _reader.GetStatus(), + "last_index_run.rebuild_reclaim", + DbReader.StatusMetadataInvalidJsonReason, + observedUtf8Bytes: 1); + } + + [Fact] + public void GetStatus_RejectsFileErrorItemAndNestedStringLimitViolations_Issue5161() + { + _writer.SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, "failed"); + var tooManyErrors = Enumerable.Range(0, StatusMetadataLimits.MaxFileErrors + 1) + .Select(i => new StatusIndexFileError + { + File = $"src/File{i}.cs", + Category = "file_read_error", + Phase = "reading", + Detail = "access denied", + }) + .ToList(); + SetFileErrors(tooManyErrors); + + var countLimitedStatus = _reader.GetStatus(); + Assert.Null(countLimitedStatus.LastFailedOrPartialIndexRun?.FileErrors); + AssertMetadataDiagnostic( + countLimitedStatus, + "last_failed_or_partial_index_run.file_errors", + DbReader.StatusMetadataSemanticValidationFailedReason); + + SetFileErrors( + [ + new StatusIndexFileError + { + File = new string('p', StatusMetadataLimits.MaxPathCharacters + 1), + Category = "file_read_error", + Phase = "reading", + Detail = "access denied", + }, + ]); + + var stringLimitedStatus = _reader.GetStatus(); + Assert.Null(stringLimitedStatus.LastFailedOrPartialIndexRun?.FileErrors); + AssertMetadataDiagnostic( + stringLimitedStatus, + "last_failed_or_partial_index_run.file_errors", + DbReader.StatusMetadataSemanticValidationFailedReason); + } + + [Fact] + public void GetStatus_RejectsReferenceCapAndReclaimSemanticViolations_Issue5161() + { + _writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "rebuild"); + var tooManyFiles = Enumerable.Range(0, StatusMetadataLimits.MaxReferenceCapHitFiles + 1) + .Select(i => new ReferenceExtractionFileCapHits + { + File = $"src/File{i}.cs", + HitCount = 1, + Reasons = ["lookup_symbol_limit"], + }) + .ToList(); + var invalidCapHits = new ReferenceExtractionCapHitSummary + { + HitCount = tooManyFiles.Count, + AffectedFileCount = tooManyFiles.Count, + Reasons = ["lookup_symbol_limit"], + Files = tooManyFiles, + FilesTruncated = false, + FileLimit = StatusMetadataLimits.MaxReferenceCapHitFiles, + }; + _writer.SetMeta( + DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, + JsonSerializer.Serialize( + invalidCapHits, + StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary)); + + var capLimitedStatus = _reader.GetStatus(); + Assert.Null(capLimitedStatus.LastIndexRun?.ReferenceExtractionCapHits); + AssertMetadataDiagnostic( + capLimitedStatus, + "last_index_run.reference_extraction_cap_hits", + DbReader.StatusMetadataSemanticValidationFailedReason); + + var validCapHits = new ReferenceExtractionCapHitSummary + { + FileLimit = StatusMetadataLimits.MaxReferenceCapHitFiles, + }; + _writer.SetMeta( + DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, + JsonSerializer.Serialize( + validCapHits, + StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary)); + _writer.SetMeta( + DbContext.LastIndexRunRebuildReclaimMetaKey, + JsonSerializer.Serialize( + new StatusRebuildReclaim + { + State = "completed", + Reason = "threshold_reached", + DurationMs = -1, + }, + StatusMetadataJsonContext.Default.StatusRebuildReclaim)); + + var reclaimLimitedStatus = _reader.GetStatus(); + Assert.Null(reclaimLimitedStatus.LastIndexRun?.RebuildReclaim); + AssertMetadataDiagnostic( + reclaimLimitedStatus, + "last_index_run.rebuild_reclaim", + DbReader.StatusMetadataSemanticValidationFailedReason); + } + + [Fact] + public void GetStatus_RoundTripsValidStructuredMetadata_Issue5161() + { + var capHits = new ReferenceExtractionCapHitSummary + { + HitCount = 2, + AffectedFileCount = 1, + Reasons = ["lookup_symbol_limit"], + Files = + [ + new ReferenceExtractionFileCapHits + { + File = "src/App.cs", + HitCount = 2, + Reasons = ["lookup_symbol_limit"], + }, + ], + FilesTruncated = false, + FileLimit = StatusMetadataLimits.MaxReferenceCapHitFiles, + }; + var reclaim = new StatusRebuildReclaim + { + State = "completed", + Reason = "threshold_reached", + DurationMs = 12, + PageSizeBytes = 4096, + PagesReclaimed = 7, + FreelistRatioBefore = 0.25, + FreelistRatioAfter = 0.01, + AutoVacuumMode = 2, + }; + var fileErrors = new List + { + new() + { + File = "src/Broken.cs", + Category = "file_read_error", + Phase = "reading", + Detail = "access denied", + Line = 3, + Column = 4, + }, + }; + _writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "rebuild"); + _writer.SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, "failed"); + _writer.SetMeta( + DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, + JsonSerializer.Serialize( + capHits, + StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary)); + _writer.SetMeta( + DbContext.LastIndexRunRebuildReclaimMetaKey, + JsonSerializer.Serialize(reclaim, StatusMetadataJsonContext.Default.StatusRebuildReclaim)); + SetFileErrors(fileErrors); + + var status = _reader.GetStatus(); + + Assert.Equal(2, status.LastIndexRun?.ReferenceExtractionCapHits?.HitCount); + Assert.Equal("src/App.cs", status.LastIndexRun?.ReferenceExtractionCapHits?.Files[0].File); + Assert.Equal(7, status.LastIndexRun?.RebuildReclaim?.PagesReclaimed); + Assert.Equal("src/Broken.cs", status.LastFailedOrPartialIndexRun?.FileErrors?[0].File); + Assert.Equal(3, status.LastFailedOrPartialIndexRun?.FileErrors?[0].Line); + Assert.Null(status.StatusMetadataDiagnostics); + } + + private void SetFileErrors(List fileErrors) + => _writer.SetMeta( + DbContext.LastFailedIndexRunFileErrorsMetaKey, + JsonSerializer.Serialize( + fileErrors, + StatusMetadataJsonContext.Default.ListStatusIndexFileError)); + + private static void AssertMetadataDiagnostic( + StatusResult status, + string field, + string reason, + long? observedUtf8Bytes = null) + { + var diagnostic = Assert.Single(status.StatusMetadataDiagnostics!); + Assert.Equal(field, diagnostic.Field); + Assert.Equal(reason, diagnostic.Reason); + Assert.Equal(StatusMetadataLimits.MaxRawUtf8Bytes, diagnostic.MaxUtf8Bytes); + if (observedUtf8Bytes != null) + Assert.Equal(observedUtf8Bytes, diagnostic.ObservedUtf8Bytes); + } +} diff --git a/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs new file mode 100644 index 0000000000..c3a2929b6c --- /dev/null +++ b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs @@ -0,0 +1,201 @@ +using System.Text.Json; +using CodeIndex.Cli; +using CodeIndex.Database; +using CodeIndex.Models; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Tests; + +[Collection("Console sensitive")] +public sealed class StatusMetadataOutputIssue5161Tests +{ + private const string TruncationMarker = "... QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only", "--json"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, result.ExitCode); + Assert.Equal(string.Empty, result.Stderr); + using var document = JsonDocument.Parse(result.Stdout); + var root = document.RootElement; + var lastRun = root.GetProperty("last_index_run"); + Assert.Equal("rebuild", lastRun.GetProperty("mode").GetString()); + Assert.False(lastRun.TryGetProperty("rebuild_reclaim", out _)); + var diagnostic = Assert.Single( + root.GetProperty("status_metadata_diagnostics").EnumerateArray()); + Assert.Equal("last_index_run.rebuild_reclaim", diagnostic.GetProperty("field").GetString()); + Assert.Equal("raw_size_exceeded", diagnostic.GetProperty("reason").GetString()); + Assert.Equal( + StatusMetadataLimits.MaxRawUtf8Bytes, + diagnostic.GetProperty("max_utf8_bytes").GetInt32()); + Assert.Equal( + StatusMetadataLimits.MaxRawUtf8Bytes + 1L, + diagnostic.GetProperty("observed_utf8_bytes").GetInt64()); + } + finally + { + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunStatus_PreservesJsonAndSanitizesHumanFailureDiagnostics_Issue5161() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_status_metadata_5161"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var file = CreateControlledValue("file", 150); + var category = CreateControlledValue("category", 125); + var phase = CreateControlledValue("phase", 125); + var detail = CreateControlledValue("detail", 150); + var recoveryHint = CreateControlledValue("recovery", 150); + SetFailedStatus(dbPath, file, category, phase, detail, recoveryHint); + PrepareImmutableRead(dbPath); + + var json = ConsoleCapture.Capture(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only", "--json"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, json.ExitCode); + Assert.Equal(string.Empty, json.Stderr); + using (var document = JsonDocument.Parse(json.Stdout)) + { + var root = document.RootElement; + var failed = root.GetProperty("last_failed_or_partial_index_run"); + var firstFailure = failed.GetProperty("file_errors")[0]; + Assert.Equal(file, firstFailure.GetProperty("file").GetString()); + Assert.Equal(category, firstFailure.GetProperty("category").GetString()); + Assert.Equal(phase, firstFailure.GetProperty("phase").GetString()); + Assert.Equal(detail, firstFailure.GetProperty("detail").GetString()); + Assert.Equal(recoveryHint, failed.GetProperty("recovery_hint").GetString()); + Assert.False(root.TryGetProperty("status_metadata_diagnostics", out _)); + Assert.True(root.GetProperty("sqlite_connection_policy").GetProperty("immutable_uri").GetBoolean()); + } + + var human = ConsoleCapture.Capture(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, human.ExitCode); + var humanOutput = human.Stdout + human.Stderr; + var firstFailureLine = Assert.Single( + humanOutput.Split('\n').Where(line => line.Contains("First failure:", StringComparison.Ordinal))); + Assert.Equal(4, CountOccurrences(firstFailureLine, TruncationMarker)); + Assert.DoesNotContain('\r', firstFailureLine); + Assert.DoesNotContain('\t', firstFailureLine); + Assert.DoesNotContain('\u001b', firstFailureLine); + Assert.DoesNotContain('\u0001', firstFailureLine); + var hintLine = Assert.Single( + humanOutput.Split('\n').Where(line => + line.TrimStart().StartsWith("Hint", StringComparison.Ordinal) + && line.Contains(TruncationMarker, StringComparison.Ordinal))); + Assert.Contains(TruncationMarker, hintLine, StringComparison.Ordinal); + Assert.DoesNotContain('\r', hintLine); + Assert.DoesNotContain('\t', hintLine); + Assert.DoesNotContain('\u001b', hintLine); + Assert.DoesNotContain('\u0001', hintLine); + + SetFailedStatus( + dbPath, + "src/App.cs", + "file_read_error", + "reading", + "access denied", + "Retry indexing."); + PrepareImmutableRead(dbPath); + var shortHuman = ConsoleCapture.Capture(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only"], + _jsonOptions)); + var shortOutput = shortHuman.Stdout + shortHuman.Stderr; + Assert.Contains( + "First failure: src/App.cs (file_read_error, reading): access denied", + shortOutput, + StringComparison.Ordinal); + Assert.Contains("Retry indexing.", shortOutput, StringComparison.Ordinal); + Assert.DoesNotContain(TruncationMarker, shortOutput, StringComparison.Ordinal); + } + finally + { + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + private static string CreateControlledValue(string prefix, int length) + { + var rawPrefix = prefix + "\r\n\t\u001b\u0001-"; + return rawPrefix + new string('x', length - rawPrefix.Length); + } + + private static void SetFailedStatus( + string dbPath, + string file, + string category, + string phase, + string detail, + string recoveryHint) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.MarkIndexIncomplete(["file_index_error"]); + writer.SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, "failed"); + writer.SetMeta(DbContext.LastFailedIndexRunModeMetaKey, "incremental"); + writer.SetMeta(DbContext.LastFailedIndexRunRecoveryHintMetaKey, recoveryHint); + writer.SetMeta( + DbContext.LastFailedIndexRunFileErrorsMetaKey, + JsonSerializer.Serialize( + new List + { + new() + { + File = file, + Category = category, + Phase = phase, + Detail = detail, + }, + }, + StatusMetadataJsonContext.Default.ListStatusIndexFileError)); + } + + private static void PrepareImmutableRead(string dbPath) + { + SqliteConnection.ClearAllPools(); + using (var connection = new SqliteConnection( + new SqliteConnectionStringBuilder { DataSource = dbPath, Pooling = false }.ConnectionString)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA wal_checkpoint(TRUNCATE)"; + using (var checkpoint = command.ExecuteReader()) + { + Assert.True(checkpoint.Read()); + Assert.Equal(0L, checkpoint.GetInt64(0)); + } + command.CommandText = "PRAGMA journal_mode=DELETE"; + Assert.Equal("delete", command.ExecuteScalar()?.ToString(), ignoreCase: true); + } + SqliteConnection.ClearAllPools(); + } + + private static int CountOccurrences(string value, string search) + => value.Split(search, StringSplitOptions.None).Length - 1; +} From c48d0bc9d8973c87d694923fa6e95370109dc802 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 24 Aug 2026 11:13:09 +0900 Subject: [PATCH 2/3] Fix Windows status output assertion (#5161) --- tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs index c3a2929b6c..1c1e19cb87 100644 --- a/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs +++ b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs @@ -98,14 +98,15 @@ public void RunStatus_PreservesJsonAndSanitizesHumanFailureDiagnostics_Issue5161 Assert.Equal(CommandExitCodes.Success, human.ExitCode); var humanOutput = human.Stdout + human.Stderr; var firstFailureLine = Assert.Single( - humanOutput.Split('\n').Where(line => line.Contains("First failure:", StringComparison.Ordinal))); + humanOutput.Split(Environment.NewLine, StringSplitOptions.None) + .Where(line => line.Contains("First failure:", StringComparison.Ordinal))); Assert.Equal(4, CountOccurrences(firstFailureLine, TruncationMarker)); Assert.DoesNotContain('\r', firstFailureLine); Assert.DoesNotContain('\t', firstFailureLine); Assert.DoesNotContain('\u001b', firstFailureLine); Assert.DoesNotContain('\u0001', firstFailureLine); var hintLine = Assert.Single( - humanOutput.Split('\n').Where(line => + humanOutput.Split(Environment.NewLine, StringSplitOptions.None).Where(line => line.TrimStart().StartsWith("Hint", StringComparison.Ordinal) && line.Contains(TruncationMarker, StringComparison.Ordinal))); Assert.Contains(TruncationMarker, hintLine, StringComparison.Ordinal); From 19f6ed7b04977249f262f8572f259ba6788410dc Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 24 Aug 2026 11:18:59 +0900 Subject: [PATCH 3/3] Address adversarial review findings (#5161) --- tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs index 1c1e19cb87..cbaa935315 100644 --- a/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs +++ b/tests/CodeIndex.Tests/StatusMetadataOutputIssue5161Tests.cs @@ -102,6 +102,7 @@ public void RunStatus_PreservesJsonAndSanitizesHumanFailureDiagnostics_Issue5161 .Where(line => line.Contains("First failure:", StringComparison.Ordinal))); Assert.Equal(4, CountOccurrences(firstFailureLine, TruncationMarker)); Assert.DoesNotContain('\r', firstFailureLine); + Assert.DoesNotContain('\n', firstFailureLine); Assert.DoesNotContain('\t', firstFailureLine); Assert.DoesNotContain('\u001b', firstFailureLine); Assert.DoesNotContain('\u0001', firstFailureLine); @@ -111,6 +112,7 @@ public void RunStatus_PreservesJsonAndSanitizesHumanFailureDiagnostics_Issue5161 && line.Contains(TruncationMarker, StringComparison.Ordinal))); Assert.Contains(TruncationMarker, hintLine, StringComparison.Ordinal); Assert.DoesNotContain('\r', hintLine); + Assert.DoesNotContain('\n', hintLine); Assert.DoesNotContain('\t', hintLine); Assert.DoesNotContain('\u001b', hintLine); Assert.DoesNotContain('\u0001', hintLine);