Add read-only real repo validation

Author: fqjony

README.md

@@ -181,6 +181,7 @@ All commands support `--json`.
 - [Environment Config](docs/environment-config.md)
 - [Context Coverage](docs/context-coverage.md)
 - [Integration](docs/integration.md)
+- [Real Repo Validation](docs/real-repo-validation.md)
 - [Smart Dependency Detection](docs/smart-dependency-detection.md)
 - [Reference Docs](docs/references/README.md)
 - [Reference: Command and Workflow Surfaces](docs/references/command-surfaces.md)
@@ -199,5 +200,5 @@ bash tests/worker-smoke.sh
 ```
 
 ```bash
-bash tests/real-repos.sh /path/to/repo1 /path/to/repo2
+bash tests/real-repos.sh --check /path/to/repo1 /path/to/repo2
 ```

docs/real-repo-validation.md

@@ -0,0 +1,47 @@
+# Real Repo Validation
+
+dev.kit is validated at two levels:
+
+- fixtures verify deterministic contracts inside this repo
+- real repos verify whether dev.kit reads repo-owned evidence well enough to guide work
+
+Keep fixtures small. They should cover stable output shape, gap categories, manifest handling, and known regressions. They should not try to model every UDX repo.
+
+Use real repos as optimization probes before a release. Run them read-only by default so the probe reports what dev.kit sees without changing the target repo.
+
+## Local UDX Matrix
+
+Use the matrix declared in `src/configs/repo-validation.yaml` when the listed repos are available locally.
+
+Example:
+
+```bash
+bash tests/real-repos.sh --check ./reusable-workflows ./github-rabbit-action
+```
+
+The summary should be used to compare:
+
+- home context status
+- repo workflow status
+- repo context status
+- gap count
+- read-first ref count
+
+When one repo looks noisy or inconsistent, repair the strongest repo-owned gap or dev.kit normalization issue, rerun the matrix, and verify the output changed.
+
+## Public Repo Probes
+
+Public repos are useful for compatibility checks, but they should be optional and pinned when used for repeatable release evidence. Upstream repos change for reasons unrelated to dev.kit.
+
+Use public probes to test broad ecosystem recognition:
+
+- package manifests and scripts
+- docs-first repos
+- workflow-only repos
+- container repos
+
+Do not assert exact output for moving public repos in the default suite.
+
+## Write Mode
+
+`tests/real-repos.sh --write` generates `.rabbit/context.yaml` in the target repo. Use it only for temp clones or repos intentionally selected for context regeneration.

lib/commands/repo.sh

@@ -3,11 +3,7 @@
 # @description: Analyze repo structure and factors
 
 dev_kit_repo_recommended_repos_text() {
-  cat <<'EOF'
-https://github.com/udx/worker
-https://github.com/udx/reusable-workflows
-https://github.com/udx/github-rabbit-action
-EOF
+  dev_kit_repo_validation_list "recommended_repos"
 }
 
 dev_kit_repo_recommended_repos_json() {

lib/modules/config_catalog.sh

@@ -24,6 +24,14 @@ dev_kit_context_config_path() {
   dev_kit_config_path "src/configs/context-config.yaml"
 }
 
+dev_kit_repo_validation_config_path() {
+  dev_kit_config_path "src/configs/repo-validation.yaml"
+}
+
+dev_kit_repo_validation_list() {
+  dev_kit_yaml_config_list "$(dev_kit_repo_validation_config_path)" "$1"
+}
+
 dev_kit_context_list() {
   dev_kit_yaml_config_list "$(dev_kit_context_config_path)" "$1"
 }

src/configs/repo-validation.yaml

@@ -0,0 +1,25 @@
+kind: repoValidationConfig
+version: udx.dev/dev.kit/v1
+description: Public repo probes and validation defaults for dev.kit release checks
+
+config:
+  recommended_repos:
+    - https://github.com/udx/worker
+    - https://github.com/udx/reusable-workflows
+    - https://github.com/udx/github-rabbit-action
+  real_repo_probe_examples:
+    - udx/reusable-workflows
+    - udx/github-rabbit-action
+  real_repo_matrix:
+    site_enc_apps:
+      repo: udx/site-enc-apps
+      purpose: many .rabbit/infra_configs environment definitions; validates manifest coverage without flooding first-read refs
+    gh_workflows:
+      repo: udx/gh-workflows
+      purpose: workflow-heavy repo; validates pipeline and reusable workflow context
+    rabbit_automation_action:
+      repo: udx/rabbit-automation-action
+      purpose: action, container, docs, and workflow mix
+    dev_kit:
+      repo: udx/dev.kit
+      purpose: self-hosting sanity check

tests/real-repos.sh

@@ -6,22 +6,37 @@ TEST_HOME="${DEV_KIT_TEST_HOME:-$(mktemp -d "${TMPDIR:-/tmp}/dev-kit-real-repos.
 DEV_KIT_BIN_DIR="$TEST_HOME/.local/bin"
 KEEP_HOME="${DEV_KIT_TEST_KEEP_HOME:-0}"
 REAL_REPOS_CSV="${DEV_KIT_TEST_REAL_REPOS:-}"
+MODE="check"
+REPORT_DIR="${DEV_KIT_TEST_REPORT_DIR:-$TEST_HOME/reports}"
+COMMAND_SOFT_TIMEOUT="${DEV_KIT_TEST_SOFT_TIMEOUT:-15}"
+COMMAND_HARD_TIMEOUT="${DEV_KIT_TEST_HARD_TIMEOUT:-180}"
+
+# shellcheck disable=SC1091
+. "$REPO_DIR/lib/modules/output.sh"
 
 usage() {
   cat <<'EOF'
-Usage: bash tests/real-repos.sh [/abs/path/to/repo ...]
+Usage: bash tests/real-repos.sh [--check|--write] [/abs/path/to/repo ...]
 
 Runs the current dev.kit working tree against real local repos using a temporary
 plain `dev.kit` shim, without changing the global install.
 
+Default mode is --check, which is read-only for target repos.
+
 Options via environment:
   DEV_KIT_TEST_REAL_REPOS   Colon-separated repo paths when no CLI args are given
   DEV_KIT_TEST_HOME         Temp home for the test run
+  DEV_KIT_TEST_REPORT_DIR   Directory for per-repo JSON reports
+  DEV_KIT_TEST_SOFT_TIMEOUT Seconds before a slow-command notice (default: 15)
+  DEV_KIT_TEST_HARD_TIMEOUT Seconds before stopping a command (default: 180)
   DEV_KIT_TEST_KEEP_HOME    Keep temp home after exit (default: 0)
 
 Examples:
-  bash tests/real-repos.sh ~/git/udx/reusable-workflows ~/git/udx/www.peakclt.com
-  DEV_KIT_TEST_REAL_REPOS="$HOME/git/udx/reusable-workflows:$HOME/git/udx/www.peakclt.com" bash tests/real-repos.sh
+  bash tests/real-repos.sh --check ./reusable-workflows ./github-rabbit-action
+  DEV_KIT_TEST_REAL_REPOS="./reusable-workflows:./github-rabbit-action" bash tests/real-repos.sh --check
+  bash tests/real-repos.sh --write /tmp/dev-kit-probe-clone
+
+Probe examples and release matrix candidates live in src/configs/repo-validation.yaml.
 EOF
 }
 
@@ -33,59 +48,131 @@ cleanup() {
 
 trap cleanup EXIT
 
-if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
-  usage
-  exit 0
-fi
-
 repo_paths=()
-if [ "$#" -gt 0 ]; then
-  while [ "$#" -gt 0 ]; do
-    repo_paths+=("$1")
-    shift
-  done
-elif [ -n "$REAL_REPOS_CSV" ]; then
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --check) MODE="check" ;;
+    --write) MODE="write" ;;
+    -h|--help) usage; exit 0 ;;
+    --*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 1 ;;
+    *) repo_paths+=("$1") ;;
+  esac
+  shift
+done
+
+if [ "${#repo_paths[@]}" -eq 0 ] && [ -n "$REAL_REPOS_CSV" ]; then
   while IFS= read -r repo_path; do
     [ -n "$repo_path" ] || continue
     repo_paths+=("$repo_path")
   done <<EOF
 $(printf '%s' "$REAL_REPOS_CSV" | tr ':' '\n')
 EOF
-else
+fi
+
+if [ "${#repo_paths[@]}" -eq 0 ]; then
   usage >&2
   exit 1
 fi
 
 mkdir -p "$DEV_KIT_BIN_DIR"
+mkdir -p "$REPORT_DIR"
 ln -sfn "$REPO_DIR/bin/dev-kit" "$DEV_KIT_BIN_DIR/dev.kit"
 
 export HOME="$TEST_HOME"
 export PATH="$DEV_KIT_BIN_DIR:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
 unset DEV_KIT_HOME
 unset DEV_KIT_BIN_DIR
 
+require_jq() {
+  if ! command -v jq >/dev/null 2>&1; then
+    printf 'jq is required for real repo report summaries\n' >&2
+    exit 1
+  fi
+}
+
+report_name_for_repo() {
+  basename "$1" | tr -c 'A-Za-z0-9._-' '-'
+}
+
+print_check_summary() {
+  local repo_path="$1"
+  local home_json="$2"
+  local repo_json="$3"
+  local refs_count=""
+  local gap_count=""
+  local workflow_status=""
+  local context_status=""
+  local home_context_status=""
+
+  home_context_status="$(jq -r '.synced.context_status // "none"' "$home_json")"
+  workflow_status="$(jq -r '.workflow.jobs[] | select(.id == "repo") | .status' "$repo_json")"
+  context_status="$(jq -r '.workflow.jobs[] | select(.id == "repo") | .context_status' "$repo_json")"
+  gap_count="$(jq -r '.workflow.jobs[] | select(.id == "repo") | .gap_count' "$repo_json")"
+  refs_count="$(jq -r '[.workflow.jobs[] | select(.id == "repo") | .steps[]? | select(.id == "read_repo") | .refs[]?] | length' "$repo_json")"
+
+  printf '%s\tmode=%s\thome_context=%s\trepo_status=%s\trepo_context=%s\tgaps=%s\tread_refs=%s\n' \
+    "$repo_path" "$MODE" "$home_context_status" "$workflow_status" "$context_status" "$gap_count" "$refs_count"
+}
+
+print_write_summary() {
+  local repo_path="$1"
+  local repo_json="$2"
+  local context_yaml="$repo_path/.rabbit/context.yaml"
+  local gap_count=""
+  local manifest_count=0
+  local dep_count=0
+
+  gap_count="$(jq -r '.workflow.jobs[] | select(.id == "repo") | .gap_count' "$repo_json")"
+  if [ -f "$context_yaml" ]; then
+    manifest_count="$(awk '/^manifests:/{flag=1;next} flag && /^[^[:space:]#]/{exit} flag && /^  - path:/{count += 1} END{print count + 0}' "$context_yaml")"
+    dep_count="$(awk '/^dependencies:/{flag=1;next} /^# Manifests/{if(flag) exit} flag && /^  - repo:/{count += 1} END{print count + 0}' "$context_yaml")"
+  fi
+
+  printf '%s\tmode=%s\tcontext=%s\tgaps=%s\tmanifests=%s\tdependencies=%s\n' \
+    "$repo_path" "$MODE" "$context_yaml" "$gap_count" "$manifest_count" "$dep_count"
+}
+
+run_report_command() {
+  local label="$1"
+  local output_file="$2"
+  shift 2
+
+  local tmp_file="${output_file}.tmp"
+  rm -f "$tmp_file"
+  DEV_KIT_SPINNER_DISABLE=1 dev_kit_run_guarded \
+    "$label" \
+    "$COMMAND_SOFT_TIMEOUT" \
+    "$COMMAND_HARD_TIMEOUT" \
+    "$label is taking longer than usual; still resolving repo evidence" \
+    "$@" >"$tmp_file"
+  mv "$tmp_file" "$output_file"
+}
+
+require_jq
+printf 'report_dir: %s\n' "$REPORT_DIR"
+printf 'mode: %s\n' "$MODE"
+
 for repo_path in "${repo_paths[@]}"; do
   repo_path="$(cd "$repo_path" 2>/dev/null && pwd || true)"
   [ -n "$repo_path" ] || { printf 'repo not found\n' >&2; exit 1; }
   [ -d "$repo_path/.git" ] || { printf 'not a git repo: %s\n' "$repo_path" >&2; exit 1; }
-  run_out="$(mktemp "$TEST_HOME/dev-kit-real.XXXXXX.out")"
+  repo_report_name="$(report_name_for_repo "$repo_path")"
+  home_json="$REPORT_DIR/${repo_report_name}.home.json"
+  repo_json="$REPORT_DIR/${repo_report_name}.repo.json"
 
   printf '\n===== %s =====\n' "$repo_path"
-  (
-    cd "$repo_path"
-    if ! dev.kit >"$run_out"; then
-      cat "$run_out" >&2
-      exit 1
-    fi
-    dev.kit repo >/dev/null
-    [ -f .rabbit/context.yaml ] || { printf 'missing .rabbit/context.yaml\n' >&2; exit 1; }
-
-    printf 'context: %s\n' "$repo_path/.rabbit/context.yaml"
-    printf '\nrepo:\n'
-    sed -n '1,20p' .rabbit/context.yaml
-    printf '\ngaps:\n'
-    awk '/^gaps:/{flag=1;next} /^# Dependencies/{if(flag) exit} flag{print}' .rabbit/context.yaml || true
-    printf '\ndependencies:\n'
-    awk '/^dependencies:/{flag=1;next} /^# Manifests/{if(flag) exit} flag{print}' .rabbit/context.yaml | sed -n '1,80p'
-  )
+  if [ "$MODE" = "check" ]; then
+    (
+      cd "$repo_path"
+      run_report_command "dev.kit home check: $repo_path" "$home_json" dev.kit --json
+      run_report_command "dev.kit repo check: $repo_path" "$repo_json" dev.kit repo --json --check
+    )
+    print_check_summary "$repo_path" "$home_json" "$repo_json"
+  else
+    (
+      cd "$repo_path"
+      run_report_command "dev.kit repo write: $repo_path" "$repo_json" dev.kit repo --json
+    )
+    print_write_summary "$repo_path" "$repo_json"
+  fi
 done