diff --git a/.dockerignore b/.dockerignore index 4873ca2..ad85cf6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,13 +12,36 @@ htmlcov build dist *.egg-info +scratch *.session +*.session-shm +*.session-wal *.session-journal *.db +*.db-shm +*.db-wal +*.db-journal *.sqlite +*.sqlite-shm +*.sqlite-wal +*.sqlite-journal *.sqlite3 +*.sqlite3-shm +*.sqlite3-wal +*.sqlite3-journal config.json +config.local.json .env +.env.local +DB_TARGETS.txt +*_state.json +.tg_msg_manager.lock +delete_log.txt +*.log +DB_EXPORTS +PUBLIC_GROUPS +PRIVAT_DIALOGS exports +LOGS logs TG_MSG_MANAGER diff --git a/deploy/local/tg-state-sync.sh b/deploy/local/tg-state-sync.sh index 4c0c9fb..f3cb014 100755 --- a/deploy/local/tg-state-sync.sh +++ b/deploy/local/tg-state-sync.sh @@ -1,8 +1,11 @@ #!/bin/sh set -eu +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +DEFAULT_LOCAL_STATE_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd) + TGMM_SSH_HOST="${TGMM_SSH_HOST:-vps}" -TGMM_LOCAL_STATE_DIR="${TGMM_LOCAL_STATE_DIR:-$HOME/TG_MSG_MANAGER}" +TGMM_LOCAL_STATE_DIR="${TGMM_LOCAL_STATE_DIR:-$DEFAULT_LOCAL_STATE_DIR}" TGMM_REMOTE_STATE_DIR="${TGMM_REMOTE_STATE_DIR:-/opt/tg-msg-manager/state}" usage() { @@ -15,26 +18,151 @@ usage() { } run_rsync() { - rsync -az --partial --info=progress2 "$@" + rsync -azr --partial --info=progress2 "$@" +} + +remote_shell_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +warn_if_locked() { + if [ -e "$TGMM_LOCAL_STATE_DIR/.tg_msg_manager.lock" ]; then + printf '%s\n' "WARNING: local .tg_msg_manager.lock exists; stop tg-msg-manager before push. Lock file will not be synced." >&2 + fi +} + +append_existing_path() { + files_from=$1 + relative_path=$2 + + if [ -e "$TGMM_LOCAL_STATE_DIR/$relative_path" ]; then + printf '%s\0' "$relative_path" >> "$files_from" + fi +} + +append_root_matches() { + files_from=$1 + pattern=$2 + + find "$TGMM_LOCAL_STATE_DIR" -maxdepth 1 -name "$pattern" -exec sh -c ' + files_from=$1 + shift + for path do + relative_path=${path##*/} + printf "%s\0" "$relative_path" >> "$files_from" + done + ' sh "$files_from" {} + +} + +build_push_file_list() { + files_from=$1 + + append_existing_path "$files_from" config.json + append_existing_path "$files_from" config.local.json + append_existing_path "$files_from" DB_TARGETS.txt + append_existing_path "$files_from" delete_log.txt + append_existing_path "$files_from" export_state.json + append_existing_path "$files_from" deep_export_state.json + append_existing_path "$files_from" deep_json_export_state.json + append_existing_path "$files_from" json_export_state.json + append_existing_path "$files_from" pm_export_state.json + + append_root_matches "$files_from" "*.log" + append_root_matches "$files_from" "*.session" + append_root_matches "$files_from" "*.session-shm" + append_root_matches "$files_from" "*.session-wal" + append_root_matches "$files_from" "*.session-journal" + append_root_matches "$files_from" "*.db" + append_root_matches "$files_from" "*.db-shm" + append_root_matches "$files_from" "*.db-wal" + append_root_matches "$files_from" "*.db-journal" + append_root_matches "$files_from" "*.sqlite" + append_root_matches "$files_from" "*.sqlite-shm" + append_root_matches "$files_from" "*.sqlite-wal" + append_root_matches "$files_from" "*.sqlite-journal" + append_root_matches "$files_from" "*.sqlite3" + append_root_matches "$files_from" "*.sqlite3-shm" + append_root_matches "$files_from" "*.sqlite3-wal" + append_root_matches "$files_from" "*.sqlite3-journal" + + append_existing_path "$files_from" DB_EXPORTS/ + append_existing_path "$files_from" PUBLIC_GROUPS/ + append_existing_path "$files_from" PRIVAT_DIALOGS/ + append_existing_path "$files_from" exports/ + append_existing_path "$files_from" LOGS/ } push_state() { mode=$1 - mkdir -p "$TGMM_LOCAL_STATE_DIR" + files_from=$(mktemp) + trap 'rm -f "$files_from"' EXIT HUP INT TERM + + warn_if_locked + build_push_file_list "$files_from" + if [ "$mode" = "--dry-run" ]; then - run_rsync --dry-run "$TGMM_LOCAL_STATE_DIR/" "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/" + run_rsync --dry-run --from0 --files-from="$files_from" "$TGMM_LOCAL_STATE_DIR/" "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/" elif [ "$mode" = "--apply" ]; then - run_rsync "$TGMM_LOCAL_STATE_DIR/" "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/" + quoted_remote_dir=$(remote_shell_quote "$TGMM_REMOTE_STATE_DIR") + ssh "$TGMM_SSH_HOST" "mkdir -p $quoted_remote_dir" + run_rsync --from0 --files-from="$files_from" "$TGMM_LOCAL_STATE_DIR/" "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/" else usage exit 2 fi } -pull_dir() { +remote_path_exists() { + relative_path=$1 + quoted_remote_path=$(remote_shell_quote "$TGMM_REMOTE_STATE_DIR/$relative_path") + ssh "$TGMM_SSH_HOST" "test -e $quoted_remote_path" +} + +remote_root_logs_exist() { + quoted_remote_dir=$(remote_shell_quote "$TGMM_REMOTE_STATE_DIR") + ssh "$TGMM_SSH_HOST" "set -- $quoted_remote_dir/*.log; [ -e \"\$1\" ]" +} + +pull_optional_dir() { + name=$1 + if remote_path_exists "$name"; then + mkdir -p "$TGMM_LOCAL_STATE_DIR/$name" + run_rsync "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/$name/" "$TGMM_LOCAL_STATE_DIR/$name/" + else + printf '%s\n' "skip missing remote path: $name" >&2 + fi +} + +pull_optional_file() { name=$1 - mkdir -p "$TGMM_LOCAL_STATE_DIR/$name" - run_rsync "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/$name/" "$TGMM_LOCAL_STATE_DIR/$name/" + if remote_path_exists "$name"; then + mkdir -p "$TGMM_LOCAL_STATE_DIR" + run_rsync "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/$name" "$TGMM_LOCAL_STATE_DIR/" + else + printf '%s\n' "skip missing remote path: $name" >&2 + fi +} + +pull_root_logs() { + if remote_root_logs_exist; then + mkdir -p "$TGMM_LOCAL_STATE_DIR" + run_rsync "$TGMM_SSH_HOST:$TGMM_REMOTE_STATE_DIR/"*.log "$TGMM_LOCAL_STATE_DIR/" + else + printf '%s\n' "skip missing remote path: *.log" >&2 + fi +} + +pull_exports() { + pull_optional_dir exports + pull_optional_dir DB_EXPORTS + pull_optional_dir PUBLIC_GROUPS + pull_optional_dir PRIVAT_DIALOGS +} + +pull_logs() { + pull_optional_dir LOGS + pull_optional_file delete_log.txt + pull_root_logs } if [ "$#" -lt 2 ]; then @@ -47,14 +175,14 @@ case "$1:$2" in push_state "$2" ;; pull:exports) - pull_dir exports + pull_exports ;; pull:logs) - pull_dir logs + pull_logs ;; pull:all) - pull_dir exports - pull_dir logs + pull_exports + pull_logs ;; *) usage diff --git a/deploy/vps/README.md b/deploy/vps/README.md index d603172..58abf96 100644 --- a/deploy/vps/README.md +++ b/deploy/vps/README.md @@ -63,9 +63,11 @@ tgd На локальной машине остановите локальный запуск приложения перед push, чтобы не копировать состояние во время записи. +По умолчанию local source — корень текущего checkout, из которого запускается helper, например `/Users/maczone/dev/TG_CLEANER`. Это удобно, если runtime-файлы лежат рядом с исходным кодом, но helper синхронизирует только allowlist runtime state, а не исходный код, `.git`, `docs`, `tests`, `deploy`, virtualenv или build artifacts. + ```bash +cd /Users/maczone/dev/TG_CLEANER export TGMM_SSH_HOST=vps -export TGMM_LOCAL_STATE_DIR="$HOME/TG_MSG_MANAGER" export TGMM_REMOTE_STATE_DIR=/opt/tg-msg-manager/state ./deploy/local/tg-state-sync.sh push --dry-run @@ -75,4 +77,21 @@ export TGMM_REMOTE_STATE_DIR=/opt/tg-msg-manager/state ./deploy/local/tg-state-sync.sh pull all ``` -Sync helper использует `rsync` over SSH, `--partial` и `--info=progress2`. Он не использует `--delete`. +Если state находится не в checkout root, задайте `TGMM_LOCAL_STATE_DIR` вручную перед запуском. + +`push` передаёт только: + +- `config.json`, `config.local.json`, `DB_TARGETS.txt`, `delete_log.txt`; +- root-level `*.log`; +- root-level Telegram `*.session` вместе с `*.session-shm`, `*.session-wal`, `*.session-journal`; +- root-level SQLite `*.db`, `*.sqlite`, `*.sqlite3` вместе с WAL/SHM/journal sidecars; +- `*_state.json` из поддерживаемых runtime state-файлов; +- директории `DB_EXPORTS/`, `PUBLIC_GROUPS/`, `PRIVAT_DIALOGS/`, `exports/`, `LOGS/`. + +`.tg_msg_manager.lock` не синхронизируется. Если lock существует, helper выводит предупреждение; остановите локальное приложение и повторите push. + +`pull exports` забирает только export directories: `exports/`, `DB_EXPORTS/`, `PUBLIC_GROUPS/`, `PRIVAT_DIALOGS/`. `pull logs` забирает `LOGS/`, `delete_log.txt` и root-level `*.log`. Pull не загружает обратно `config.json`, `config.local.json`, Telegram sessions или SQLite databases. + +Sync helper использует `rsync` over SSH, archive mode, compression, `--partial` и `--info=progress2`. Он не использует `--delete`. + +Не коммитьте `config.json`, `config.local.json`, Telegram sessions, SQLite databases, exports или logs; не прикладывайте их к PR/issues и не включайте в Docker image. diff --git a/docs/stages/README.md b/docs/stages/README.md index e5afceb..ad74163 100644 --- a/docs/stages/README.md +++ b/docs/stages/README.md @@ -28,6 +28,7 @@ Stage 7A.1 pytest authority and CI gate alignment is recorded in [`reports/STAGE Stage 7A.2 fixture contract snapshot gate is recorded in [`reports/STAGE_7A_2_FIXTURE_CONTRACT_SNAPSHOT_GATE_REPORT.md`](reports/STAGE_7A_2_FIXTURE_CONTRACT_SNAPSHOT_GATE_REPORT.md). Stage 7A.3 coverage and typecheck gate precheck is recorded in [`reports/STAGE_7A_3_COVERAGE_AND_TYPECHECK_GATE_PRECHECK_REPORT.md`](reports/STAGE_7A_3_COVERAGE_AND_TYPECHECK_GATE_PRECHECK_REPORT.md). Stage 7B.0 VPS Docker Compose deployment wrapper is recorded in [`reports/STAGE_7B_0_VPS_DOCKER_COMPOSE_DEPLOYMENT_REPORT.md`](reports/STAGE_7B_0_VPS_DOCKER_COMPOSE_DEPLOYMENT_REPORT.md). +Stage 7B.1 VPS state sync safety correction is recorded in [`reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md`](reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md). Stage 6B.0 direct export failure exit code is recorded in [`reports/STAGE_6B_0_DIRECT_EXPORT_FAILURE_EXIT_CODE_REPORT.md`](reports/STAGE_6B_0_DIRECT_EXPORT_FAILURE_EXIT_CODE_REPORT.md). Stage 6B.1 direct PM archive failure exit code is recorded in [`reports/STAGE_6B_1_DIRECT_PM_ARCHIVE_FAILURE_EXIT_CODE_REPORT.md`](reports/STAGE_6B_1_DIRECT_PM_ARCHIVE_FAILURE_EXIT_CODE_REPORT.md). Stage 6C.0 SQLite writer flush failure diagnosis is recorded in [`reports/STAGE_6C_0_SQLITE_WRITER_FLUSH_FAILURE_DIAGNOSIS_REPORT.md`](reports/STAGE_6C_0_SQLITE_WRITER_FLUSH_FAILURE_DIAGNOSIS_REPORT.md). @@ -205,6 +206,7 @@ Completed stage files under [`completed/`](completed/) are historical instructio Current completed prompt groups: +- Stage 7B.1 VPS state sync safety correction task prompt. - Stage 7B.0 VPS Docker Compose deployment wrapper task prompt. - Stage 7A.3 coverage and typecheck gate precheck task prompt. - Stage 7A.2 fixture contract snapshot gate task prompt. diff --git a/docs/stages/completed/stage_7b_1_vps_state_sync_safety_correction.md b/docs/stages/completed/stage_7b_1_vps_state_sync_safety_correction.md new file mode 100644 index 0000000..2cc76b0 --- /dev/null +++ b/docs/stages/completed/stage_7b_1_vps_state_sync_safety_correction.md @@ -0,0 +1,120 @@ +# STAGE 7B.1 — VPS state sync safety correction + +Status: completed +Stage: 7B.1 +Type: implementation +Depends on: Stage 7B.0 merge commit `b912b51`. + +## 0. CODEX ENTRY CONTRACT + +Read `AGENTS.md` first. Use `stage-reviewer` before implementation. Use `architecture-guard` only if implementation touches CLI code, services, storage, protected files, compatibility wrappers, dataset/export behavior, or architecture rules. + +Work only on a separate branch and through a PR. Do not connect to a real VPS. Do not use real `config.json`, Telegram sessions, SQLite databases, exports, or logs. + +## 1. PURPOSE + +Correct the Stage 7B.0 local state sync helper so it syncs only tg-msg-manager runtime state, not the source checkout, while preserving Docker Compose behavior and all application behavior. + +## 2. FILES TO INSPECT + +Required: + +```text +AGENTS.md +.dockerignore +deploy/local/tg-state-sync.sh +deploy/vps/README.md +docs/stages/README.md +``` + +May create or edit only: + +```text +.dockerignore +deploy/local/tg-state-sync.sh +deploy/vps/README.md +docs/stages/active/stage_7b_1_vps_state_sync_safety_correction.md +docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md +docs/stages/README.md +docs/stages/completed/stage_7b_1_vps_state_sync_safety_correction.md +``` + +## 3. HARD PROHIBITIONS + +- Do not change Dockerfile, compose.yaml, CLI code, services, storage, application tests, or Stage 7B.0 shell aliases unless a blocker proves it is required. +- Do not change public CLI behavior, SQLite schema, export formats, Telegram authorization, Docker Compose logic, service behavior, storage behavior, or logging behavior. +- Do not add systemd, cron, CI workflows, registry setup, secrets, or network services. +- Do not use `--delete` in sync helpers. +- Do not connect to a real VPS or transfer real config, sessions, SQLite databases, exports, or logs. + +## 4. ATOMIC IMPLEMENTATION TASKS + +1. Change `deploy/local/tg-state-sync.sh` default `TGMM_LOCAL_STATE_DIR` to the checkout root two levels above the script, while preserving manual override. +2. Make push use an explicit allowlist of runtime root files and directories only; preserve relative structure, warn to stderr if `.tg_msg_manager.lock` exists, never sync it, never sync source/docs/tests/deploy/.git/venvs/build/scratch/unknown files, and do not create remote directories during `push --dry-run`. +3. Make `push --apply` create `TGMM_REMOTE_STATE_DIR` if needed, then rsync with archive mode, compression, `--partial`, and `--info=progress2`. +4. Preserve pull commands. `pull exports` fetches `exports/`, `DB_EXPORTS/`, `PUBLIC_GROUPS/`, `PRIVAT_DIALOGS/`; `pull logs` fetches `LOGS/`, `delete_log.txt`, and root-level `*.log`; `pull all` runs both. Missing optional paths must be skipped with a short message and must not stop remaining pulls. +5. Update `.dockerignore` to exclude runtime state and sidecars without weakening existing protections. +6. Update `deploy/vps/README.md` in Russian with the corrected local source, allowlist behavior, examples, lock handling, WAL/SHM sidecars, and secret warnings. +7. Create the factual report in Russian, then perform lifecycle cleanup according to `AGENTS.md`. + +## 5. REQUIRED DOCS + +Required: + +```text +deploy/vps/README.md +docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md +docs/stages/README.md +``` + +## 6. TESTS / VERIFICATION + +Run and record: + +```bash +sh -n deploy/local/tg-state-sync.sh +git diff --check +make verify +make pre-commit +``` + +Check manually by code inspection that push cannot transfer `.git`, source code, `docs`, `tests`, or virtualenvs. + +Run Docker build/Compose checks only if Docker daemon is available. If unavailable, record exact reason without marking them passed. + +## 7. REPORT + +Write `docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md` in Russian. + +Include: + +```text +scope summary +changed files +behavior preservation notes +sync allowlist notes +pull behavior notes +secret/data exclusion notes +check results +Docker availability result +manual safety inspection result +blockers or caveats +stage-reviewer: applied from .skills/stage-reviewer/SKILL.md +architecture-guard: applied from .skills/architecture-guard/SKILL.md or not required with reason +stage-completion-auditor: applied from .skills/stage-completion-auditor/SKILL.md +``` + +## 8. COMPLETION CRITERIA + +- Local sync helper defaults to checkout root but only pushes explicit runtime state allowlist. +- Pull behavior preserves export/log retrieval and does not pull config, sessions, or SQLite databases. +- `.dockerignore` excludes required runtime state and sidecars. +- Russian VPS docs are updated. +- Required checks are run and recorded, or environment blockers are recorded honestly. +- Factual report exists in Russian. +- `docs/stages/README.md` is updated and lifecycle cleanup is complete. +- PR exists for the branch. + +## 9. OUTPUT LIMITS + +Final response must follow `AGENTS.md` final format, be in Russian, and stay under 1200 characters. diff --git a/docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md b/docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md new file mode 100644 index 0000000..f41c4da --- /dev/null +++ b/docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md @@ -0,0 +1,58 @@ +# STAGE 7B.1 — VPS state sync safety correction report + +## Scope summary + +Исправлен `deploy/local/tg-state-sync.sh`: default local source теперь корень checkout, но `push` синхронизирует только явный allowlist runtime state tg-msg-manager. Исходный код, `.git`, docs/tests/deploy, virtualenv, build/scratch и неизвестные root-level файлы не попадают в push. + +## Changed files + +- `.dockerignore` +- `deploy/local/tg-state-sync.sh` +- `deploy/vps/README.md` +- `docs/stages/reports/STAGE_7B_1_VPS_STATE_SYNC_SAFETY_CORRECTION_REPORT.md` +- `docs/stages/README.md` +- `docs/stages/completed/stage_7b_1_vps_state_sync_safety_correction.md` + +## Behavior preservation notes + +Код приложения, CLI, SQLite schema, export formats, Telegram authorization, Dockerfile, `compose.yaml`, сервисы, storage, тесты приложения и shell aliases Stage 7B.0 не изменялись. + +## Sync allowlist notes + +`push` строит NUL-delimited `--files-from` allowlist только из разрешённых runtime root files/patterns и директорий. `push --dry-run` не создаёт remote directory. `push --apply` создаёт только `TGMM_REMOTE_STATE_DIR`, затем запускает rsync. `--delete` не используется. + +Если локальный `.tg_msg_manager.lock` существует, helper пишет предупреждение в stderr, но lock не синхронизирует и не удаляет. + +## Pull behavior notes + +`pull exports` получает только `exports/`, `DB_EXPORTS/`, `PUBLIC_GROUPS/`, `PRIVAT_DIALOGS/`. `pull logs` получает только `LOGS/`, `delete_log.txt` и root-level `*.log`. Отсутствующие optional paths пропускаются с коротким сообщением. Pull не загружает обратно configs, Telegram sessions или SQLite databases. + +## Secret/data exclusion notes + +`.dockerignore` усилен для `config.local.json`, `.env.local`, session/SQLite sidecars, `DB_TARGETS.txt`, `*_state.json`, `.tg_msg_manager.lock`, `delete_log.txt`, root-level `*.log`, `DB_EXPORTS`, `PUBLIC_GROUPS`, `PRIVAT_DIALOGS`, `LOGS`, `exports`, `logs`, без ослабления `.git`, venv/cache/build protections. + +## Check results + +- `sh -n deploy/local/tg-state-sync.sh`: passed. +- `git diff --check`: passed. +- `make verify`: passed, `634 passed`. +- `make pre-commit`: passed, embedded `make verify` completed with `634 passed`. + +## Docker availability result + +- `docker info`: failed after sandbox escalation; Docker daemon unavailable: `Cannot connect to the Docker daemon at unix:///Users/maczone/.docker/run/docker.sock. Is the docker daemon running?` +- Docker build/Compose checks were not run because Docker daemon was unavailable. + +## Manual safety inspection result + +Code inspection confirms push can only pass entries emitted by `build_push_file_list` into `rsync --from0 --files-from`. That list never includes `.git`, source package directories, `docs`, `tests`, `deploy`, `.github`, virtualenvs, build artifacts, `scratch`, `.tg_msg_manager.lock`, or unknown root-level files. + +## Blockers or caveats + +Docker daemon was unavailable in the local Codex environment. No real VPS or real runtime data was used. + +## Skill application + +- `stage-reviewer: applied from .skills/stage-reviewer/SKILL.md` +- `architecture-guard: not required; stage changed deployment helper/docs/.dockerignore only and did not touch CLI code, services, storage, protected files, compatibility wrappers, dataset/export behavior, or architecture rules` +- `stage-completion-auditor: applied from .skills/stage-completion-auditor/SKILL.md`