publish development /data/ copy #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: publish development /data/ copy | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| publish-dev-data: | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: publish-dev-data | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| env: | |
| REMOTE: deploy@45.76.134.105 | |
| TARGET: /var/www/fridge.dev | |
| REMOTE_WORK_DIR: /home/deploy | |
| GDRIVE_REMOTE: gdrive | |
| GDRIVE_FOLDER_ID: ${{ vars.GDRIVE_DEV_DATA_FOLDER_ID }} | |
| steps: | |
| - name: check out repository | |
| uses: actions/checkout@v4 | |
| - name: install ssh, rclone, and jq | |
| run: sudo apt-get update && sudo apt-get install -y openssh-client rclone jq | |
| - name: set up ssh and rclone | |
| env: | |
| DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} | |
| RCLONE_CONFIG_CONTENT: ${{ secrets.RCLONE_CONFIG }} | |
| run: | | |
| mkdir -p ~/.ssh ~/.config/rclone | |
| printf '%s\n' "$DEPLOY_KEY" | tr -d '\r' > ~/.ssh/id_ed25519 | |
| chmod 600 ~/.ssh/id_ed25519 | |
| ssh-keyscan -H 45.76.134.105 >> ~/.ssh/known_hosts | |
| printf '%s' "$RCLONE_CONFIG_CONTENT" > ~/.config/rclone/rclone.conf | |
| chmod 600 ~/.config/rclone/rclone.conf | |
| - name: validate publish settings | |
| run: | | |
| set -euo pipefail | |
| [ -n "$GDRIVE_FOLDER_ID" ] || { echo "GDRIVE_DEV_DATA_FOLDER_ID is not configured" >&2; exit 1; } | |
| - name: create remote development workspace | |
| id: remote-workspace | |
| run: | | |
| set -euo pipefail | |
| remote_work_dir="$( | |
| ssh "$REMOTE" TARGET="$TARGET" REMOTE_WORK_DIR="$REMOTE_WORK_DIR" bash -s <<'EOF' | |
| set -euo pipefail | |
| [ "$TARGET" = "/var/www/fridge.dev" ] || { echo "refusing to publish data from unexpected target: $TARGET" >&2; exit 1; } | |
| [ -d "$TARGET" ] || { echo "site directory not found: $TARGET" >&2; exit 1; } | |
| cd "$TARGET" | |
| command -v php >/dev/null 2>&1 || { echo "php is not installed on the server" >&2; exit 1; } | |
| command -v rsync >/dev/null 2>&1 || { echo "rsync is not installed on the server" >&2; exit 1; } | |
| [ -d data ] || { echo "data directory not found at $TARGET/data" >&2; exit 1; } | |
| [ -w "$REMOTE_WORK_DIR" ] || { echo "remote work directory is not writable: $REMOTE_WORK_DIR" >&2; exit 1; } | |
| # Repair files created by older PHP writers with owner-only modes. | |
| # deploy belongs to the http group and needs read/traverse access | |
| # for rsync; the rebuildable hard-ban index remains excluded. | |
| sudo -n -u http find data \ | |
| -path 'data/etc/banlists/index' -prune -o \ | |
| -exec chmod g+rX {} + 2>/dev/null || true | |
| # A failed or cancelled earlier run can leave its workspace behind. | |
| # Runs are serialized above, so this cannot remove an active run's files. | |
| while IFS= read -r -d '' stale_work_dir; do | |
| echo "Removing stale development workspace: $stale_work_dir" >&2 | |
| rm -rf -- "$stale_work_dir" | |
| done < <(find "$REMOTE_WORK_DIR" -mindepth 1 -maxdepth 1 -type d -name 'dev-data.*' -print0) | |
| work_dir="" | |
| cleanup_failed_workspace() { | |
| status=$? | |
| [ -z "$work_dir" ] || rm -rf -- "$work_dir" | |
| exit "$status" | |
| } | |
| trap cleanup_failed_workspace ERR | |
| work_dir="$(mktemp -d "${REMOTE_WORK_DIR%/}/dev-data.XXXXXX")" | |
| sanitized_data="${work_dir%/}/data" | |
| echo "Copying production data to temporary workspace: $work_dir" >&2 | |
| mkdir -p "$sanitized_data" | |
| rsync -a --exclude '/etc/banlists/index/' data/ "$sanitized_data/" | |
| trap - ERR | |
| printf "%s" "$work_dir" | |
| EOF | |
| )" | |
| echo "remote_work_dir=$remote_work_dir" >> "$GITHUB_OUTPUT" | |
| - name: upload sanitizer script | |
| run: | | |
| set -euo pipefail | |
| scp .github/scripts/sanitize-dev-data.php "$REMOTE:${{ steps.remote-workspace.outputs.remote_work_dir }}/sanitize-dev-data.php" | |
| - name: sanitize and compress development data | |
| id: remote-dev-data | |
| run: | | |
| set -euo pipefail | |
| remote_archive_path="$( | |
| ssh "$REMOTE" REMOTE_WORK_DIR="${{ steps.remote-workspace.outputs.remote_work_dir }}" bash -s <<'EOF' | |
| set -euo pipefail | |
| command -v zip >/dev/null 2>&1 || { echo "zip is not installed on the server" >&2; exit 1; } | |
| command -v php >/dev/null 2>&1 || { echo "php is not installed on the server" >&2; exit 1; } | |
| work_dir="$REMOTE_WORK_DIR" | |
| archive_name="$(date +"%d-%m-%y_%H-%M-%S").zip" | |
| archive_path="${work_dir%/}/$archive_name" | |
| sanitized_data="${work_dir%/}/data" | |
| sanitizer="${work_dir%/}/sanitize-dev-data.php" | |
| zip_log="${work_dir%/}/zip.log" | |
| php "$sanitizer" "$sanitized_data" | |
| echo "Creating sanitized development archive at $archive_path" >&2 | |
| cd "$work_dir" | |
| dev_data_excludes=( | |
| 'data/etc/hard-banned-ips.txt' | |
| 'data/etc/hard-ban-identities.json' | |
| 'data/etc/access.json*' | |
| 'data/etc/banlists/*' | |
| ) | |
| if zip -r "$archive_path" data -x "${dev_data_excludes[@]}" >"$zip_log" 2>&1; then | |
| : | |
| else | |
| status=$? | |
| echo "zip failed with exit code $status" >&2 | |
| cat "$zip_log" >&2 | |
| exit "$status" | |
| fi | |
| ls -lh "$archive_path" >&2 | |
| printf "%s" "$archive_path" | |
| EOF | |
| )" | |
| archive_name="$(basename "$remote_archive_path")" | |
| echo "remote_archive_path=$remote_archive_path" >> "$GITHUB_OUTPUT" | |
| echo "archive_name=$archive_name" >> "$GITHUB_OUTPUT" | |
| - name: download development archive | |
| run: | | |
| set -euo pipefail | |
| scp "$REMOTE:${{ steps.remote-dev-data.outputs.remote_archive_path }}" "${{ steps.remote-dev-data.outputs.archive_name }}" | |
| - name: upload development archive to Google Drive | |
| run: | | |
| set -euo pipefail | |
| rclone copy \ | |
| "${{ steps.remote-dev-data.outputs.archive_name }}" \ | |
| "${GDRIVE_REMOTE}:" \ | |
| --drive-root-folder-id "${GDRIVE_FOLDER_ID}" | |
| - name: keep only the 10 newest development archives | |
| run: | | |
| set -euo pipefail | |
| old_files="$( | |
| rclone lsjson "${GDRIVE_REMOTE}:" \ | |
| --drive-root-folder-id "${GDRIVE_FOLDER_ID}" \ | |
| --files-only \ | |
| | jq -r 'sort_by(.ModTime) | reverse | .[10:][]?.Path' | |
| )" | |
| if [ -z "$old_files" ]; then | |
| echo "No old development archives to delete." | |
| exit 0 | |
| fi | |
| while IFS= read -r file_path; do | |
| [ -n "$file_path" ] || continue | |
| rclone deletefile "${GDRIVE_REMOTE}:${file_path}" --drive-root-folder-id "${GDRIVE_FOLDER_ID}" | |
| done <<< "$old_files" | |
| - name: add download link to workflow summary | |
| run: | | |
| cat >> "$GITHUB_STEP_SUMMARY" <<'EOF' | |
| Development /data/ copy available for download at: [https://drive.google.com/drive/folders/1dltxdqQjfUfGwEEXVxUrOw5fuv9nk_ex](https://drive.google.com/drive/folders/1dltxdqQjfUfGwEEXVxUrOw5fuv9nk_ex) | |
| EOF | |
| - name: remove temporary files | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| archive_name="${{ steps.remote-dev-data.outputs.archive_name }}" | |
| remote_work_dir="${{ steps.remote-workspace.outputs.remote_work_dir }}" | |
| [ -z "$archive_name" ] || rm -f "$archive_name" | |
| [ -z "$remote_work_dir" ] || ssh "$REMOTE" "rm -rf '$remote_work_dir'" |