From d8265e9b7b46a72f299e2bba3bd25dc14676f1c1 Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Sun, 7 May 2017 15:28:39 +0200 Subject: [PATCH 1/3] Solved 'ssh -n' issue, improved portability - Improved consistency: if target machine is "localhost", ssh is not used at all (solved "ssh -n" issue) - Added description of parameters expected in config file, and a reminder to set properties on datasets - `zfs` command location is now auto-discovered - Added sudo fallback, if your machine does not have pfexec --- example.cfg | 13 ++++++++++++- zfs-backup.sh | 20 ++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/example.cfg b/example.cfg index 387475c..3ca2757 100644 --- a/example.cfg +++ b/example.cfg @@ -1,8 +1,19 @@ # sample configuration for hourly backups -# Don't forget to schedule a cron job such as: +# Don't forget to set the $PROP property for the filesystem, and schedule a cron job such as: # 30 * * * * /path/to/zfs-backup.sh /path/to/this.cfg +# Parameter list +# - PROP: the name of the user-property to search for, among the data sets, to identify which are eligible for backup. +# - TAG: a string that is matched against the snapshots' names, to identify which are eligible for backup. +# - REMHOST: IP/hostname of receiving host. Commands will be sent through ssh, but if this equals to "localhost", ssh will be bypassed. +# - REMUSER: user to use during remote ssh connection. +# - REMPOOL: name of the remote pool, to save the snapshot into. +# - DEBUG: if set to not null, dry-run is enabled. +# - VERBOSE: set to "-v" for verbose mode. +# - LOCK: file that is created in case zfs-backup ever has problems, and enters manteinance mode. No need to change this. +# - PID: pid file for zfs-backup. No need to change this. + TAG="zfs-auto-snap_hourly" LOCK="/var/tmp/zfsbackup.lock" PID="/var/tmp/zfsbackup.pid" diff --git a/zfs-backup.sh b/zfs-backup.sh index af45ab6..c81dad1 100755 --- a/zfs-backup.sh +++ b/zfs-backup.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env ksh +#!/usr/bin/env sh # Needs a POSIX-compatible sh, like ash (Debian & FreeBSD /bin/sh), ksh, or # bash. On Solaris 10 you need to use /usr/xpg4/bin/sh (the POSIX shell) or # /bin/ksh -- its /bin/sh is an ancient Bourne shell, which does not work. @@ -66,9 +66,8 @@ VERBOSE="" # "-v" for verbose, null string for quiet LOCK="/var/tmp/zfsbackup.lock" PID="/var/tmp/zfsbackup.pid" CFG="/var/lib/zfssnap/zfs-backup.cfg" -ZFS="/usr/sbin/zfs" -# Replace with sudo(8) if pfexec(1) is not available on your OS -PFEXEC=`which pfexec` +ZFS="$(which zfs)" +PFEXEC=`which pfexec || which sudo` # local settings -- datasets to back up are now found by property TAG="zfs-auto-snap_daily" @@ -193,16 +192,9 @@ do_backup() { snap2=${newest_local#*@} [ "$DEBUG" -o "$VERBOSE" ] && echo "$msg $snap2" - if [ "$REMHOST" = "localhost" ]; then - newest_remote="$($ZFS list -t snapshot -H -S creation -o name -d 1 $TARGET | grep $TAG | head -1)" - err_msg="Error fetching snapshot listing for local target pool $REMPOOL." - else - # ssh needs public key auth configured beforehand - # Not using $REMZFS_CMD because we need 'ssh -n' here, but must not use - # 'ssh -n' for the actual zfs recv. - newest_remote="$(ssh -n $REMUSER@$REMHOST $REMZFS list -t snapshot -H -S creation -o name -d 1 $TARGET | grep $TAG | head -1)" - err_msg="Error fetching remote snapshot listing via ssh to $REMUSER@$REMHOST." - fi + all_remotes=$($REMZFS_CMD list -t snapshot -H -S creation -o name -d 1 $TARGET < /dev/null) + newest_remote="$(echo "$all_remotes" | grep $TAG | head -1)" + err_msg="Error fetching remote snapshot listing to $REMHOST." if [ -z $newest_remote ]; then echo "$err_msg" >&2 [ $DEBUG ] || touch $LOCK From 19f6c2168784b8957b781ac51b3c2ee3666693b2 Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Sun, 7 May 2017 17:17:18 +0200 Subject: [PATCH 2/3] Auto recovery from locally-missing remote snapshot - Added automatic recovery option, in case the latest remote snapshot does not exist locally (added MAXAGE parameter). If enabled, $MAXAGE-1 older remote snapshots are searched in the local ones, to act as the new incremental base. This will prevent manteinance mode, but do know that non-matched snapshots newer than the matching one will be destroyed, before being replaced by the even newer ones. Cloning the non-matched remote snapshots is one (arguably unclean) solution for this, but has not been implemented. - Added 'zfs hold' to local snapshot, to ensure retention before send operation. - Removed redundant code (now $RECENT is used programmatically, instead of branching) --- README | 21 ++++++----- example.cfg | 6 ++++ zfs-backup.sh | 96 +++++++++++++++++++++++++++------------------------ 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/README b/README index b058392..7d5610a 100644 --- a/README +++ b/README @@ -29,8 +29,8 @@ This aims to be much more robust than the backup functionality of zfs-auto-snapshot, namely: * it uses 'zfs send -I' to send all intermediate snapshots (including any daily/weekly/etc.), and should still work even if it isn't run - every hour -- as long as the newest remote snapshot hasn't been - rotated out locally yet + every hour -- as long as there is at least one remote snapshot that has + not been rotated out locally yet * 'zfs recv -dF' on the destination host removes any snapshots not present locally so you don't have to worry about manually removing old snapshots there. @@ -80,8 +80,10 @@ Given the hierarchy pool/a/b, into top-level child filesystems, and not touch any unknown filesystems. If this backup is not run for a long enough period that the newest -remote snapshot has been removed locally, manually run an incremental -zfs send/recv to bring it up to date, a la +remote snapshot has been removed locally, then MAXAGE-1 older snapshots +are tried instead, until a matching snapshot is found. You can choose +to disable this behaviour using MAXAGE=1 and intervene manually instead, +using a zfs send/recv to bring it up to date, a la zfs send -I zfs-auto-snap_daily-(latest on remote) -R $POOL/$FS@zfs-auto-snap_daily-(latest local) | ssh $REMUSER@REMHOST zfs recv -dvF $REMPOOL It's probably best to do a dry-run first (zfs recv -ndvF). @@ -99,8 +101,9 @@ of your sync. This is a good use case for an alternate configuration file. PROCEDURE: - * find newest local hourly snapshot - * find newest remote hourly snapshot (via ssh) - * check that both $newest_local and $latest_remote snaps exist locally - * zfs send incremental (-I) from $newest_remote to $latest_local to dsthost - * if anything fails, set svc to maint. and exit +* find newest local hourly snapshot +* find newest remote hourly snapshot (via ssh) +* check that both $newest_local and $latest_remote snaps exist locally +* if $latest_remote does not exist locally, try up to $MAXAGE-1 older remotes +* zfs send incremental (-I) from matching remote to newest local to dsthost +* if anything fails, set svc to maint. and exit diff --git a/example.cfg b/example.cfg index 3ca2757..672507f 100644 --- a/example.cfg +++ b/example.cfg @@ -13,6 +13,12 @@ # - VERBOSE: set to "-v" for verbose mode. # - LOCK: file that is created in case zfs-backup ever has problems, and enters manteinance mode. No need to change this. # - PID: pid file for zfs-backup. No need to change this. +# - MAXAGE: if the latest remote snapshot is not found locally, to base the increment on, then up to MAXAGE-1 older snapshot are +# tried instead. Set to 0 to try indefinitely. Set to 1 (default) to skip this behaviour. +# WARNING: this parameter allows automatic recovery in case of local backup rotation, but be aware that when zfs tries to +# receive from a snapshot other than the latest, the newer snapshots are DESTROYED, and then replaced by the new history. +# You will lose the snapshot on the backup, although newer ones will be created instead. The default value of 1 prevents +# this behaviour, requiring manual intervention (e.g. cloning of the remote recent snapshots). TAG="zfs-auto-snap_hourly" LOCK="/var/tmp/zfsbackup.lock" diff --git a/zfs-backup.sh b/zfs-backup.sh index c81dad1..75a3398 100755 --- a/zfs-backup.sh +++ b/zfs-backup.sh @@ -45,8 +45,12 @@ # Consult the README file for details. # If this backup is not run for a long enough period that the newest -# remote snapshot has been removed locally, manually run an incremental -# zfs send/recv to bring it up to date, a la +# remote snapshot has been removed locally, you can choose to +# automatically allow older remote snapshots to be matched against +# the local ones by setting a MAXAGE value other than 1. Be aware that +# this will destroy the snapshots newer than the matched one on the +# remote, before replacing them with newer ones. You might prefer a +# manual zfs send/recv to bring it up to date, a la # zfs send -I zfs-auto-snap_daily-(latest on remote) -R \ # $POOL/$FS@zfs-auto-snap_daily-(latest local) | \ # ssh $REMUSER@REMHOST zfs recv -dvF $REMPOOL @@ -57,7 +61,8 @@ # * find newest local hourly snapshot # * find newest remote hourly snapshot (via ssh) # * check that both $newest_local and $latest_remote snaps exist locally -# * zfs send incremental (-I) from $newest_remote to $latest_local to dsthost +# * if $latest_remote does not exist locally, try up to $MAXAGE-1 older remotes +# * zfs send incremental (-I) from matching remote to newest local to dsthost # * if anything fails, set svc to maint. and exit # all of the following variables (except CFG) may be set in the config file @@ -78,6 +83,7 @@ REMUSER="zfsbak" REMHOST="backupserver.my.domain" REMPOOL="backuppool" REMZFS="$ZFS" +MAXAGE=1 usage() { @@ -136,6 +142,7 @@ fi [ "$recent_flag" ] && RECENT=$recent_flag # set default value so integer tests work if [ -z "$RECENT" ]; then RECENT=0; fi +if [ $RECENT -lt 1 ]; then RECENT=1; fi # local (non-ssh) backup handling: REMHOST=localhost if [ "$REMHOST" = "localhost" ]; then REMZFS_CMD="$ZFS" @@ -173,53 +180,43 @@ do_backup() { return 2 fi - if [ $RECENT -gt 1 ]; then - newest_local="$($ZFS list -t snapshot -H -S creation -o name -d 1 $DATASET | grep $TAG | awk NR==$RECENT)" - if [ -z "$newest_local" ]; then - echo "Error: could not find $(ord $RECENT) most recent snapshot matching tag" >&2 - echo "'$TAG' for ${DATASET}!" >&2 - return 1 - fi - msg="using local snapshot ($(ord $RECENT) most recent):" - else - newest_local="$($ZFS list -t snapshot -H -S creation -o name -d 1 $DATASET | grep $TAG | head -1)" - if [ -z "$newest_local" ]; then - echo "Error: no snapshots matching tag '$TAG' for ${DATASET}!" >&2 - return 1 - fi - msg="newest local snapshot:" - fi - snap2=${newest_local#*@} - [ "$DEBUG" -o "$VERBOSE" ] && echo "$msg $snap2" - + all_locals=$($ZFS list -t snapshot -H -S creation -o name -d 1 $DATASET) all_remotes=$($REMZFS_CMD list -t snapshot -H -S creation -o name -d 1 $TARGET < /dev/null) - newest_remote="$(echo "$all_remotes" | grep $TAG | head -1)" err_msg="Error fetching remote snapshot listing to $REMHOST." - if [ -z $newest_remote ]; then - echo "$err_msg" >&2 - [ $DEBUG ] || touch $LOCK - return 1 + if [ -z "$all_remotes" ]; then + echo "$err_msg" >&2 + [ $DEBUG ] || touch $LOCK + return 1 fi - snap1=${newest_remote#*@} - [ "$DEBUG" -o "$VERBOSE" ] && echo "newest remote snapshot: $snap1" - if ! $ZFS list -t snapshot -H $DATASET@$snap1 > /dev/null 2>&1; then - exec 1>&2 - echo "Newest remote snapshot '$snap1' does not exist locally!" - echo "Perhaps it has been already rotated out." - echo "" - echo "Manually run zfs send/recv to bring $TARGET on $REMHOST" - echo "to a snapshot that exists on this host (newest local snapshot with the" - echo "tag $TAG is $snap2)." - [ $DEBUG ] || touch $LOCK - return 1 + newest_local=$(echo "$all_locals" | grep $TAG | head -n $RECENT | tail -n1) + if [ -z "$newest_local" ]; then + echo "Error: no snapshots matching tag '$TAG' for ${DATASET}!" >&2 + return 1 fi - if ! $ZFS list -t snapshot -H $DATASET@$snap2 > /dev/null 2>&1; then - exec 1>&2 - echo "Something has gone horribly wrong -- local snapshot $snap2" - echo "has suddenly disappeared!" - [ $DEBUG ] || touch $LOCK - return 1 + msg="using local snapshot ($(ord $RECENT) most recent):" + snap2=${newest_local#*@} + [ "$DEBUG" -o "$VERBOSE" ] && echo "$msg $snap2" + + snap1="" + tried_snapshots=0 + for remote in $all_remotes; do + snap1_candidate=${remote#*@} + if (echo "$all_locals" | grep -q $snap1_candidate); then + snap1=$snap1_candidate + break + else + [ "$DEBUG" -o "$VERBOSE" ] && echo "Remote snapshot not found locally: \"$remote\" (probably already rotated out); skipping." + fi + tried_snapshots=$((tried_snapshots+1)) + [ $MAXAGE -gt 0 -a $tried_snapshots -ge $MAXAGE ] && break + done + if [ -z "$snap1" ]; then + echo "Error: no matching snapshot between remote and local found in the last $MAXAGE snapshots." >&2 + echo "This means that too many snapshot have been rotated out in the local, and manual intervention is required." >&2 + echo "Entering manteinance mode." >&2 + touch $LOCK + exit 4 fi if [ "$snap1" = "$snap2" ]; then @@ -240,6 +237,15 @@ do_backup() { echo "would run: $PFEXEC $ZFS send -R -I $snap1 $DATASET@$snap2 |" echo " $REMZFS_CMD recv $VERBOSE $RECV_OPT -F $REMPOOL" else + if ! $ZFS hold $TAG $DATASET@$snap2; then + echo "Error: \"$snap1\" was rotated out while this program was running." >&2 + echo "You should wait for the rotation script to be finished," >&2 + echo "before running this script. Entering mantainance mode." >&2 + touch $LOCK + exit 8 + else + trap "$ZFS release $TAG $DATASET@$snap2" EXIT + fi if ! $PFEXEC $ZFS send -R -I $snap1 $DATASET@$snap2 | \ $REMZFS_CMD recv $VERBOSE $RECV_OPT -F $REMPOOL; then echo 1>&2 "Error sending snapshot." From f9b46e5ea62f5b6ca8d5178b0ce8e2dcdfb1f4eb Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Mon, 8 May 2017 21:01:20 +0200 Subject: [PATCH 3/3] Fix bug that prevented multi-dataset backup - Fix regression in last commit, which executed `zfs hold` for each dataset, but `zfs release` only for the last one. Now it is called right after the send/receive, and then the trap is cleared. - Schedule `release` before even calling the `hold`, to guarantee its execution. --- zfs-backup.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zfs-backup.sh b/zfs-backup.sh index 75a3398..300c548 100755 --- a/zfs-backup.sh +++ b/zfs-backup.sh @@ -237,14 +237,14 @@ do_backup() { echo "would run: $PFEXEC $ZFS send -R -I $snap1 $DATASET@$snap2 |" echo " $REMZFS_CMD recv $VERBOSE $RECV_OPT -F $REMPOOL" else + RELEASE_CMD="$ZFS release $TAG $DATASET@$snap2" + trap "$RELEASE_CMD" EXIT if ! $ZFS hold $TAG $DATASET@$snap2; then echo "Error: \"$snap1\" was rotated out while this program was running." >&2 echo "You should wait for the rotation script to be finished," >&2 echo "before running this script. Entering mantainance mode." >&2 touch $LOCK exit 8 - else - trap "$ZFS release $TAG $DATASET@$snap2" EXIT fi if ! $PFEXEC $ZFS send -R -I $snap1 $DATASET@$snap2 | \ $REMZFS_CMD recv $VERBOSE $RECV_OPT -F $REMPOOL; then @@ -252,6 +252,8 @@ do_backup() { touch $LOCK return 1 fi + $RELEASE_CMD + trap - EXIT fi }