From bf03e48f535106ba37eb82a39e0b3666d53e1df1 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Mon, 6 Apr 2015 18:08:10 +0800 Subject: [PATCH 01/17] Get django-backup to work purely on private keys --- django_backup/management/commands/backup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index a078e28..21fec3a 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -181,6 +181,8 @@ def handle(self, *args, **options): self.ftp_server = getattr(settings, 'BACKUP_FTP_SERVER', '') self.ftp_username = getattr(settings, 'BACKUP_FTP_USERNAME', '') self.ftp_password = getattr(settings, 'BACKUP_FTP_PASSWORD', '') + self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) + if self.clean_rsync: print 'cleaning broken rsync backups' @@ -279,6 +281,8 @@ def get_connection(self): ''' get the ssh connection to the remote server. ''' + if self.private_key: + return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, self.private_key) return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) def get_blacklist_tables(self): From e0045e195c7d17d51c447bb055b48b5c6c6ae5fe Mon Sep 17 00:00:00 2001 From: fruitschen Date: Mon, 6 Apr 2015 18:45:32 +0800 Subject: [PATCH 02/17] bug fix --- django_backup/management/commands/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 21fec3a..b2704bb 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -282,7 +282,7 @@ def get_connection(self): get the ssh connection to the remote server. ''' if self.private_key: - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, self.private_key) + return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) def get_blacklist_tables(self): From 2fc352a6ede4aa2ccf2162634cdda227b0111279 Mon Sep 17 00:00:00 2001 From: Nick Ward Date: Mon, 27 Apr 2015 14:22:42 +0300 Subject: [PATCH 03/17] Option to backup certain apps only --- django_backup/management/commands/backup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index a078e28..235f969 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -137,6 +137,8 @@ class Command(BaseCommand): help='Clean up local broken rsync backups'), make_option('--cleanremotersync', action='store_true', default=False, dest='clean_remote_rsync', help='Clean up remote broken rsync backups'), + make_option('--application', '-a', action='append', default=[], dest='apps', + help='Optionally only back up certain Djano apps'), ) help = "Backup database. Only Mysql and Postgresql engines are implemented" @@ -160,6 +162,7 @@ def handle(self, *args, **options): self.clean_remote_rsync = options.get('clean_remote_rsync') and self.rsync #only when rsync is True self.no_local = options.get('no_local') self.delete_local = options.get('delete_local') + self.apps = options.get('apps') try: self.engine = settings.DATABASES['default']['ENGINE'] @@ -287,6 +290,13 @@ def get_blacklist_tables(self): ''' return getattr(settings, 'BACKUP_TABLES_BLACKLIST', []) + def get_tables_for_apps(self, *apps): + '''Get table names for all for the given applications.''' + tables = connection.introspection.django_table_names(only_existing=True) + def check_table(table): + return any(table.startswith('%s_' %app) for app in apps) + return filter(check_table, tables) + def store_ftp(self, local_files=[]): sftp = self.get_connection() if self.remote_dir: @@ -344,6 +354,9 @@ def do_compress(self, infile, outfile): os.system('rm %s' % infile) def do_mysql_backup(self, outfile): + + if self.apps: + raise NotImplementedError("Backuping up only ceratain apps not implemented in MySQL") args = [] if self.user: args += ["--user='%s'" % self.user] @@ -383,7 +396,10 @@ def do_postgresql_backup(self, outfile): if self.passwd: os.environ['PGPASSWORD'] = self.passwd - pgdump_cmd = '%s %s --clean > %s' % (pgdump_path, ' '.join(args), outfile) + table_args = ' '.join( + '-t %s '%table for table in self.get_tables_for_apps(*self.apps) + ) + pgdump_cmd = '%s %s --clean %s > %s' % (pgdump_path, ' '.join(args), table_args, outfile) print pgdump_cmd os.system(pgdump_cmd) From adaec9b266a3c5f1450d8c82eae65ff75c2f56c9 Mon Sep 17 00:00:00 2001 From: Nick Ward Date: Mon, 27 Apr 2015 15:02:19 +0300 Subject: [PATCH 04/17] Don't dump schema when only certain apps --- django_backup/management/commands/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 391b703..e79e1a8 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -403,6 +403,8 @@ def do_postgresql_backup(self, outfile): table_args = ' '.join( '-t %s '%table for table in self.get_tables_for_apps(*self.apps) ) + if table_args: + table_args = '-d %s' %table_args pgdump_cmd = '%s %s --clean %s > %s' % (pgdump_path, ' '.join(args), table_args, outfile) print pgdump_cmd os.system(pgdump_cmd) From 9e525738ae367c5e11762b01410e8959d04629ae Mon Sep 17 00:00:00 2001 From: Nick Ward Date: Mon, 27 Apr 2015 15:19:22 +0300 Subject: [PATCH 05/17] Typo --- django_backup/management/commands/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index e79e1a8..2b729b2 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -404,7 +404,7 @@ def do_postgresql_backup(self, outfile): '-t %s '%table for table in self.get_tables_for_apps(*self.apps) ) if table_args: - table_args = '-d %s' %table_args + table_args = '-a %s' %table_args pgdump_cmd = '%s %s --clean %s > %s' % (pgdump_path, ' '.join(args), table_args, outfile) print pgdump_cmd os.system(pgdump_cmd) From cc990aaddb3e5eca92e8ef7aa04ed43d1437314a Mon Sep 17 00:00:00 2001 From: Nick Ward Date: Mon, 27 Apr 2015 15:24:51 +0300 Subject: [PATCH 06/17] --clean and -a cannot be used together --- django_backup/management/commands/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 2b729b2..def5e57 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -405,7 +405,7 @@ def do_postgresql_backup(self, outfile): ) if table_args: table_args = '-a %s' %table_args - pgdump_cmd = '%s %s --clean %s > %s' % (pgdump_path, ' '.join(args), table_args, outfile) + pgdump_cmd = '%s %s %s > %s' % (pgdump_path, ' '.join(args), table_args or '--clean', outfile) print pgdump_cmd os.system(pgdump_cmd) From 82516349727247fab49b4eaf07838650edcc8034 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Thu, 30 Apr 2015 22:58:24 +0800 Subject: [PATCH 07/17] add DIRECTORY_TO_BACKUP setting to allow us change files backup directory --- django_backup/management/commands/backup.py | 4 ++-- django_backup/management/commands/restore.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index def5e57..9af48d5 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -185,7 +185,7 @@ def handle(self, *args, **options): self.ftp_username = getattr(settings, 'BACKUP_FTP_USERNAME', '') self.ftp_password = getattr(settings, 'BACKUP_FTP_PASSWORD', '') self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) - + self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) if self.clean_rsync: print 'cleaning broken rsync backups' @@ -248,7 +248,7 @@ def handle(self, *args, **options): # Backing up media directories, if self.media: - self.directories += [settings.MEDIA_ROOT] + self.directories += self.directory_to_backup # Backing up directories dir_outfiles = [] diff --git a/django_backup/management/commands/restore.py b/django_backup/management/commands/restore.py index 8668e3f..ed0cd86 100644 --- a/django_backup/management/commands/restore.py +++ b/django_backup/management/commands/restore.py @@ -45,6 +45,7 @@ def handle(self, *args, **options): self.ftp_username = settings.BACKUP_FTP_USERNAME self.ftp_password = settings.BACKUP_FTP_PASSWORD self.restore_media = options.get('media') + self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) print 'Connecting to %s...' % self.ftp_server sftp = self.get_connection() @@ -82,7 +83,7 @@ def handle(self, *args, **options): media_dir = os.path.join(media_remote_full_path, "media") #A trailing slash to transfer only the contents of the folder remote_rsync = '%s@%s:%s/' % (self.ftp_username, self.ftp_server, media_dir) - rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, settings.MEDIA_ROOT) + rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, self.directory_to_backup) print 'Running rsync restore command: ', rsync_restore_cmd os.system(rsync_restore_cmd) else: @@ -112,7 +113,7 @@ def uncompress(self, file): return os.system(cmd) def uncompress_media(self, file): - cmd = u'tar -C %s -xzf %s' % (settings.MEDIA_ROOT, file) + cmd = u'tar -C %s -xzf %s' % (self.directory_to_backup, file) print u'\t', cmd os.system(cmd) From 0638feedd342b7c0a3462a22e8b78e0b432e1df9 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Thu, 30 Apr 2015 23:15:59 +0800 Subject: [PATCH 08/17] bug fix --- django_backup/management/commands/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 9af48d5..50f2132 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -248,7 +248,7 @@ def handle(self, *args, **options): # Backing up media directories, if self.media: - self.directories += self.directory_to_backup + self.directories += [self.directory_to_backup] # Backing up directories dir_outfiles = [] From 4d5cbc7dddc32778618e3f28cbf875adbef00607 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Thu, 30 Apr 2015 23:40:03 +0800 Subject: [PATCH 09/17] diretory rsync backup should follow symlink --- django_backup/management/commands/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 50f2132..0e50e7a 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -518,7 +518,7 @@ def do_media_rsync_backup(self): 'local_backup_target': local_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - local_rsync_cmd = 'rsync -az --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info + local_rsync_cmd = 'rsync -az --copy-dirlinks --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info local_mark_cmd = 'touch %(local_backup_target)s/%(rsync_flag)s' % local_info local_link_cmd = 'rm -f %(local_current_backup)s && ln -s %(local_backup_target)s %(local_current_backup)s' % local_info cmd = '\n'.join(['%s&&%s' % (local_rsync_cmd, local_mark_cmd), local_link_cmd]) @@ -538,7 +538,7 @@ def do_media_rsync_backup(self): 'remote_backup_target': remote_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - remote_rsync_cmd = 'rsync -az --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info + remote_rsync_cmd = 'rsync -az --copy-dirlinks --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info remote_mark_cmd = 'ssh %(host)s "touch %(remote_backup_target)s/%(rsync_flag)s"' % remote_info remote_link_cmd = 'ssh %(host)s "rm -f %(remote_current_backup)s && ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info cmd = '\n'.join(['%s&&%s' % (remote_rsync_cmd, remote_mark_cmd), remote_link_cmd]) From 9f36821d4e58932e2234fd86141ecb73b26d67bc Mon Sep 17 00:00:00 2001 From: fruitschen Date: Fri, 1 May 2015 01:20:11 +0800 Subject: [PATCH 10/17] reuse ssh connections, and add hourly backup settings --- django_backup/management/commands/backup.py | 25 ++++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 0e50e7a..d541aa2 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -74,6 +74,10 @@ def reserve_interval(backups, type, num): delta = timedelta(1) interval_end = datetime(now.year, now.month, now.day) + delta interval_start = interval_end - delta + elif type == 'hourly': + delta = timedelta(minutes=60) + interval_end = datetime(now.year, now.month, now.day, now.hour) + delta + interval_start = interval_end - delta for i in range(1, num + 1): for backup in backups: if between_interval(backup, interval_start, interval_end): @@ -93,6 +97,8 @@ def decide_remove(backups, config): reserve += reserve_interval(backups, 'monthly', config['monthly']) reserve += reserve_interval(backups, 'weekly', config['weekly']) reserve += reserve_interval(backups, 'daily', config['daily']) + if config.get('hourly', 0): + reserve += reserve_interval(backups, 'hourly', config.get('hourly', 0)) for i in backups: if i not in reserve: remove_list.append(i) @@ -273,6 +279,9 @@ def handle(self, *args, **options): print "Saving to remote server" self.store_ftp(local_files=[os.path.join(os.getcwd(), x) for x in dir_outfiles + [outfile]]) + print 'Closing connection' + self.close_connection() + def compress_dir(self, directory, outfile): print 'Backup directories ...' command = 'cd %s && tar -czf %s *' % (directory, outfile) @@ -284,9 +293,17 @@ def get_connection(self): ''' get the ssh connection to the remote server. ''' + if getattr(self, '_ssh', None): + return self._ssh if self.private_key: - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) + else: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + return self._ssh + + def close_connection(self): + if getattr(self, '_ssh', None): + self._ssh.close() def get_blacklist_tables(self): ''' @@ -312,7 +329,6 @@ def store_ftp(self, local_files=[]): filename = os.path.split(local_file)[-1] print 'Saving %s to remote server ' % local_file sftp.put(local_file, os.path.join(self.remote_dir or '', filename)) - sftp.close() if self.delete_local: backups = os.listdir(self.backup_dir) backups = filter(is_backup, backups) @@ -449,7 +465,6 @@ def clean_remote_surplus_db(self): print '=' * 70 print 'Running Command on remote server: %s' % command sftp.execute(command) - sftp.close() except ImportError: print 'cleaned nothing, because BACKUP_DATABASE_COPIES is missing' @@ -502,7 +517,6 @@ def clean_remote_surplus_media(self): print '=' * 70 print 'Running Command on remote server: %s' % command sftp.execute(command) - sftp.close() except ImportError: print 'cleaned nothing, because BACKUP_MEDIA_COPIES is missing' @@ -570,7 +584,6 @@ def clean_remote_broken_rsync(self): full_cmd = '\n'.join(commands) print full_cmd sftp.execute(full_cmd) - sftp.close() def clean_local_broken_rsync(self): # local(web server) From 2419b3f4b17e155e3602f2e269f4394300b0d147 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Tue, 19 May 2015 14:23:43 +0800 Subject: [PATCH 11/17] remove one backup for one interval constraint --- django_backup/management/commands/backup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index d541aa2..45517b3 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -82,7 +82,9 @@ def reserve_interval(backups, type, num): for backup in backups: if between_interval(backup, interval_start, interval_end): result.append(backup) - break # reserve only one backup per interval + # we need to remove the constraint below or sometimes we remove the 'current' rsync backup, which would cause + # rsync backup to be a lot slower than it needed to be. + # break # reserve only one backup per interval interval_end = interval_end - delta interval_start = interval_start - delta return result From 7fe5aca3f77d7faeaef4825acc7eec871703d2e4 Mon Sep 17 00:00:00 2001 From: fruitschen Date: Fri, 7 Aug 2015 17:09:48 +0800 Subject: [PATCH 12/17] allow us to disable symlink when doing rsync backup --- django_backup/management/commands/backup.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 45517b3..b215d35 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -194,6 +194,7 @@ def handle(self, *args, **options): self.ftp_password = getattr(settings, 'BACKUP_FTP_PASSWORD', '') self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) + self.rsyncnosymlink = getattr(settings, 'BACKUP_DISABLE_RSYNC_SYMLINK', False) # Disable symlink directories when doing rsync backup if self.clean_rsync: print 'cleaning broken rsync backups' @@ -523,6 +524,10 @@ def clean_remote_surplus_media(self): print 'cleaned nothing, because BACKUP_MEDIA_COPIES is missing' def do_media_rsync_backup(self): + if self.rsyncnosymlink: + rsync_options = '-rlptgoDz' + else: + rsync_options = '-az --copy-dirlinks' #local media rsync backup if not self.delete_local and not self.no_local: print 'Doing local media rsync backup' @@ -534,7 +539,10 @@ def do_media_rsync_backup(self): 'local_backup_target': local_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - local_rsync_cmd = 'rsync -az --copy-dirlinks --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info + # We used to use -az, which equals to -rlptgoD and -z. + # Now we need more control over this, using -rptgoD instead of -a to disable symlink backup. + local_info.update({'rsync_options': rsync_options, }) + local_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info local_mark_cmd = 'touch %(local_backup_target)s/%(rsync_flag)s' % local_info local_link_cmd = 'rm -f %(local_current_backup)s && ln -s %(local_backup_target)s %(local_current_backup)s' % local_info cmd = '\n'.join(['%s&&%s' % (local_rsync_cmd, local_mark_cmd), local_link_cmd]) @@ -554,7 +562,8 @@ def do_media_rsync_backup(self): 'remote_backup_target': remote_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - remote_rsync_cmd = 'rsync -az --copy-dirlinks --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info + remote_info.update({'rsync_options': rsync_options, }) + remote_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info remote_mark_cmd = 'ssh %(host)s "touch %(remote_backup_target)s/%(rsync_flag)s"' % remote_info remote_link_cmd = 'ssh %(host)s "rm -f %(remote_current_backup)s && ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info cmd = '\n'.join(['%s&&%s' % (remote_rsync_cmd, remote_mark_cmd), remote_link_cmd]) From a0d33df62116084f747bb81c8f644747f7c60ea0 Mon Sep 17 00:00:00 2001 From: Chen Zhe Date: Tue, 15 Sep 2015 14:47:06 +0800 Subject: [PATCH 13/17] update restore command to allow using private key --- django_backup/management/commands/restore.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/django_backup/management/commands/restore.py b/django_backup/management/commands/restore.py index ed0cd86..72bfd19 100644 --- a/django_backup/management/commands/restore.py +++ b/django_backup/management/commands/restore.py @@ -44,6 +44,7 @@ def handle(self, *args, **options): self.ftp_server = settings.BACKUP_FTP_SERVER self.ftp_username = settings.BACKUP_FTP_USERNAME self.ftp_password = settings.BACKUP_FTP_PASSWORD + self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) self.restore_media = options.get('media') self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) @@ -105,7 +106,13 @@ def get_connection(self): ''' get the ssh connection to the remote server. ''' - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + if getattr(self, '_ssh', None): + return self._ssh + if self.private_key: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) + else: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + return self._ssh def uncompress(self, file): cmd = 'cd %s;gzip -df %s' % (self.tempdir, file) From 3c8724e7b294a30c1d2ac59e2b8e8167eb4a53a1 Mon Sep 17 00:00:00 2001 From: Chen Zhe Date: Thu, 7 Jan 2016 23:34:03 +0800 Subject: [PATCH 14/17] Revert "remove one backup for one interval constraint" This reverts commit 2419b3f4b17e155e3602f2e269f4394300b0d147. --- django_backup/management/commands/backup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index b215d35..11c348d 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -82,9 +82,7 @@ def reserve_interval(backups, type, num): for backup in backups: if between_interval(backup, interval_start, interval_end): result.append(backup) - # we need to remove the constraint below or sometimes we remove the 'current' rsync backup, which would cause - # rsync backup to be a lot slower than it needed to be. - # break # reserve only one backup per interval + break # reserve only one backup per interval interval_end = interval_end - delta interval_start = interval_start - delta return result From 78d1b438e0fb1139a7306329c72ba42cd98566af Mon Sep 17 00:00:00 2001 From: Chen Zhe Date: Tue, 19 Jan 2016 22:58:07 +0800 Subject: [PATCH 15/17] convert remote dir to absolute path if necessary --- django_backup/management/commands/backup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 11c348d..657a367 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -193,6 +193,9 @@ def handle(self, *args, **options): self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) self.rsyncnosymlink = getattr(settings, 'BACKUP_DISABLE_RSYNC_SYMLINK', False) # Disable symlink directories when doing rsync backup + # convert remote dir to absolute path if necessary + if not self.remote_dir.startswith('/') and self.ftp_username: + self.remote_dir = '/home/%s/%s' % (self.ftp_username, self.remote_dir) if self.clean_rsync: print 'cleaning broken rsync backups' From 61f985bf8847778a6ba58126cdae6aa996d3ea48 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Thu, 21 Jan 2016 10:31:18 +0000 Subject: [PATCH 16/17] Make it clear this is not the up to date version --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 1ed556a..1896775 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,7 @@ + +LEGACY VERSION - Please use https://github.com/django-backup/django-backup +======================================== + django-backup ============= http://github.com/andybak/django-backup From 88c7f74ffa441033a50427f4d73e8173f04d82fc Mon Sep 17 00:00:00 2001 From: Chen Zhe Date: Fri, 2 Sep 2016 14:48:14 +0800 Subject: [PATCH 17/17] while cleaning rsync backups, make sure the current symlink is valid --- django_backup/management/commands/backup.py | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index 657a367..79839db 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -545,7 +545,7 @@ def do_media_rsync_backup(self): local_info.update({'rsync_options': rsync_options, }) local_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info local_mark_cmd = 'touch %(local_backup_target)s/%(rsync_flag)s' % local_info - local_link_cmd = 'rm -f %(local_current_backup)s && ln -s %(local_backup_target)s %(local_current_backup)s' % local_info + local_link_cmd = 'rm %(local_current_backup)s; ln -s %(local_backup_target)s %(local_current_backup)s' % local_info cmd = '\n'.join(['%s&&%s' % (local_rsync_cmd, local_mark_cmd), local_link_cmd]) print cmd os.system(cmd) @@ -566,7 +566,7 @@ def do_media_rsync_backup(self): remote_info.update({'rsync_options': rsync_options, }) remote_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info remote_mark_cmd = 'ssh %(host)s "touch %(remote_backup_target)s/%(rsync_flag)s"' % remote_info - remote_link_cmd = 'ssh %(host)s "rm -f %(remote_current_backup)s && ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info + remote_link_cmd = 'ssh %(host)s "rm %(remote_current_backup)s; ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info cmd = '\n'.join(['%s&&%s' % (remote_rsync_cmd, remote_mark_cmd), remote_link_cmd]) print cmd sftp = self.get_connection() @@ -597,6 +597,19 @@ def clean_remote_broken_rsync(self): print full_cmd sftp.execute(full_cmd) + # after we clean the backups, recreate the "current" symlink. + backups = [i.strip() for i in sftp.execute('ls %s' % self.remote_dir)] + backups = list(filter(is_media_backup, backups)) + backups.sort() + if backups: + host = '%s@%s' % (self.ftp_username, self.ftp_server) + current_link = os.path.join(self.remote_dir, 'current') + latest_backup = os.path.join(self.remote_dir, backups[-1]) + remote_info = dict(current_link=current_link, latest_backup=latest_backup, host=host) + remote_link_cmd = 'ssh %(host)s "rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s"' % remote_info + print remote_link_cmd + os.system(remote_link_cmd) + def clean_local_broken_rsync(self): # local(web server) backups = os.listdir(self.backup_dir) @@ -612,3 +625,16 @@ def clean_local_broken_rsync(self): full_cmd = '\n'.join(commands) print full_cmd os.system(full_cmd) + + # after we clean the backups, recreate the "current" symlink. + backups = os.listdir(self.backup_dir) + backups = list(filter(is_media_backup, backups)) + backups.sort() + if backups: + current_link = os.path.join(self.backup_dir, 'current') + latest_backup = os.path.join(self.backup_dir, backups[-1]) + info = dict(current_link=current_link, latest_backup=latest_backup) + link_cmd = 'rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s' % info + print link_cmd + os.system(link_cmd) +