diff --git a/README.rst b/README.rst index 1ed556a..1896775 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,7 @@ + +LEGACY VERSION - Please use https://github.com/django-backup/django-backup +======================================== + django-backup ============= http://github.com/andybak/django-backup diff --git a/django_backup/management/commands/backup.py b/django_backup/management/commands/backup.py index a078e28..79839db 100644 --- a/django_backup/management/commands/backup.py +++ b/django_backup/management/commands/backup.py @@ -74,6 +74,10 @@ def reserve_interval(backups, type, num): delta = timedelta(1) interval_end = datetime(now.year, now.month, now.day) + delta interval_start = interval_end - delta + elif type == 'hourly': + delta = timedelta(minutes=60) + interval_end = datetime(now.year, now.month, now.day, now.hour) + delta + interval_start = interval_end - delta for i in range(1, num + 1): for backup in backups: if between_interval(backup, interval_start, interval_end): @@ -93,6 +97,8 @@ def decide_remove(backups, config): reserve += reserve_interval(backups, 'monthly', config['monthly']) reserve += reserve_interval(backups, 'weekly', config['weekly']) reserve += reserve_interval(backups, 'daily', config['daily']) + if config.get('hourly', 0): + reserve += reserve_interval(backups, 'hourly', config.get('hourly', 0)) for i in backups: if i not in reserve: remove_list.append(i) @@ -137,6 +143,8 @@ class Command(BaseCommand): help='Clean up local broken rsync backups'), make_option('--cleanremotersync', action='store_true', default=False, dest='clean_remote_rsync', help='Clean up remote broken rsync backups'), + make_option('--application', '-a', action='append', default=[], dest='apps', + help='Optionally only back up certain Djano apps'), ) help = "Backup database. Only Mysql and Postgresql engines are implemented" @@ -160,6 +168,7 @@ def handle(self, *args, **options): self.clean_remote_rsync = options.get('clean_remote_rsync') and self.rsync #only when rsync is True self.no_local = options.get('no_local') self.delete_local = options.get('delete_local') + self.apps = options.get('apps') try: self.engine = settings.DATABASES['default']['ENGINE'] @@ -181,6 +190,12 @@ def handle(self, *args, **options): self.ftp_server = getattr(settings, 'BACKUP_FTP_SERVER', '') self.ftp_username = getattr(settings, 'BACKUP_FTP_USERNAME', '') self.ftp_password = getattr(settings, 'BACKUP_FTP_PASSWORD', '') + self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) + self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) + self.rsyncnosymlink = getattr(settings, 'BACKUP_DISABLE_RSYNC_SYMLINK', False) # Disable symlink directories when doing rsync backup + # convert remote dir to absolute path if necessary + if not self.remote_dir.startswith('/') and self.ftp_username: + self.remote_dir = '/home/%s/%s' % (self.ftp_username, self.remote_dir) if self.clean_rsync: print 'cleaning broken rsync backups' @@ -243,7 +258,7 @@ def handle(self, *args, **options): # Backing up media directories, if self.media: - self.directories += [settings.MEDIA_ROOT] + self.directories += [self.directory_to_backup] # Backing up directories dir_outfiles = [] @@ -268,6 +283,9 @@ def handle(self, *args, **options): print "Saving to remote server" self.store_ftp(local_files=[os.path.join(os.getcwd(), x) for x in dir_outfiles + [outfile]]) + print 'Closing connection' + self.close_connection() + def compress_dir(self, directory, outfile): print 'Backup directories ...' command = 'cd %s && tar -czf %s *' % (directory, outfile) @@ -279,7 +297,17 @@ def get_connection(self): ''' get the ssh connection to the remote server. ''' - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + if getattr(self, '_ssh', None): + return self._ssh + if self.private_key: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) + else: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + return self._ssh + + def close_connection(self): + if getattr(self, '_ssh', None): + self._ssh.close() def get_blacklist_tables(self): ''' @@ -287,6 +315,13 @@ def get_blacklist_tables(self): ''' return getattr(settings, 'BACKUP_TABLES_BLACKLIST', []) + def get_tables_for_apps(self, *apps): + '''Get table names for all for the given applications.''' + tables = connection.introspection.django_table_names(only_existing=True) + def check_table(table): + return any(table.startswith('%s_' %app) for app in apps) + return filter(check_table, tables) + def store_ftp(self, local_files=[]): sftp = self.get_connection() if self.remote_dir: @@ -298,7 +333,6 @@ def store_ftp(self, local_files=[]): filename = os.path.split(local_file)[-1] print 'Saving %s to remote server ' % local_file sftp.put(local_file, os.path.join(self.remote_dir or '', filename)) - sftp.close() if self.delete_local: backups = os.listdir(self.backup_dir) backups = filter(is_backup, backups) @@ -344,6 +378,9 @@ def do_compress(self, infile, outfile): os.system('rm %s' % infile) def do_mysql_backup(self, outfile): + + if self.apps: + raise NotImplementedError("Backuping up only ceratain apps not implemented in MySQL") args = [] if self.user: args += ["--user='%s'" % self.user] @@ -383,7 +420,12 @@ def do_postgresql_backup(self, outfile): if self.passwd: os.environ['PGPASSWORD'] = self.passwd - pgdump_cmd = '%s %s --clean > %s' % (pgdump_path, ' '.join(args), outfile) + table_args = ' '.join( + '-t %s '%table for table in self.get_tables_for_apps(*self.apps) + ) + if table_args: + table_args = '-a %s' %table_args + pgdump_cmd = '%s %s %s > %s' % (pgdump_path, ' '.join(args), table_args or '--clean', outfile) print pgdump_cmd os.system(pgdump_cmd) @@ -427,7 +469,6 @@ def clean_remote_surplus_db(self): print '=' * 70 print 'Running Command on remote server: %s' % command sftp.execute(command) - sftp.close() except ImportError: print 'cleaned nothing, because BACKUP_DATABASE_COPIES is missing' @@ -480,11 +521,14 @@ def clean_remote_surplus_media(self): print '=' * 70 print 'Running Command on remote server: %s' % command sftp.execute(command) - sftp.close() except ImportError: print 'cleaned nothing, because BACKUP_MEDIA_COPIES is missing' def do_media_rsync_backup(self): + if self.rsyncnosymlink: + rsync_options = '-rlptgoDz' + else: + rsync_options = '-az --copy-dirlinks' #local media rsync backup if not self.delete_local and not self.no_local: print 'Doing local media rsync backup' @@ -496,9 +540,12 @@ def do_media_rsync_backup(self): 'local_backup_target': local_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - local_rsync_cmd = 'rsync -az --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info + # We used to use -az, which equals to -rlptgoD and -z. + # Now we need more control over this, using -rptgoD instead of -a to disable symlink backup. + local_info.update({'rsync_options': rsync_options, }) + local_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info local_mark_cmd = 'touch %(local_backup_target)s/%(rsync_flag)s' % local_info - local_link_cmd = 'rm -f %(local_current_backup)s && ln -s %(local_backup_target)s %(local_current_backup)s' % local_info + local_link_cmd = 'rm %(local_current_backup)s; ln -s %(local_backup_target)s %(local_current_backup)s' % local_info cmd = '\n'.join(['%s&&%s' % (local_rsync_cmd, local_mark_cmd), local_link_cmd]) print cmd os.system(cmd) @@ -516,9 +563,10 @@ def do_media_rsync_backup(self): 'remote_backup_target': remote_backup_target, 'rsync_flag': GOOD_RSYNC_FLAG, } - remote_rsync_cmd = 'rsync -az --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info + remote_info.update({'rsync_options': rsync_options, }) + remote_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info remote_mark_cmd = 'ssh %(host)s "touch %(remote_backup_target)s/%(rsync_flag)s"' % remote_info - remote_link_cmd = 'ssh %(host)s "rm -f %(remote_current_backup)s && ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info + remote_link_cmd = 'ssh %(host)s "rm %(remote_current_backup)s; ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info cmd = '\n'.join(['%s&&%s' % (remote_rsync_cmd, remote_mark_cmd), remote_link_cmd]) print cmd sftp = self.get_connection() @@ -548,7 +596,19 @@ def clean_remote_broken_rsync(self): full_cmd = '\n'.join(commands) print full_cmd sftp.execute(full_cmd) - sftp.close() + + # after we clean the backups, recreate the "current" symlink. + backups = [i.strip() for i in sftp.execute('ls %s' % self.remote_dir)] + backups = list(filter(is_media_backup, backups)) + backups.sort() + if backups: + host = '%s@%s' % (self.ftp_username, self.ftp_server) + current_link = os.path.join(self.remote_dir, 'current') + latest_backup = os.path.join(self.remote_dir, backups[-1]) + remote_info = dict(current_link=current_link, latest_backup=latest_backup, host=host) + remote_link_cmd = 'ssh %(host)s "rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s"' % remote_info + print remote_link_cmd + os.system(remote_link_cmd) def clean_local_broken_rsync(self): # local(web server) @@ -565,3 +625,16 @@ def clean_local_broken_rsync(self): full_cmd = '\n'.join(commands) print full_cmd os.system(full_cmd) + + # after we clean the backups, recreate the "current" symlink. + backups = os.listdir(self.backup_dir) + backups = list(filter(is_media_backup, backups)) + backups.sort() + if backups: + current_link = os.path.join(self.backup_dir, 'current') + latest_backup = os.path.join(self.backup_dir, backups[-1]) + info = dict(current_link=current_link, latest_backup=latest_backup) + link_cmd = 'rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s' % info + print link_cmd + os.system(link_cmd) + diff --git a/django_backup/management/commands/restore.py b/django_backup/management/commands/restore.py index 8668e3f..72bfd19 100644 --- a/django_backup/management/commands/restore.py +++ b/django_backup/management/commands/restore.py @@ -44,7 +44,9 @@ def handle(self, *args, **options): self.ftp_server = settings.BACKUP_FTP_SERVER self.ftp_username = settings.BACKUP_FTP_USERNAME self.ftp_password = settings.BACKUP_FTP_PASSWORD + self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None) self.restore_media = options.get('media') + self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT) print 'Connecting to %s...' % self.ftp_server sftp = self.get_connection() @@ -82,7 +84,7 @@ def handle(self, *args, **options): media_dir = os.path.join(media_remote_full_path, "media") #A trailing slash to transfer only the contents of the folder remote_rsync = '%s@%s:%s/' % (self.ftp_username, self.ftp_server, media_dir) - rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, settings.MEDIA_ROOT) + rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, self.directory_to_backup) print 'Running rsync restore command: ', rsync_restore_cmd os.system(rsync_restore_cmd) else: @@ -104,7 +106,13 @@ def get_connection(self): ''' get the ssh connection to the remote server. ''' - return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + if getattr(self, '_ssh', None): + return self._ssh + if self.private_key: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key) + else: + self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password) + return self._ssh def uncompress(self, file): cmd = 'cd %s;gzip -df %s' % (self.tempdir, file) @@ -112,7 +120,7 @@ def uncompress(self, file): return os.system(cmd) def uncompress_media(self, file): - cmd = u'tar -C %s -xzf %s' % (settings.MEDIA_ROOT, file) + cmd = u'tar -C %s -xzf %s' % (self.directory_to_backup, file) print u'\t', cmd os.system(cmd)