Get django-backup to work purely on private keys

README.rst

@@ -1,3 +1,7 @@
+
+LEGACY VERSION - Please use https://github.com/django-backup/django-backup
+========================================
+
 django-backup
 =============
 http://github.com/andybak/django-backup

django_backup/management/commands/backup.py

@@ -74,6 +74,10 @@ def reserve_interval(backups, type, num):
         delta = timedelta(1)
         interval_end = datetime(now.year, now.month, now.day) + delta
         interval_start = interval_end - delta
+    elif type == 'hourly':
+        delta = timedelta(minutes=60)
+        interval_end = datetime(now.year, now.month, now.day, now.hour) + delta
+        interval_start = interval_end - delta
     for i in range(1, num + 1):
         for backup in backups:
             if between_interval(backup, interval_start, interval_end):
@@ -93,6 +97,8 @@ def decide_remove(backups, config):
     reserve += reserve_interval(backups, 'monthly', config['monthly'])
     reserve += reserve_interval(backups, 'weekly', config['weekly'])
     reserve += reserve_interval(backups, 'daily', config['daily'])
+    if config.get('hourly', 0):
+        reserve += reserve_interval(backups, 'hourly', config.get('hourly', 0))
     for i in backups:
         if i not in reserve:
             remove_list.append(i)
@@ -137,6 +143,8 @@ class Command(BaseCommand):
             help='Clean up local broken rsync backups'),
         make_option('--cleanremotersync', action='store_true', default=False, dest='clean_remote_rsync',
             help='Clean up remote broken rsync backups'),
+        make_option('--application', '-a', action='append', default=[], dest='apps',
+            help='Optionally only back up certain Djano apps'),
     )
     help = "Backup database. Only Mysql and Postgresql engines are implemented"
 
@@ -160,6 +168,7 @@ def handle(self, *args, **options):
         self.clean_remote_rsync = options.get('clean_remote_rsync') and self.rsync #only when rsync is True
         self.no_local = options.get('no_local')
         self.delete_local = options.get('delete_local')
+        self.apps = options.get('apps')
 
         try:
             self.engine = settings.DATABASES['default']['ENGINE']
@@ -181,6 +190,12 @@ def handle(self, *args, **options):
         self.ftp_server = getattr(settings, 'BACKUP_FTP_SERVER', '')
         self.ftp_username = getattr(settings, 'BACKUP_FTP_USERNAME', '')
         self.ftp_password = getattr(settings, 'BACKUP_FTP_PASSWORD', '')
+        self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None)
+        self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT)
+        self.rsyncnosymlink = getattr(settings, 'BACKUP_DISABLE_RSYNC_SYMLINK', False)  # Disable symlink directories when doing rsync backup
+        # convert remote dir to absolute path if necessary
+        if not self.remote_dir.startswith('/') and self.ftp_username:
+            self.remote_dir = '/home/%s/%s' % (self.ftp_username, self.remote_dir)
 
         if self.clean_rsync:
             print 'cleaning broken rsync backups'
@@ -243,7 +258,7 @@ def handle(self, *args, **options):
 
         # Backing up media directories,
         if self.media:
-            self.directories += [settings.MEDIA_ROOT]
+            self.directories += [self.directory_to_backup]
 
         # Backing up directories
         dir_outfiles = []
@@ -268,6 +283,9 @@ def handle(self, *args, **options):
             print "Saving to remote server"
             self.store_ftp(local_files=[os.path.join(os.getcwd(), x) for x in dir_outfiles + [outfile]])
 
+        print 'Closing connection'
+        self.close_connection()
+
     def compress_dir(self, directory, outfile):
         print 'Backup directories ...'
         command = 'cd %s && tar -czf %s *' % (directory, outfile)
@@ -279,14 +297,31 @@ def get_connection(self):
         '''
         get the ssh connection to the remote server.
         '''
-        return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password)
+        if getattr(self, '_ssh', None):
+            return self._ssh
+        if self.private_key:
+            self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key)
+        else:
+            self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password)
+        return self._ssh
+
+    def close_connection(self):
+        if getattr(self, '_ssh', None):
+            self._ssh.close()
 
     def get_blacklist_tables(self):
         '''
         exclude BACKUP_TABLES_BLACKLIST if it's defined.
         '''
         return getattr(settings, 'BACKUP_TABLES_BLACKLIST', [])
 
+    def get_tables_for_apps(self, *apps):
+        '''Get table names for all for the given applications.'''
+        tables = connection.introspection.django_table_names(only_existing=True)
+        def check_table(table):
+            return any(table.startswith('%s_' %app) for app in apps)
+        return filter(check_table, tables)
+
     def store_ftp(self, local_files=[]):
         sftp = self.get_connection()
         if self.remote_dir:
@@ -298,7 +333,6 @@ def store_ftp(self, local_files=[]):
             filename = os.path.split(local_file)[-1]
             print 'Saving %s to remote server ' % local_file
             sftp.put(local_file, os.path.join(self.remote_dir or '', filename))
-        sftp.close()
         if self.delete_local:
             backups = os.listdir(self.backup_dir)
             backups = filter(is_backup, backups)
@@ -344,6 +378,9 @@ def do_compress(self, infile, outfile):
         os.system('rm %s' % infile)
 
     def do_mysql_backup(self, outfile):
+
+        if self.apps:
+            raise NotImplementedError("Backuping up only ceratain apps not implemented in MySQL")
         args = []
         if self.user:
             args += ["--user='%s'" % self.user]
@@ -383,7 +420,12 @@ def do_postgresql_backup(self, outfile):
 
         if self.passwd:
             os.environ['PGPASSWORD'] = self.passwd
-        pgdump_cmd = '%s %s --clean > %s' % (pgdump_path, ' '.join(args), outfile)
+        table_args = ' '.join(
+            '-t %s '%table for table in self.get_tables_for_apps(*self.apps)
+        )
+        if table_args:
+            table_args = '-a %s' %table_args
+        pgdump_cmd = '%s %s %s > %s' % (pgdump_path, ' '.join(args), table_args or '--clean', outfile)
         print pgdump_cmd
         os.system(pgdump_cmd)
 
@@ -427,7 +469,6 @@ def clean_remote_surplus_db(self):
                 print '=' * 70
                 print 'Running Command on remote server: %s' % command
                 sftp.execute(command)
-            sftp.close()
         except ImportError:
             print 'cleaned nothing, because BACKUP_DATABASE_COPIES is missing'
 
@@ -480,11 +521,14 @@ def clean_remote_surplus_media(self):
                 print '=' * 70
                 print 'Running Command on remote server: %s' % command
                 sftp.execute(command)
-            sftp.close()
         except ImportError:
             print 'cleaned nothing, because BACKUP_MEDIA_COPIES is missing'
 
     def do_media_rsync_backup(self):
+        if self.rsyncnosymlink:
+            rsync_options = '-rlptgoDz'
+        else:
+            rsync_options = '-az --copy-dirlinks'
         #local media rsync backup
         if not self.delete_local and not self.no_local:
             print 'Doing local media rsync backup'
@@ -496,9 +540,12 @@ def do_media_rsync_backup(self):
                 'local_backup_target': local_backup_target,
                 'rsync_flag': GOOD_RSYNC_FLAG,
             }
-            local_rsync_cmd = 'rsync -az --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info
+            # We used to use -az, which equals to -rlptgoD and -z. 
+            # Now we need more control over this, using -rptgoD instead of -a to disable symlink backup. 
+            local_info.update({'rsync_options': rsync_options, })
+            local_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(local_current_backup)s %(all_directories)s %(local_backup_target)s' % local_info
             local_mark_cmd = 'touch %(local_backup_target)s/%(rsync_flag)s' % local_info
-            local_link_cmd = 'rm -f %(local_current_backup)s && ln -s %(local_backup_target)s %(local_current_backup)s' % local_info
+            local_link_cmd = 'rm %(local_current_backup)s; ln -s %(local_backup_target)s %(local_current_backup)s' % local_info
             cmd = '\n'.join(['%s&&%s' % (local_rsync_cmd, local_mark_cmd), local_link_cmd])
             print cmd
             os.system(cmd)
@@ -516,9 +563,10 @@ def do_media_rsync_backup(self):
                 'remote_backup_target': remote_backup_target,
                 'rsync_flag': GOOD_RSYNC_FLAG,
             }
-            remote_rsync_cmd = 'rsync -az --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info
+            remote_info.update({'rsync_options': rsync_options, })
+            remote_rsync_cmd = 'rsync %(rsync_options)s --link-dest=%(remote_current_backup)s %(all_directories)s %(host)s:%(remote_backup_target)s' % remote_info
             remote_mark_cmd = 'ssh %(host)s "touch %(remote_backup_target)s/%(rsync_flag)s"' % remote_info
-            remote_link_cmd = 'ssh %(host)s "rm -f %(remote_current_backup)s && ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info
+            remote_link_cmd = 'ssh %(host)s "rm %(remote_current_backup)s; ln -s %(remote_backup_target)s %(remote_current_backup)s"' % remote_info
             cmd = '\n'.join(['%s&&%s' % (remote_rsync_cmd, remote_mark_cmd), remote_link_cmd])
             print cmd
             sftp = self.get_connection()
@@ -548,7 +596,19 @@ def clean_remote_broken_rsync(self):
         full_cmd = '\n'.join(commands)
         print full_cmd
         sftp.execute(full_cmd)
-        sftp.close()
+
+        # after we clean the backups, recreate the "current" symlink.
+        backups = [i.strip() for i in sftp.execute('ls %s' % self.remote_dir)]
+        backups = list(filter(is_media_backup, backups))
+        backups.sort()
+        if backups:
+            host = '%s@%s' % (self.ftp_username, self.ftp_server)
+            current_link = os.path.join(self.remote_dir, 'current')
+            latest_backup = os.path.join(self.remote_dir, backups[-1])
+            remote_info = dict(current_link=current_link, latest_backup=latest_backup, host=host)
+            remote_link_cmd = 'ssh %(host)s "rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s"' % remote_info
+            print remote_link_cmd
+            os.system(remote_link_cmd)
 
     def clean_local_broken_rsync(self):
         # local(web server)
@@ -565,3 +625,16 @@ def clean_local_broken_rsync(self):
         full_cmd = '\n'.join(commands)
         print full_cmd
         os.system(full_cmd)
+
+        # after we clean the backups, recreate the "current" symlink.
+        backups = os.listdir(self.backup_dir)
+        backups = list(filter(is_media_backup, backups))
+        backups.sort()
+        if backups:
+            current_link = os.path.join(self.backup_dir, 'current')
+            latest_backup = os.path.join(self.backup_dir, backups[-1])
+            info = dict(current_link=current_link, latest_backup=latest_backup)
+            link_cmd = 'rm %(current_link)s; ln -s %(latest_backup)s %(current_link)s' % info
+            print link_cmd
+            os.system(link_cmd)
+

django_backup/management/commands/restore.py

@@ -44,7 +44,9 @@ def handle(self, *args, **options):
         self.ftp_server = settings.BACKUP_FTP_SERVER
         self.ftp_username = settings.BACKUP_FTP_USERNAME
         self.ftp_password = settings.BACKUP_FTP_PASSWORD
+        self.private_key = getattr(settings, 'BACKUP_FTP_PRIVATE_KEY', None)
         self.restore_media = options.get('media')
+        self.directory_to_backup = getattr(settings, 'DIRECTORY_TO_BACKUP', settings.MEDIA_ROOT)
 
         print 'Connecting to %s...' % self.ftp_server
         sftp = self.get_connection()
@@ -82,7 +84,7 @@ def handle(self, *args, **options):
                 media_dir = os.path.join(media_remote_full_path, "media")
                 #A trailing slash to transfer only the contents of the folder
                 remote_rsync = '%s@%s:%s/' % (self.ftp_username, self.ftp_server, media_dir)
-                rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, settings.MEDIA_ROOT)
+                rsync_restore_cmd = 'rsync -az %s %s' % (remote_rsync, self.directory_to_backup)
                 print 'Running rsync restore command: ', rsync_restore_cmd
                 os.system(rsync_restore_cmd)
             else:
@@ -104,15 +106,21 @@ def get_connection(self):
         '''
         get the ssh connection to the remote server.
         '''
-        return ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password)
+        if getattr(self, '_ssh', None):
+            return self._ssh
+        if self.private_key:
+            self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=None, private_key=self.private_key)
+        else:
+            self._ssh = ssh.Connection(host=self.ftp_server, username=self.ftp_username, password=self.ftp_password)
+        return self._ssh
 
     def uncompress(self, file):
         cmd = 'cd %s;gzip -df %s' % (self.tempdir, file)
         print '\t', cmd
         return os.system(cmd)
 
     def uncompress_media(self, file):
-        cmd = u'tar -C %s -xzf %s' % (settings.MEDIA_ROOT, file)
+        cmd = u'tar -C %s -xzf %s' % (self.directory_to_backup, file)
         print u'\t', cmd
         os.system(cmd)