Multiple backup storage backends #235

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 .Python
 .env
 env/
+venv/
 build/
 develop-eggs/
 dist/
@@ -45,6 +46,7 @@ coverage.xml
 *,cover
 tests/media/
 coverage_html_report/
+file:memorydb_default?mode=memory&cache=shared
 
 # Translations
 *.mo

dbbackup/management/commands/dbbackup.py

@@ -8,7 +8,7 @@
 
 from ._base import BaseDbBackupCommand, make_option
 from ...db.base import get_connector
-from ...storage import get_storage, StorageError
+from ...storage import get_storage, get_backup_storage, StorageError
 from ... import utils, settings
 
 
@@ -31,10 +31,12 @@ class Command(BaseDbBackupCommand):
                     help="Encrypt the backup files"),
         make_option("-o", "--output-filename", default=None,
                     help="Specify filename on storage"),
-        make_option("-O", "--output-path", default=None,
-                    help="Specify where to store on local filesystem"),
         make_option("-x", "--exclude-tables", default=None,
-                    help="Exclude tables from backup")
+                    help="Exclude tables from backup"),
+        make_option("-p", "--output-path", default=None,
+                    help="Specify where to store on local filesystem"),
+        make_option("--storage", default=None,
+                    help="Specify storage from DBACKUP_STORAGES to use"),
     )
 
     @utils.email_uncaught_exception
@@ -49,10 +51,16 @@ def handle(self, **options):
         self.compress = options.get('compress')
         self.encrypt = options.get('encrypt')
 
+        self.db_storage = options.get('storage')
+
         self.filename = options.get('output_filename')
         self.path = options.get('output_path')
         self.exclude_tables = options.get("exclude_tables")
-        self.storage = get_storage()
+
+        if self.db_storage:
+            self.storage = get_backup_storage(self.db_storage)
+        else:
+            self.storage = get_storage()
 
         self.database = options.get('database') or ''
         database_keys = self.database.split(',') or settings.DATABASES

dbbackup/management/commands/dbrestore.py

@@ -6,12 +6,13 @@
 
 from django.conf import settings
 from django.core.management.base import CommandError
+
 from django.db import connection
 
 from ._base import BaseDbBackupCommand, make_option
 from ... import utils
 from ...db.base import get_connector
-from ...storage import get_storage, StorageError
+from ...storage import get_storage, get_backup_storage, StorageError
 
 
 class Command(BaseDbBackupCommand):
@@ -30,7 +31,9 @@ class Command(BaseDbBackupCommand):
                     help="Decrypt data before restoring"),
         make_option("-p", "--passphrase", help="Passphrase for decrypt file", default=None),
         make_option("-z", "--uncompress", action='store_true', default=False,
-                    help="Uncompress gzip data before restoring")
+                    help="Uncompress gzip data before restoring"),
+        make_option("--storage", default=None,
+                    help="Specify storage from DBACKUP_STORAGES to use"),
     )
 
     def handle(self, *args, **options):
@@ -48,8 +51,14 @@ def handle(self, *args, **options):
             self.uncompress = options.get('uncompress')
             self.passphrase = options.get('passphrase')
             self.interactive = options.get('interactive')
+            self.db_storage = options.get('storage')
             self.database_name, self.database = self._get_database(options)
-            self.storage = get_storage()
+
+            if self.db_storage:
+                self.storage = get_backup_storage(self.db_storage)
+            else:
+                self.storage = get_storage()
+
             self._restore_backup()
         except StorageError as err:
             raise CommandError(err)
@@ -59,8 +68,8 @@ def _get_database(self, options):
         database_name = options.get('database')
         if not database_name:
             if len(settings.DATABASES) > 1:
-                errmsg = "Because this project contains more than one database, you"\
-                    " must specify the --database option."
+                errmsg = "Because this project contains more than one database, you" \
+                         " must specify the --database option."
                 raise CommandError(errmsg)
             database_name = list(settings.DATABASES.keys())[0]
         if database_name not in settings.DATABASES:

dbbackup/settings.py

@@ -30,6 +30,11 @@
 
 STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'django.core.files.storage.FileSystemStorage')
 STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {})
+STORAGES = getattr(settings, 'DBBACKUP_STORAGES', {
+    'default': {
+        'storage': 'django.core.files.storage.FileSystemStorage'
+    }
+})
 
 CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {})
 

dbbackup/storage.py

@@ -30,6 +30,20 @@ def get_storage(path=None, options=None):
     return Storage(path, **options)
 
 
+def get_backup_storage(db_storage):
+    if db_storage not in settings.STORAGES:
+        raise ImproperlyConfigured('You must specify a storage class using '
+                                   'DBBACKUP_STORAGES settings.')
+    storage_options = settings.STORAGES[db_storage].copy()
+    if 'storage' in storage_options:
+        storage = storage_options.pop('storage', None)
+        options = storage_options
+        return get_storage(path=storage, options=options)
+    print(settings.STORAGES)
+    raise ImproperlyConfigured('You must specify a storage class "storage" using '
+                               'DBBACKUP_STORAGES settings.')
+
+
 class StorageError(Exception):
     pass
 
@@ -44,6 +58,7 @@ class Storage(object):
     list and filter files. It uses a Django storage object for low-level
     operations.
     """
+
     @property
     def logger(self):
         if not hasattr(self, '_logger'):
@@ -61,7 +76,9 @@ def __init__(self, storage_path=None, **options):
         """
         self._storage_path = storage_path or settings.STORAGE
         options = options.copy()
-        options.update(settings.STORAGE_OPTIONS)
+        for option in settings.STORAGE_OPTIONS.keys():
+            if option not in options:
+                options[option] = settings.STORAGE_OPTIONS[option]
         options = dict([(key.lower(), value) for key, value in options.items()])
         self.storageCls = get_storage_class(self._storage_path)
         self.storage = self.storageCls(**options)

dbbackup/tests/commands/test_dbbackup.py

@@ -2,15 +2,22 @@
 Tests for dbbackup command.
 """
 import os
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files.storage import FileSystemStorage
+from six import StringIO
+from django.core.management import execute_from_command_line
+
 from mock import patch
 
 from django.test import TestCase
+from storages.backends.s3boto3 import S3Boto3Storage
 
 from dbbackup.management.commands.dbbackup import Command as DbbackupCommand
 from dbbackup.db.base import get_connector
-from dbbackup.storage import get_storage
+from dbbackup.storage import get_storage, get_backup_storage
 from dbbackup.tests.utils import (TEST_DATABASE, add_public_gpg, clean_gpg_keys,
-                                  DEV_NULL)
+                                  DEV_NULL, FakeStorage)
 
 
 @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test')
@@ -50,6 +57,14 @@ def test_path(self):
         # tearDown
         os.remove(self.command.path)
 
+    def test_fallback(self):
+        stdout = StringIO()
+        with self.assertRaises(ImproperlyConfigured) as ic:
+            with patch('sys.stdout', stdout):
+                execute_from_command_line(['', 'dbbackup', '--storage=s3'])
+        self.assertEqual(str(ic.exception),
+                         'You must specify a storage class using DBBACKUP_STORAGES settings.')
+
 
 @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test')
 @patch('sys.stdout', DEV_NULL)
@@ -73,3 +88,67 @@ def tearDown(self):
     def test_func(self, mock_run_commands, mock_handle_size):
         self.command._save_new_backup(TEST_DATABASE)
         self.assertTrue(mock_run_commands.called)
+
+
+class DbbackupCommandSaveMultipleStorages(TestCase):
+    def setUp(self):
+        self.command = DbbackupCommand()
+        self.command.servername = 'foo-server'
+        self.command.encrypt = False
+        self.command.compress = False
+        self.command.connector = get_connector()
+        self.command.stdout = DEV_NULL
+        self.command.filename = None
+        self.command.path = None
+
+    def test_default_func(self):
+        self.command.database = TEST_DATABASE['NAME']
+        self.command.storage = get_backup_storage('default')
+        self.command._save_new_backup(TEST_DATABASE)
+
+    def test_fake_func(self):
+        self.command.database = TEST_DATABASE['NAME']
+        self.command.storage = get_backup_storage('fake_storage')
+        self.command._save_new_backup(TEST_DATABASE)
+
+    def test_default(self):
+        self.storage = get_backup_storage('default')
+        self.assertIsInstance(self.storage.storage, FileSystemStorage)
+
+
+class DbbackupCommandMultipleStorages(TestCase):
+    def setUp(self):
+        self.command = DbbackupCommand()
+        self.command.stdout = DEV_NULL
+        self.command.uncompress = False
+        self.command.decrypt = False
+        self.command.backup_extension = 'bak'
+        self.command.filename = 'foofile'
+        self.command.database = TEST_DATABASE
+        self.command.passphrase = None
+        self.command.interactive = True
+        self.command.database_name = 'default'
+        self.command.connector = get_connector('default')
+
+    @staticmethod
+    def fake_backup():
+        return True
+
+    def test_default(self):
+        self.command.handle(storage='default', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, FileSystemStorage)
+
+    @patch.object(DbbackupCommand, '_save_new_backup')
+    def test_fake(self, fake_backup):
+        self.command.handle(storage='fake_storage', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, FakeStorage)
+
+    @patch.object(DbbackupCommand, '_save_new_backup')
+    def test_S3(self, fake_backup):
+        self.command.handle(storage='s3_storage', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, S3Boto3Storage)
+        self.assertEqual(vars(self.command.storage.storage)['_constructor_args'][1],
+                         {'access_key': 'my_id',
+                          'secret_key': 'my_secret',
+                          'bucket_name': 'my_bucket_name',
+                          'default_acl': 'private'})

dbbackup/tests/commands/test_dbrestore.py

@@ -1,24 +1,29 @@
 """
 Tests for dbrestore command.
 """
-from mock import patch
-from tempfile import mktemp
 from shutil import copyfileobj
+from tempfile import mktemp
 
-from django.test import TestCase
-from django.core.management.base import CommandError
-from django.core.files import File
 from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
+from django.core.management import execute_from_command_line
+from django.core.management.base import CommandError
+from django.test import TestCase
+from mock import patch
+from six import StringIO
+from storages.backends.s3boto3 import S3Boto3Storage
 
 from dbbackup import utils
 from dbbackup.db.base import get_connector
 from dbbackup.db.mongodb import MongoDumpConnector
 from dbbackup.management.commands.dbrestore import Command as DbrestoreCommand
-from dbbackup.storage import get_storage
 from dbbackup.settings import HOSTNAME
+from dbbackup.storage import get_storage
 from dbbackup.tests.utils import (TEST_DATABASE, add_private_gpg, DEV_NULL,
                                   clean_gpg_keys, HANDLED_FILES, TEST_MONGODB, TARED_FILE,
-                                  get_dump, get_dump_name)
+                                  get_dump, get_dump_name, FakeStorage)
 
 
 @patch('dbbackup.management.commands._base.input', return_value='y')
@@ -90,6 +95,16 @@ def test_path(self, *args):
         )
         self.command._restore_backup()
 
+    def test_fallback(self, *args):
+        stdout = StringIO()
+        with self.assertRaises(ImproperlyConfigured) as ic:
+            with patch('sys.stdout', stdout):
+                execute_from_command_line(['', 'dbrestore', '--storage=s3'])
+        self.assertEqual(str(ic.exception),
+                         'You must specify a storage class using DBBACKUP_STORAGES settings.')
+
+        # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful restore.
+
 
 class DbrestoreCommandGetDatabaseTest(TestCase):
     def setUp(self):
@@ -139,3 +154,43 @@ def test_mongo_settings_backup_command(self, mock_runcommands, *args):
         HANDLED_FILES['written_files'].append((TARED_FILE, open(TARED_FILE, 'rb')))
         self.command._restore_backup()
         self.assertTrue(mock_runcommands.called)
+
+
+class DbrestoreCommandRestoreMultipleBackupTest(TestCase):
+    def setUp(self):
+        self.command = DbrestoreCommand()
+        self.command.stdout = DEV_NULL
+        self.command.uncompress = False
+        self.command.decrypt = False
+        self.command.backup_extension = 'bak'
+        self.command.filename = 'foofile'
+        self.command.database = TEST_DATABASE
+        self.command.passphrase = None
+        self.command.interactive = True
+        self.command.servername = HOSTNAME
+        self.command.database_name = 'default'
+        self.command.connector = get_connector('default')
+        HANDLED_FILES.clean()
+
+    @staticmethod
+    def fake_restore():
+        return True
+
+    def test_default(self):
+        self.command.handle(storage='default', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, FileSystemStorage)
+
+    @patch.object(DbrestoreCommand, '_restore_backup')
+    def test_fake(self, fake_restore):
+        self.command.handle(storage='fake_storage', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, FakeStorage)
+
+    @patch.object(DbrestoreCommand, '_restore_backup')
+    def test_S3(self, fake_restore):
+        self.command.handle(storage='s3_storage', verbosity=1)
+        self.assertIsInstance(self.command.storage.storage, S3Boto3Storage)
+        self.assertEqual(vars(self.command.storage.storage)['_constructor_args'][1],
+                         {'access_key': 'my_id',
+                          'secret_key': 'my_secret',
+                          'bucket_name': 'my_bucket_name',
+                          'default_acl': 'private'})