From acf0d1d19b43cb4bdd215235f1267f546603e248 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Fri, 2 Oct 2020 12:17:19 -0400 Subject: [PATCH 01/11] add fallback storage command --- .gitignore | 1 + dbbackup/management/commands/dbbackup.py | 15 ++++++++++++--- dbbackup/management/commands/dbrestore.py | 16 +++++++++++++--- dbbackup/settings.py | 1 + dbbackup/storage.py | 7 +++++++ dbbackup/tests/commands/test_dbbackup.py | 13 +++++++++++++ 6 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 9ba6f875..63573cb3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +venv/ build/ develop-eggs/ dist/ diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index cc85e84a..33ac2179 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -4,11 +4,12 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +from django.core.exceptions import ImproperlyConfigured from django.core.management.base import CommandError from ._base import BaseDbBackupCommand, make_option from ...db.base import get_connector -from ...storage import get_storage, StorageError +from ...storage import get_storage, get_fallback_storage, StorageError from ... import utils, settings @@ -32,7 +33,9 @@ class Command(BaseDbBackupCommand): make_option("-o", "--output-filename", default=None, help="Specify filename on storage"), make_option("-O", "--output-path", default=None, - help="Specify where to store on local filesystem") + help="Specify where to store on local filesystem"), + make_option("-f", "--fallback", action="store_true", default=False, + help="Use alternate (fallback) storage class.") ) @utils.email_uncaught_exception @@ -47,9 +50,15 @@ def handle(self, **options): self.compress = options.get('compress') self.encrypt = options.get('encrypt') + self.fallback = options.get('fallback') + self.filename = options.get('output_filename') self.path = options.get('output_path') - self.storage = get_storage() + + if self.fallback: + self.storage = get_fallback_storage() + else: + self.storage = get_storage() self.database = options.get('database') or '' database_keys = self.database.split(',') or settings.DATABASES diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 4cacfab3..407c969e 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -5,13 +5,15 @@ print_function, unicode_literals) from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.core.management.base import CommandError + from django.db import connection from ._base import BaseDbBackupCommand, make_option from ... import utils from ...db.base import get_connector -from ...storage import get_storage, StorageError +from ...storage import get_storage, get_fallback_storage, StorageError class Command(BaseDbBackupCommand): @@ -30,7 +32,9 @@ class Command(BaseDbBackupCommand): help="Decrypt data before restoring"), make_option("-p", "--passphrase", help="Passphrase for decrypt file", default=None), make_option("-z", "--uncompress", action='store_true', default=False, - help="Uncompress gzip data before restoring") + help="Uncompress gzip data before restoring"), + make_option("-f", "--fallback", action="store_true", default=False, + help="Use alternate (fallback) storage class.") ) def handle(self, *args, **options): @@ -48,8 +52,14 @@ def handle(self, *args, **options): self.uncompress = options.get('uncompress') self.passphrase = options.get('passphrase') self.interactive = options.get('interactive') + self.fallback = options.get('fallback') self.database_name, self.database = self._get_database(options) - self.storage = get_storage() + + if self.fallback: + self.storage = get_fallback_storage() + else: + self.storage = get_storage() + self._restore_backup() except StorageError as err: raise CommandError(err) diff --git a/dbbackup/settings.py b/dbbackup/settings.py index c37468fe..f22374ec 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -32,6 +32,7 @@ STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'django.core.files.storage.FileSystemStorage') STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {}) +FALLBACK_STORAGE = getattr(settings, 'DBBACKUP_FALLBACK_STORAGE', None) CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {}) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index 3a0719cf..ce7fc196 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -30,6 +30,13 @@ def get_storage(path=None, options=None): return Storage(path, **options) +def get_fallback_storage(): + if not settings.FALLBACK_STORAGE: + raise ImproperlyConfigured('You must specify a storage class using ' + 'DBBACKUP_FALLBACK_STORAGE settings.') + return get_storage(path=settings.FALLBACK_STORAGE) + + class StorageError(Exception): pass diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index 19df725b..f8b363f8 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -2,6 +2,9 @@ Tests for dbbackup command. """ import os +from six import StringIO +from django.core.management import execute_from_command_line + from mock import patch from django.test import TestCase @@ -50,6 +53,16 @@ def test_path(self): # tearDown os.remove(self.command.path) + # def test_fallback(self): + # stdout = StringIO() + # with patch('sys.stdout', stdout): + # execute_from_command_line(['', 'dbbackup', '--fallback']) + # stdout.seek(0) + # stdout.readline() + # for line in stdout.readlines(): + # self.assertIn('You must specify a storage class using ' + # 'DBBACKUP_FALLBACK_STORAGE settings.', line) + @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') @patch('sys.stdout', DEV_NULL) From 806e0e600c5c3182f594583a2817b9de9a9e6bc4 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Fri, 2 Oct 2020 12:42:15 -0400 Subject: [PATCH 02/11] add unit tests to check for ImproperlyConfigured and add TODOs --- dbbackup/tests/commands/test_dbbackup.py | 20 +++++++++++--------- dbbackup/tests/commands/test_dbrestore.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index f8b363f8..b5b98049 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -2,6 +2,8 @@ Tests for dbbackup command. """ import os + +from django.core.exceptions import ImproperlyConfigured from six import StringIO from django.core.management import execute_from_command_line @@ -53,15 +55,15 @@ def test_path(self): # tearDown os.remove(self.command.path) - # def test_fallback(self): - # stdout = StringIO() - # with patch('sys.stdout', stdout): - # execute_from_command_line(['', 'dbbackup', '--fallback']) - # stdout.seek(0) - # stdout.readline() - # for line in stdout.readlines(): - # self.assertIn('You must specify a storage class using ' - # 'DBBACKUP_FALLBACK_STORAGE settings.', line) + def test_fallback(self): + stdout = StringIO() + with self.assertRaises(ImproperlyConfigured) as ic: + with patch('sys.stdout', stdout): + execute_from_command_line(['', 'dbbackup', '--fallback']) + self.assertEqual(str(ic.exception), + 'You must specify a storage class using DBBACKUP_FALLBACK_STORAGE settings.') + + # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful backup. @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index 72f8cffa..af5f8094 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -1,6 +1,8 @@ """ Tests for dbrestore command. """ +from django.core.exceptions import ImproperlyConfigured +from django.core.management import execute_from_command_line from mock import patch from tempfile import mktemp from shutil import copyfileobj @@ -9,6 +11,7 @@ from django.core.management.base import CommandError from django.core.files import File from django.conf import settings +from six import StringIO from dbbackup import utils from dbbackup.db.base import get_connector @@ -90,6 +93,16 @@ def test_path(self, *args): ) self.command._restore_backup() + def test_fallback(self, *args): + stdout = StringIO() + with self.assertRaises(ImproperlyConfigured) as ic: + with patch('sys.stdout', stdout): + execute_from_command_line(['', 'dbrestore', '--fallback']) + self.assertEqual(str(ic.exception), + 'You must specify a storage class using DBBACKUP_FALLBACK_STORAGE settings.') + + # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful restore. + class DbrestoreCommandGetDatabaseTest(TestCase): def setUp(self): From 560ef0fa7114b2cda7e179843ffe9e311d7a2574 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Fri, 2 Oct 2020 17:57:02 -0400 Subject: [PATCH 03/11] add option for dictionary of storages --- .gitignore | 1 + dbbackup/management/commands/dbbackup.py | 12 ++++++------ dbbackup/management/commands/dbrestore.py | 16 ++++++++-------- dbbackup/settings.py | 2 +- dbbackup/storage.py | 8 ++++---- dbbackup/tests/commands/test_dbbackup.py | 4 ++-- dbbackup/tests/commands/test_dbrestore.py | 4 ++-- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 63573cb3..0d0cb131 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ coverage.xml *,cover tests/media/ coverage_html_report/ +file:memorydb_default?mode=memory&cache=shared # Translations *.mo diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index 33ac2179..e35f12bc 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -9,7 +9,7 @@ from ._base import BaseDbBackupCommand, make_option from ...db.base import get_connector -from ...storage import get_storage, get_fallback_storage, StorageError +from ...storage import get_storage, get_db_storage, StorageError from ... import utils, settings @@ -34,8 +34,8 @@ class Command(BaseDbBackupCommand): help="Specify filename on storage"), make_option("-O", "--output-path", default=None, help="Specify where to store on local filesystem"), - make_option("-f", "--fallback", action="store_true", default=False, - help="Use alternate (fallback) storage class.") + make_option("--storage", default=None, + help="Specify storage from DBACKUP_STORAGES to use"), ) @utils.email_uncaught_exception @@ -50,13 +50,13 @@ def handle(self, **options): self.compress = options.get('compress') self.encrypt = options.get('encrypt') - self.fallback = options.get('fallback') + self.db_storage = options.get('storage') self.filename = options.get('output_filename') self.path = options.get('output_path') - if self.fallback: - self.storage = get_fallback_storage() + if self.db_storage: + self.storage = get_db_storage(self.db_storage) else: self.storage = get_storage() diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 407c969e..df6c8a68 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -13,7 +13,7 @@ from ._base import BaseDbBackupCommand, make_option from ... import utils from ...db.base import get_connector -from ...storage import get_storage, get_fallback_storage, StorageError +from ...storage import get_storage, get_db_storage, StorageError class Command(BaseDbBackupCommand): @@ -33,8 +33,8 @@ class Command(BaseDbBackupCommand): make_option("-p", "--passphrase", help="Passphrase for decrypt file", default=None), make_option("-z", "--uncompress", action='store_true', default=False, help="Uncompress gzip data before restoring"), - make_option("-f", "--fallback", action="store_true", default=False, - help="Use alternate (fallback) storage class.") + make_option("--storage", default=None, + help="Specify storage from DBACKUP_STORAGES to use"), ) def handle(self, *args, **options): @@ -52,11 +52,11 @@ def handle(self, *args, **options): self.uncompress = options.get('uncompress') self.passphrase = options.get('passphrase') self.interactive = options.get('interactive') - self.fallback = options.get('fallback') + self.db_storage = options.get('storage') self.database_name, self.database = self._get_database(options) - if self.fallback: - self.storage = get_fallback_storage() + if self.db_storage: + self.storage = get_db_storage(self.db_storage) else: self.storage = get_storage() @@ -69,8 +69,8 @@ def _get_database(self, options): database_name = options.get('database') if not database_name: if len(settings.DATABASES) > 1: - errmsg = "Because this project contains more than one database, you"\ - " must specify the --database option." + errmsg = "Because this project contains more than one database, you" \ + " must specify the --database option." raise CommandError(errmsg) database_name = list(settings.DATABASES.keys())[0] if database_name not in settings.DATABASES: diff --git a/dbbackup/settings.py b/dbbackup/settings.py index f22374ec..1e4463a8 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -32,7 +32,7 @@ STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'django.core.files.storage.FileSystemStorage') STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {}) -FALLBACK_STORAGE = getattr(settings, 'DBBACKUP_FALLBACK_STORAGE', None) +STORAGES = getattr(settings, 'DBBACKUP_STORAGES', {'default': 'django.core.files.storage.FileSystemStorage'}) CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {}) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index ce7fc196..fe136164 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -30,11 +30,11 @@ def get_storage(path=None, options=None): return Storage(path, **options) -def get_fallback_storage(): - if not settings.FALLBACK_STORAGE: +def get_db_storage(db_storage): + if db_storage not in settings.STORAGES: raise ImproperlyConfigured('You must specify a storage class using ' - 'DBBACKUP_FALLBACK_STORAGE settings.') - return get_storage(path=settings.FALLBACK_STORAGE) + 'DBBACKUP_STORAGES settings.') + return get_storage(path=settings.STORAGES[db_storage]) class StorageError(Exception): diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index b5b98049..b1c228fc 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -59,9 +59,9 @@ def test_fallback(self): stdout = StringIO() with self.assertRaises(ImproperlyConfigured) as ic: with patch('sys.stdout', stdout): - execute_from_command_line(['', 'dbbackup', '--fallback']) + execute_from_command_line(['', 'dbbackup', '--storage=s3']) self.assertEqual(str(ic.exception), - 'You must specify a storage class using DBBACKUP_FALLBACK_STORAGE settings.') + 'You must specify a storage class using DBBACKUP_STORAGES settings.') # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful backup. diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index af5f8094..3ce00687 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -97,9 +97,9 @@ def test_fallback(self, *args): stdout = StringIO() with self.assertRaises(ImproperlyConfigured) as ic: with patch('sys.stdout', stdout): - execute_from_command_line(['', 'dbrestore', '--fallback']) + execute_from_command_line(['', 'dbrestore', '--storage=s3']) self.assertEqual(str(ic.exception), - 'You must specify a storage class using DBBACKUP_FALLBACK_STORAGE settings.') + 'You must specify a storage class using DBBACKUP_STORAGES settings.') # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful restore. From 4d4375358bf2409c2ed49117b3869bc0377e4392 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 3 Oct 2020 08:59:15 -0400 Subject: [PATCH 04/11] update storage class and options with DBBACKUP_STORAGES --- dbbackup/management/commands/dbbackup.py | 4 ++-- dbbackup/management/commands/dbrestore.py | 4 ++-- dbbackup/settings.py | 6 +++++- dbbackup/storage.py | 11 +++++++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index e35f12bc..7559edd7 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -9,7 +9,7 @@ from ._base import BaseDbBackupCommand, make_option from ...db.base import get_connector -from ...storage import get_storage, get_db_storage, StorageError +from ...storage import get_storage, get_backup_storage, StorageError from ... import utils, settings @@ -56,7 +56,7 @@ def handle(self, **options): self.path = options.get('output_path') if self.db_storage: - self.storage = get_db_storage(self.db_storage) + self.storage = get_backup_storage(self.db_storage) else: self.storage = get_storage() diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index df6c8a68..8c8bf41f 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -13,7 +13,7 @@ from ._base import BaseDbBackupCommand, make_option from ... import utils from ...db.base import get_connector -from ...storage import get_storage, get_db_storage, StorageError +from ...storage import get_storage, get_backup_storage, StorageError class Command(BaseDbBackupCommand): @@ -56,7 +56,7 @@ def handle(self, *args, **options): self.database_name, self.database = self._get_database(options) if self.db_storage: - self.storage = get_db_storage(self.db_storage) + self.storage = get_backup_storage(self.db_storage) else: self.storage = get_storage() diff --git a/dbbackup/settings.py b/dbbackup/settings.py index 1e4463a8..c9df6c81 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -32,7 +32,11 @@ STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'django.core.files.storage.FileSystemStorage') STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {}) -STORAGES = getattr(settings, 'DBBACKUP_STORAGES', {'default': 'django.core.files.storage.FileSystemStorage'}) +STORAGES = getattr(settings, 'DBBACKUP_STORAGES', { + 'default': { + 'storage': 'django.core.files.storage.FileSystemStorage' + } +}) CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {}) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index fe136164..805e9a33 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -30,11 +30,17 @@ def get_storage(path=None, options=None): return Storage(path, **options) -def get_db_storage(db_storage): +def get_backup_storage(db_storage): if db_storage not in settings.STORAGES: raise ImproperlyConfigured('You must specify a storage class using ' 'DBBACKUP_STORAGES settings.') - return get_storage(path=settings.STORAGES[db_storage]) + storage_options = settings.STORAGES[db_storage] + if 'storage' in storage_options: + storage = storage_options.pop('storage', None) + options = storage_options + return get_storage(path=storage, options=options) + raise ImproperlyConfigured('You must specify a storage class "storage" using ' + 'DBBACKUP_STORAGES settings.') class StorageError(Exception): @@ -51,6 +57,7 @@ class Storage(object): list and filter files. It uses a Django storage object for low-level operations. """ + @property def logger(self): if not hasattr(self, '_logger'): From e155752694c8f2510af93f98c69944326fd7f35b Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 3 Oct 2020 09:40:20 -0400 Subject: [PATCH 05/11] update DBSTORAGES options from settings.STORAGE_OPTIONS if the k,v pair does not already exist --- dbbackup/storage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index 805e9a33..5f21ee82 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -75,7 +75,9 @@ def __init__(self, storage_path=None, **options): """ self._storage_path = storage_path or settings.STORAGE options = options.copy() - options.update(settings.STORAGE_OPTIONS) + for k, v in settings.STORAGE_OPTIONS: + if k not in options: + options[k] = v options = dict([(key.lower(), value) for key, value in options.items()]) self.storageCls = get_storage_class(self._storage_path) self.storage = self.storageCls(**options) From 839bf2adefa7884d5189d18c99df4079ee15b112 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 3 Oct 2020 09:48:23 -0400 Subject: [PATCH 06/11] loop through keys to avoid error if settings.STORAGE_OPTIONS is {} --- dbbackup/storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index 5f21ee82..c1045163 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -75,9 +75,9 @@ def __init__(self, storage_path=None, **options): """ self._storage_path = storage_path or settings.STORAGE options = options.copy() - for k, v in settings.STORAGE_OPTIONS: - if k not in options: - options[k] = v + for option in settings.STORAGE_OPTIONS.keys(): + if option not in options: + options[option] = settings.STORAGE_OPTIONS[option] options = dict([(key.lower(), value) for key, value in options.items()]) self.storageCls = get_storage_class(self._storage_path) self.storage = self.storageCls(**options) From ba7056e019461dc2a1dab84e58a5cd377fbfdb49 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 19 Dec 2020 13:03:57 -0500 Subject: [PATCH 07/11] remove unused import --- dbbackup/management/commands/dbrestore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 8c8bf41f..e0edd455 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -5,7 +5,6 @@ print_function, unicode_literals) from django.conf import settings -from django.core.exceptions import ImproperlyConfigured from django.core.management.base import CommandError from django.db import connection From f9dccd2869e06421af6fab6d01f587917cedc5d9 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 26 Dec 2020 10:36:26 -0500 Subject: [PATCH 08/11] fix settings pop bug with copy, add unit tests --- dbbackup/storage.py | 3 ++- dbbackup/tests/commands/test_dbbackup.py | 29 +++++++++++++++++++++++- dbbackup/tests/settings.py | 19 +++++++++++++++- dbbackup/tests/test_storage.py | 26 +++++++++++++++++---- dbbackup/tests/test_utils.py | 1 - requirements-tests.txt | 2 +- 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index c1045163..681f1c8f 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -34,11 +34,12 @@ def get_backup_storage(db_storage): if db_storage not in settings.STORAGES: raise ImproperlyConfigured('You must specify a storage class using ' 'DBBACKUP_STORAGES settings.') - storage_options = settings.STORAGES[db_storage] + storage_options = settings.STORAGES[db_storage].copy() if 'storage' in storage_options: storage = storage_options.pop('storage', None) options = storage_options return get_storage(path=storage, options=options) + print(settings.STORAGES) raise ImproperlyConfigured('You must specify a storage class "storage" using ' 'DBBACKUP_STORAGES settings.') diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index b1c228fc..72801603 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -4,6 +4,7 @@ import os from django.core.exceptions import ImproperlyConfigured +from django.core.files.storage import FileSystemStorage from six import StringIO from django.core.management import execute_from_command_line @@ -13,7 +14,7 @@ from dbbackup.management.commands.dbbackup import Command as DbbackupCommand from dbbackup.db.base import get_connector -from dbbackup.storage import get_storage +from dbbackup.storage import get_storage, get_backup_storage from dbbackup.tests.utils import (TEST_DATABASE, add_public_gpg, clean_gpg_keys, DEV_NULL) @@ -88,3 +89,29 @@ def tearDown(self): def test_func(self, mock_run_commands, mock_handle_size): self.command._save_new_backup(TEST_DATABASE) self.assertTrue(mock_run_commands.called) + + +class DbbackupCommandSaveMultipleStorages(TestCase): + def setUp(self): + self.command = DbbackupCommand() + self.command.servername = 'foo-server' + self.command.encrypt = False + self.command.compress = False + self.command.connector = get_connector() + self.command.stdout = DEV_NULL + self.command.filename = None + self.command.path = None + + def test_default_func(self): + self.command.database = TEST_DATABASE['NAME'] + self.command.storage = get_backup_storage('default') + self.command._save_new_backup(TEST_DATABASE) + + def test_fake_func(self): + self.command.database = TEST_DATABASE['NAME'] + self.command.storage = get_backup_storage('fake_storage') + self.command._save_new_backup(TEST_DATABASE) + + def test_default(self): + self.storage = get_backup_storage('default') + self.assertIsInstance(self.storage.storage, FileSystemStorage) diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 0aff0fd5..4e4b263f 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -2,8 +2,9 @@ Configuration and launcher for dbbackup tests. """ import os -import tempfile import sys +import tempfile + from dotenv import load_dotenv test = len(sys.argv) <= 1 or sys.argv[1] == 'test' @@ -59,6 +60,22 @@ os.environ.get('STORAGE_OPTIONS', '').split(',') if keyvalue]) +DBBACKUP_STORAGES = { + 'default': { + 'storage': 'django.core.files.storage.FileSystemStorage', + }, + 'fake_storage': { + 'storage': 'dbbackup.tests.utils.FakeStorage', + }, + 's3_storage': { + 'storage': 'storages.backends.s3boto3.S3Boto3Storage', + 'access_key': 'my_id', + 'secret_key': 'my_secret', + 'bucket_name': 'my_bucket_name', + 'default_acl': 'private', + } +} + LOGGING = { 'version': 1, 'disable_existing_loggers': False, diff --git a/dbbackup/tests/test_storage.py b/dbbackup/tests/test_storage.py index b46eb824..1fe32982 100644 --- a/dbbackup/tests/test_storage.py +++ b/dbbackup/tests/test_storage.py @@ -1,8 +1,11 @@ -from mock import patch from django.test import TestCase -from dbbackup.storage import get_storage, Storage -from dbbackup.tests.utils import HANDLED_FILES, FakeStorage +from django.core.files.storage import FileSystemStorage +from mock import patch +from storages.backends.s3boto3 import S3Boto3Storage + from dbbackup import utils +from dbbackup.storage import get_storage, Storage, get_backup_storage +from dbbackup.tests.utils import HANDLED_FILES, FakeStorage DEFAULT_STORAGE_PATH = 'django.core.files.storage.FileSystemStorage' STORAGE_OPTIONS = {'location': '/tmp'} @@ -174,4 +177,19 @@ def test_func(self): @patch('dbbackup.settings.CLEANUP_KEEP_FILTER', keep_only_even_files) def test_keep_filter(self): self.storage.clean_old_backups(keep_number=1) - self.assertListEqual(['2015-02-07-042810.bak'], HANDLED_FILES['deleted_files']) \ No newline at end of file + self.assertListEqual(['2015-02-07-042810.bak'], HANDLED_FILES['deleted_files']) + + +class StorageBackendsTest(TestCase): + def test_default(self): + self.storage = get_backup_storage('default') + self.assertIsInstance(self.storage.storage, FileSystemStorage) + + def test_custom(self): + self.storage = get_backup_storage('s3_storage') + self.assertIsInstance(self.storage.storage, S3Boto3Storage) + self.assertEqual(vars(self.storage.storage)['_constructor_args'][1], + {'access_key': 'my_id', + 'secret_key': 'my_secret', + 'bucket_name': 'my_bucket_name', + 'default_acl': 'private'}) diff --git a/dbbackup/tests/test_utils.py b/dbbackup/tests/test_utils.py index ce549833..52c9d797 100644 --- a/dbbackup/tests/test_utils.py +++ b/dbbackup/tests/test_utils.py @@ -12,7 +12,6 @@ from dbbackup import settings, utils from dbbackup.tests.utils import ( COMPRESSED_FILE, - DEV_NULL, ENCRYPTED_FILE, add_private_gpg, add_public_gpg, diff --git a/requirements-tests.txt b/requirements-tests.txt index 6582b0ef..080321b5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,5 @@ coverage -django-storages +django-storages[boto3] flake8 mock pep8 From 4a16ff852df801a4a2b01ea210d75017835a5efb Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 26 Dec 2020 10:50:01 -0500 Subject: [PATCH 09/11] fix linting --- dbbackup/db/postgresql.py | 1 - dbbackup/management/commands/dbbackup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/dbbackup/db/postgresql.py b/dbbackup/db/postgresql.py index d263e4ea..89707ca0 100644 --- a/dbbackup/db/postgresql.py +++ b/dbbackup/db/postgresql.py @@ -1,7 +1,6 @@ from urllib.parse import quote import logging -from dbbackup import utils from .base import BaseCommandDBConnector from .exceptions import DumpError diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index f622d967..68795ae8 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -57,7 +57,6 @@ def handle(self, **options): self.path = options.get('output_path') self.exclude_tables = options.get("exclude_tables") - if self.db_storage: self.storage = get_backup_storage(self.db_storage) else: From 8c22363bec1595bff67563f5a311df0ca34c3a96 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 26 Dec 2020 12:03:25 -0500 Subject: [PATCH 10/11] add command tests to verify storage configuration is setup correctly when setting storage option in command --- dbbackup/tests/commands/test_dbbackup.py | 41 +++++++++++++++- dbbackup/tests/commands/test_dbrestore.py | 60 +++++++++++++++++++---- 2 files changed, 91 insertions(+), 10 deletions(-) diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index 72801603..cfd8d284 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -11,12 +11,13 @@ from mock import patch from django.test import TestCase +from storages.backends.s3boto3 import S3Boto3Storage from dbbackup.management.commands.dbbackup import Command as DbbackupCommand from dbbackup.db.base import get_connector from dbbackup.storage import get_storage, get_backup_storage from dbbackup.tests.utils import (TEST_DATABASE, add_public_gpg, clean_gpg_keys, - DEV_NULL) + DEV_NULL, FakeStorage) @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') @@ -115,3 +116,41 @@ def test_fake_func(self): def test_default(self): self.storage = get_backup_storage('default') self.assertIsInstance(self.storage.storage, FileSystemStorage) + + +class DbbackupCommandMultipleStorages(TestCase): + def setUp(self): + self.command = DbbackupCommand() + self.command.stdout = DEV_NULL + self.command.uncompress = False + self.command.decrypt = False + self.command.backup_extension = 'bak' + self.command.filename = 'foofile' + self.command.database = TEST_DATABASE + self.command.passphrase = None + self.command.interactive = True + self.command.database_name = 'default' + self.command.connector = get_connector('default') + + @staticmethod + def fake_backup(): + return True + + def test_default(self): + self.command.handle(storage='default', verbosity=1) + self.assertIsInstance(self.command.storage.storage, FileSystemStorage) + + @patch.object(DbbackupCommand, '_save_new_backup') + def test_fake(self, fake_backup): + self.command.handle(storage='fake_storage', verbosity=1) + self.assertIsInstance(self.command.storage.storage, FakeStorage) + + @patch.object(DbbackupCommand, '_save_new_backup') + def test_S3(self, fake_backup): + self.command.handle(storage='s3_storage', verbosity=1) + self.assertIsInstance(self.command.storage.storage, S3Boto3Storage) + self.assertEqual(vars(self.command.storage.storage)['_constructor_args'][1], + {'access_key': 'my_id', + 'secret_key': 'my_secret', + 'bucket_name': 'my_bucket_name', + 'default_acl': 'private'}) diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index 3ce00687..64c99fc5 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -1,27 +1,29 @@ """ Tests for dbrestore command. """ -from django.core.exceptions import ImproperlyConfigured -from django.core.management import execute_from_command_line -from mock import patch -from tempfile import mktemp from shutil import copyfileobj +from tempfile import mktemp -from django.test import TestCase -from django.core.management.base import CommandError -from django.core.files import File from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.core.files import File +from django.core.files.storage import FileSystemStorage +from django.core.management import execute_from_command_line +from django.core.management.base import CommandError +from django.test import TestCase +from mock import patch from six import StringIO +from storages.backends.s3boto3 import S3Boto3Storage from dbbackup import utils from dbbackup.db.base import get_connector from dbbackup.db.mongodb import MongoDumpConnector from dbbackup.management.commands.dbrestore import Command as DbrestoreCommand -from dbbackup.storage import get_storage from dbbackup.settings import HOSTNAME +from dbbackup.storage import get_storage from dbbackup.tests.utils import (TEST_DATABASE, add_private_gpg, DEV_NULL, clean_gpg_keys, HANDLED_FILES, TEST_MONGODB, TARED_FILE, - get_dump, get_dump_name) + get_dump, get_dump_name, FakeStorage) @patch('dbbackup.management.commands._base.input', return_value='y') @@ -152,3 +154,43 @@ def test_mongo_settings_backup_command(self, mock_runcommands, *args): HANDLED_FILES['written_files'].append((TARED_FILE, open(TARED_FILE, 'rb'))) self.command._restore_backup() self.assertTrue(mock_runcommands.called) + + +class DbrestoreCommandRestoreMultipleBackupTest(TestCase): + def setUp(self): + self.command = DbrestoreCommand() + self.command.stdout = DEV_NULL + self.command.uncompress = False + self.command.decrypt = False + self.command.backup_extension = 'bak' + self.command.filename = 'foofile' + self.command.database = TEST_DATABASE + self.command.passphrase = None + self.command.interactive = True + self.command.servername = HOSTNAME + self.command.database_name = 'default' + self.command.connector = get_connector('default') + HANDLED_FILES.clean() + + @staticmethod + def fake_restore(): + return True + + def test_default(self): + self.command.handle(storage='default', verbosity=1) + self.assertIsInstance(self.command.storage.storage, FileSystemStorage) + + @patch.object(DbrestoreCommand, '_restore_backup') + def test_fake(self, fake_restore): + self.command.handle(storage='fake_storage', verbosity=1) + self.assertIsInstance(self.command.storage.storage, FakeStorage) + + @patch.object(DbrestoreCommand, '_restore_backup') + def test_S3(self, fake_restore): + self.command.handle(storage='s3_storage', verbosity=1) + self.assertIsInstance(self.command.storage.storage, S3Boto3Storage) + self.assertEqual(vars(self.command.storage.storage)['_constructor_args'][1], + {'access_key': 'my_id', + 'secret_key': 'my_secret', + 'bucket_name': 'my_bucket_name', + 'default_acl': 'private'}) From 6465092220bf02ab8fadf80694dbecba587a8332 Mon Sep 17 00:00:00 2001 From: Marc LaBelle Date: Sat, 26 Dec 2020 12:21:24 -0500 Subject: [PATCH 11/11] remove DBBACKUP_FALLBACK_STORAGE comment because I don't remember what I meant, but I think it's done --- dbbackup/tests/commands/test_dbbackup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index cfd8d284..310190ed 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -65,8 +65,6 @@ def test_fallback(self): self.assertEqual(str(ic.exception), 'You must specify a storage class using DBBACKUP_STORAGES settings.') - # TODO: Update DBBACKUP_FALLBACK_STORAGE and verify successful backup. - @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') @patch('sys.stdout', DEV_NULL)