Merge remote-tracking branch 'origin/beta'

helpers/config.py

@@ -30,8 +30,8 @@ class Config(metaclass=Singleton):
     DEFAULT_PROXY_PORT = '8080'
     DEFAULT_NGINX_PORT = '80'
     DEFAULT_NGINX_HTTPS_PORT = '443'
-    KOBO_DOCKER_BRANCH = '2.021.21a'
-    KOBO_INSTALL_VERSION = '4.4.4'
+    KOBO_DOCKER_BRANCH = '2.021.24b'
+    KOBO_INSTALL_VERSION = '5.0.0'
     MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3
 
     def __init__(self):
@@ -197,7 +197,7 @@ def build(self):
                         if self.frontend or self.secondary_backend:
                             self.__questions_private_routes()
                     else:
-                        self.__reset(private_dns=True)
+                        self.__reset(fake_dns=True)
 
                 if self.frontend_questions:
                     self.__questions_public_routes()
@@ -418,6 +418,7 @@ def get_template(cls):
             'two_databases': True,
             'use_aws': False,
             'use_backup': False,
+            'use_celery': True,
             'use_letsencrypt': True,
             'use_private_dns': False,
             'use_wal_e': False,
@@ -1028,11 +1029,9 @@ def __questions_backup(self):
                         # Back to default value
                         self.__dict['backup_from_primary'] = True
             else:
-                # Back to default value
-                self.__dict['backup_from_primary'] = True
+                self.__reset(no_backups=True)
         else:
-            self.__dict['use_backup'] = False
-            self.__dict['backup_from_primary'] = True  # Back to default value
+            self.__reset(no_backups=True)
 
     def __questions_dev_mode(self):
         """
@@ -1047,7 +1046,7 @@ def __questions_dev_mode(self):
         if self.frontend_questions:
 
             if self.local_install:
-                # NGinX different port
+                # NGINX different port
                 CLI.colored_print('Web server port?', CLI.COLOR_QUESTION)
                 self.__dict['exposed_nginx_docker_port'] = CLI.get_response(
                     r'~^\d+$', self.__dict['exposed_nginx_docker_port'])
@@ -1056,12 +1055,19 @@ def __questions_dev_mode(self):
                     default=self.__dict['dev_mode']
                 )
                 self.__dict['staging_mode'] = False
+                if self.dev_mode:
+                    self.__dict['use_celery'] = CLI.yes_no_question(
+                        'Use Celery for background tasks?',
+                        default=self.__dict['use_celery']
+                    )
+
             else:
                 self.__dict['staging_mode'] = CLI.yes_no_question(
                     'Use staging mode?',
                     default=self.__dict['staging_mode']
                 )
                 self.__dict['dev_mode'] = False
+                self.__dict['use_celery'] = True
 
             if self.dev_mode or self.staging_mode:
                 message = (
@@ -1122,7 +1128,7 @@ def __questions_dev_mode(self):
                     )
             else:
                 # Force reset paths
-                self.__reset(dev=True, reset_nginx_port=self.staging_mode)
+                self.__reset(production=True, nginx_default=self.staging_mode)
 
     def __questions_docker_prefix(self):
         """
@@ -1169,6 +1175,8 @@ def __questions_installation_type(self):
         """
         Asks for installation type
         """
+        previous_installation_type = self.__dict['local_installation']
+
         self.__dict['local_installation'] = CLI.yes_no_question(
             'What kind of installation do you need?',
             default=self.__dict['local_installation'],
@@ -1178,8 +1186,19 @@ def __questions_installation_type(self):
             ]
         )
         if self.local_install:
+            message = (
+                'WARNING!\n\n'
+                'SSRF protection is disabled with local installation'
+            )
+            CLI.framed_print(message, color=CLI.COLOR_WARNING)
+
+        if previous_installation_type != self.__dict['local_installation']:
             # Reset previous choices, in case server role is not the same.
-            self.__reset(local_install=True, private_dns=True)
+            self.__reset(
+                production=not self.local_install,
+                http=self.local_install,
+                fake_dns=self.local_install,
+            )
 
     def __questions_maintenance(self):
         if self.first_time:
@@ -2011,35 +2030,45 @@ def __reset(self, **kwargs):
         It can be useful, if user changes the type of installation on
         the same server
 
-        Returns:
-            bool
+        Kwargs:
+            production (bool): If `True`, reset config to production mode
+            http (bool): If `True`, only set values related to http/https config
+            fake_dns (bool): If `True`, reset config to fake dns on docker-compose files  # noqa
+            nginx_default (bool): If `True`, reset NGINX exposed port to default
         """
-        all = True if not kwargs else False
-        dev_mode = kwargs.get('dev', False)
-        local_install = kwargs.get('local_install', False)
-        private_dns = kwargs.get('private_dns', False)
-        reset_nginx_port = kwargs.get('reset_nginx_port', False)
+        all_ = True if not kwargs else False
+        production = kwargs.get('production', False)
+        http = kwargs.get('http', False)
+        fake_dns = kwargs.get('fake_dns', False)
+        nginx_default = kwargs.get('nginx_default', False)
+        no_backups = kwargs.get('no_backups', False)
 
-        if dev_mode or all:
+        if production or all_:
             self.__dict['dev_mode'] = False
             self.__dict['staging_mode'] = False
             self.__dict['kc_path'] = ''
             self.__dict['kpi_path'] = ''
             self.__dict['debug'] = False
-            if reset_nginx_port:
+            self.__dict['use_celery'] = True
+            if nginx_default:
                 self.__dict[
                     'exposed_nginx_docker_port'] = Config.DEFAULT_NGINX_PORT
 
-        if private_dns or all:
+        if fake_dns or all_:
             self.__dict['use_private_dns'] = False
 
-        if local_install or all:
+        if http or all_:
             self.__dict['multi'] = False
             self.__dict['https'] = False
             self.__dict['proxy'] = False
             self.__dict['nginx_proxy_port'] = Config.DEFAULT_NGINX_PORT
             self.__dict['use_letsencrypt'] = False
 
+        if no_backups or all_:
+            self.__dict['backup_from_primary'] = True
+            self.__dict['use_backup'] = False
+            self.__dict['use_wal_e'] = False
+
     def __secure_mongo(self):
         """
         Force creations of MongoDB users/passwords when users upgrade from

helpers/template.py

@@ -290,6 +290,13 @@ def _get_value(property_, true_value='', false_value='#',
                 dict_['redis_password']),
             'REDIS_PASSWORD_JS_ENCODED': json.dumps(
                 dict_['redis_password']),
+            'USE_DEV_MODE': _get_value('dev_mode'),
+            'USE_CELERY': _get_value('use_celery', comparison_value=False),
+            'ENKETO_ALLOW_PRIVATE_IP_ADDRESS': _get_value(
+                'local_installation',
+                true_value='true',
+                false_value='false'
+            )
         }
 
     @staticmethod

templates/kobo-docker/docker-compose.frontend.override.yml.tpl

@@ -16,6 +16,8 @@ services:
       - KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT}
       - KC_UWSGI_HARAKIRI=${UWSGI_HARAKIRI}
       - KC_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY}
+      ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=onadata.settings.dev
+      ${USE_CELERY}- SKIP_CELERY=True
     ${USE_EXTRA_HOSTS}extra_hosts:
       ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP}
       ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP}
@@ -43,6 +45,8 @@ services:
       - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT}
       - KPI_UWSGI_HARAKIRI=${UWSGI_HARAKIRI}
       - KPI_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY}
+      ${USE_CELERY}- SKIP_CELERY=True
+      ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=kobo.settings.dev
       ${USE_HTTPS}- SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO, https
       ${USE_NPM_FROM_HOST}- FRONTEND_DEV_MODE=host
     ${USE_EXTRA_HOSTS}extra_hosts:

templates/kobo-env/enketo_express/config.json.tpl

@@ -5,6 +5,12 @@
         "server url": "",
         "api key": "${ENKETO_API_KEY}"
     },
+    "ip filtering": {
+        "allowPrivateIPAddress": ${ENKETO_ALLOW_PRIVATE_IP_ADDRESS},
+        "allowMetaIPAddress": false,
+        "allowIPAddressList": [],
+        "denyAddressList": []
+    },
     "encryption key": "${ENKETO_ENCRYPTION_KEY}",
     "less secure encryption key": "${ENKETO_LESS_SECURE_ENCRYPTION_KEY}",
     "support": {

templates/kobo-env/envfiles/aws.txt.tpl

@@ -11,7 +11,7 @@ ${USE_AWS}AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
 
 # To use S3, the specified buckets must already exist and the owner of your `AWS_ACCESS_KEY_ID` must have the appropriate S3 permissions.
 
-${USE_AWS}KOBOCAT_DEFAULT_FILE_STORAGE=storages.backends.s3boto.S3BotoStorage
+${USE_AWS}KOBOCAT_DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage
 ${USE_AWS}KOBOCAT_AWS_STORAGE_BUCKET_NAME=${AWS_BUCKET_NAME}
 
 ${USE_AWS}KPI_DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage

templates/kobo-env/envfiles/kobocat.txt.tpl

@@ -2,7 +2,7 @@ KOBOCAT_DJANGO_DEBUG=${DEBUG}
 TEMPLATE_DEBUG=${DEBUG}
 ${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True
 
-DJANGO_SETTINGS_MODULE=onadata.settings.kc_environ
+DJANGO_SETTINGS_MODULE=onadata.settings.prod
 ENKETO_VERSION=Express
 
 KOBOCAT_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/2

tests/test_config.py

@@ -86,6 +86,7 @@ def test_dev_mode():
     with patch('helpers.cli.CLI.colored_input') as mock_colored_input:
         mock_colored_input.side_effect = iter(['8080',
                                                CHOICE_YES,
+                                               CHOICE_NO,
                                                kc_repo_path,
                                                kpi_repo_path,
                                                CHOICE_YES,
@@ -100,6 +101,7 @@ def test_dev_mode():
         assert dict_['kpi_path'] == kpi_repo_path and \
                dict_['kc_path'] == kc_repo_path
         assert dict_['npm_container'] is False
+        assert dict_['use_celery'] is False
 
     shutil.rmtree(kc_repo_path)
     shutil.rmtree(kpi_repo_path)