Merge pull request #206 from kobotoolbox/update-enketo-logo

Update the KoboToolbox logo in Enketo

helpers/config.py

@@ -140,6 +140,7 @@ def build(self):
                     self.__questions_google()
                     self.__questions_raven()
                     self.__questions_uwsgi()
+                    self.__questions_service_account()
 
                 self.__questions_custom_yml()
 
@@ -364,6 +365,7 @@ def get_template(cls):
             'multi': False,
             'nginx_proxy_port': Config.DEFAULT_PROXY_PORT,
             'npm_container': True,
+            'postgres_cpus': '1',
             'postgres_backup_schedule': '0 2 * * 0',
             'postgres_hard_drive_type': 'hdd',
             'postgres_max_connections': '100',
@@ -373,21 +375,36 @@ def get_template(cls):
             'postgres_replication_password': Config.generate_password(),
             'postgres_settings': False,
             'postgres_settings_content': '\n'.join([
+                '# Generated by PGConfig 3.0.0 (3af8ea764b2fd9ea8401acfdc69f90cb825cdee9)',
+                '# ',
+                'https://api.pgconfig.org/v1/tuning/get-config?environment_name=Desktop&format=conf&include_pgbadger=false&cpus=1&max_connections=100&pg_version=14&total_ram=2GB&drive_type=HDD',
+                '',
                 '# Memory Configuration',
-                'shared_buffers = 512MB',
-                'effective_cache_size = 2GB',
-                'work_mem = 10MB',
-                'maintenance_work_mem = 128MB',
+                'shared_buffers = 128MB',
+                'effective_cache_size = 307MB',
+                'work_mem = 419KB',
+                'maintenance_work_mem = 20MB',
                 '',
                 '# Checkpoint Related Configuration',
-                'min_wal_size = 512MB',
-                'max_wal_size = 2GB',
+                'min_wal_size = 2GB',
+                'max_wal_size = 3GB',
                 'checkpoint_completion_target = 0.9',
-                'wal_buffers = 15MB',
+                'wal_buffers = -1',
                 '',
                 '# Network Related Configuration',
                 "listen_addresses = '*'",
                 'max_connections = 100',
+                '',
+                '# Storage Configuration',
+                'random_page_cost = 4.0',
+                'effective_io_concurrency = 2',
+                '',
+                '# Worker Processes Configuration',
+                'max_worker_processes = 8',
+                'max_parallel_workers_per_gather = 2',
+                'max_parallel_workers = 2',
+                '',
+                ''
             ]),
             'postgres_user': 'kobo',
             'postgresql_port': '5432',
@@ -404,6 +421,7 @@ def get_template(cls):
             'review_host': True,
             'run_redis_containers': True,
             'server_role': 'frontend',
+            'service_account_whitelisted_hosts': True,
             'smtp_host': '',
             'smtp_password': '',
             'smtp_port': '25',
@@ -1580,12 +1598,19 @@ def __questions_postgres(self):
                         break
                     open_port += 1
 
-                # Start pgconfig.org API docker image
-                docker_command = ['docker', 'run', '--rm', '-p',
-                                  f'127.0.0.1:{open_port}:8080',
-                                  '-d', '--name', 'pgconfig_container',
-                                  'sebastianwebber/pgconfig-api']
-                CLI.run_command(docker_command)
+                # Unfortunately, the docker image is old and does not support
+                # PostgreSQL 14. The author has rewritten the code base in Go
+                # (instead of Python), so chances are pretty low that the image
+                # gets updated someday.
+                # ToDo Keep the code below for few months just in case and
+                #  get rid of it if it never happens.
+
+                # # Start pgconfig.org API docker image
+                # docker_command = ['docker', 'run', '--rm', '-p',
+                #                   f'127.0.0.1:{open_port}:8080',
+                #                   '-d', '--name', 'pgconfig_container',
+                #                   'sebastianwebber/pgconfig-api']
+                # CLI.run_command(docker_command)
 
                 # From https://docs.pgconfig.org/api/#available-parameters
                 # Parameters are case-sensitive, for example
@@ -1597,6 +1622,11 @@ def __questions_postgres(self):
                 # - `Desktop`
                 # It's case-sensitive.
 
+                CLI.colored_print('Number of CPUs?', CLI.COLOR_QUESTION)
+                self.__dict['postgres_cpus'] = CLI.get_response(
+                    r'~^\d+$',
+                    self.__dict['postgres_cpus'])
+
                 CLI.colored_print('Total Memory in GB?', CLI.COLOR_QUESTION)
                 self.__dict['postgres_ram'] = CLI.get_response(
                     r'~^\d+$',
@@ -1639,17 +1669,24 @@ def __questions_postgres(self):
                 else:
                     self.__dict['postgres_profile'] = 'Mixed'
 
-                endpoint = 'http://127.0.0.1:{open_port}/v1/tuning/get-config' \
-                           '?environment_name={profile}&format=conf' \
-                           '&include_pgbadger=false' \
-                           '&max_connections={max_connections}' \
-                           '&pg_version=9.5' \
-                           '&total_ram={ram}GB' \
-                           '&drive_type={drive_type}'
+                # Instead of using the API locally with the docker image, let's
+                # use the (unreliable) public API.
+                # Notes: It has failed several times in the past.
+                endpoint = (
+                    'https://api.pgconfig.org/v1/tuning/get-config'
+                    '?environment_name={profile}&format=conf'
+                    '&include_pgbadger=false'
+                    '&cpus={cpus}'
+                    '&max_connections={max_connections}'
+                    '&pg_version=14'
+                    '&total_ram={ram}GB'
+                    '&drive_type={drive_type}'
+                )
                 endpoint = endpoint.format(
                     open_port=open_port,
                     profile=self.__dict['postgres_profile'],
                     ram=self.__dict['postgres_ram'],
+                    cpus=self.__dict['postgres_cpus'],
                     max_connections=self.__dict['postgres_max_connections'],
                     drive_type=self.__dict['postgres_hard_drive_type'].upper()
                 )
@@ -1662,12 +1699,14 @@ def __questions_postgres(self):
                                       'PostgreSQL settings will be used',
                                       CLI.COLOR_INFO)
 
-                # Stop container
-                docker_command = ['docker', 'stop', '-t', '0',
-                                  'pgconfig_container']
-                CLI.run_command(docker_command)
-                CLI.colored_print('pgconfig.org API container has been stopped!',
-                                  CLI.COLOR_INFO)
+                # ToDo Keep the code below for few months just in case and
+                #  get rid of it if it never happens.
+                # # Stop container
+                # docker_command = ['docker', 'stop', '-t', '0',
+                #                   'pgconfig_container']
+                # CLI.run_command(docker_command)
+                # CLI.colored_print('pgconfig.org API container has been stopped!',
+                #                   CLI.COLOR_INFO)
             else:
                 # Forcing the default settings to remain even if there
                 # is an existing value in .run.conf. Without this,
@@ -2037,6 +2076,16 @@ def __questions_secret_keys(self):
                 to_lower=False,
                 error_msg='Too short. 10 characters minimum.')
 
+    def __questions_service_account(self):
+        if not self.local_install:
+            self.__dict['service_account_whitelisted_hosts'] = CLI.yes_no_question(
+                'Do you want to restrict API calls between KPI and KoBoCAT '
+                'to their internal domain names?',
+                default=self.__dict['service_account_whitelisted_hosts']
+            )
+        else:
+            self.__dict['service_account_whitelisted_hosts'] = False
+
     def __questions_smtp(self):
         self.__dict['smtp_host'] = CLI.colored_input('SMTP server?',
                                                      CLI.COLOR_QUESTION,

helpers/template.py

@@ -311,6 +311,11 @@ def _get_value(property_, true_value='', false_value='#',
                 comparison_value='',
             ),
             'USE_LETSENSCRYPT': '#' if config.use_letsencrypt else '',
+            'USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS': (
+                '#'
+                if config.local_install
+                else _get_value('service_account_whitelisted_hosts')
+            ),
         }
 
     @staticmethod

templates/kobo-env/enketo_express/config.json.tpl

@@ -72,6 +72,10 @@
             "domain": "${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}"
         }
     },
+    "logo": {
+        "source": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz48c3ZnIGlkPSJhIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjBweCIgeT0iMHB4IiB3aWR0aD0iMjQzLjYyNCIgaGVpZ2h0PSI2MS44MDYiIHZpZXdCb3g9IjAgMCA0MzQuNDEgNTYuODUiPjxkZWZzPjxzdHlsZT4uYntmaWxsOiMyMDk1ZjM7fS5je2ZpbGw6IzU2NWU3Njt9PC9zdHlsZT48L2RlZnM+PHBhdGggY2xhc3M9ImMiIGQ9Ik0xMTIuODQsMTMuNzNjLTEwLjgsMC0xOS4yLDguNy0xOS4yLDE5LjJzOC43LDE5LjIsMTkuMiwxOS4yLDE5LjItOC43LDE5LjItMTkuMmMuMy0xMC44LTguNC0xOS4yLTE5LjItMTkuMlptMCwzMC45Yy02LjMsMC0xMS40LTUuMS0xMS40LTExLjRzNS4xLTExLjQsMTEuNC0xMS40LDExLjQsNS4xLDExLjQsMTEuNGMuMyw2LTUuMSwxMS40LTExLjQsMTEuNFoiLz48cGF0aCBjbGFzcz0iYyIgZD0iTTE1Ny44NCwxMy43M2MtNS40LS45LTEwLjUsLjYtMTQuNCwzLjZWNy40M2MwLTMtMi40LTUuMS01LjEtNS4xaDBjLTEuNSwwLTIuNCwxLjItMi40LDIuNFY0Ni43M2MwLDMsMi40LDUuMSw1LjEsNS4xaDBjMS41LDAsMi40LTEuMiwyLjQtMi40di0uOWMzLjMsMi40LDcuMiwzLjksMTEuNCwzLjksMTEuNCwwLDIwLjQtOS45LDE5LjItMjEuNi0uOS04LjctNy44LTE1LjYtMTYuMi0xNy4xWm0tNS40LDMwLjZjLTUuNC0xLjItOS02LjMtOS0xMS43aDBjMC00LjIsMi4xLTcuOCw1LjctOS42LDguNy00LjIsMTcuMSwyLjEsMTcuMSwxMC4yLC4zLDYuOS02LjMsMTIuNi0xMy44LDExLjFaIi8+PHBhdGggY2xhc3M9ImMiIGQ9Ik0xOTYuNTQsMTMuNzNjLTEwLjgsMC0xOS4yLDguNy0xOS4yLDE5LjJzOC43LDE5LjIsMTkuMiwxOS4yLDE5LjItOC43LDE5LjItMTkuMmMuMy0xMC44LTguNC0xOS4yLTE5LjItMTkuMlptMCwzMC45Yy02LjMsMC0xMS40LTUuMS0xMS40LTExLjRzNS4xLTExLjQsMTEuNC0xMS40LDExLjQsNS4xLDExLjQsMTEuNGMuMyw2LTUuMSwxMS40LTExLjQsMTEuNFoiLz48Zz48cGF0aCBjbGFzcz0iYiIgZD0iTTI1Ni44NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTI5NS41NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTM0Ni44NCwxNi4xM2MtNC44LDAtOS4zLDEuOC0xMi42LDUuMVY5LjUzYzAtMi4xLTEuOC0zLjYtMy42LTMuNi0uOSwwLTEuOCwuOS0xLjgsMS44VjQ4LjUzYzAsMi4xLDEuOCwzLjYsMy42LDMuNiwuOSwwLDEuOC0uOSwxLjgtMS44di0zYzMuMywzLDcuNSw1LjEsMTIuNiw1LjEsOS45LDAsMTgtOC4xLDE4LTE4cy03LjgtMTguMy0xOC0xOC4zWm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTM4NS44NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTMxOS4yNCw1LjYzYy0uOSwwLTEuOCwuOS0xLjgsMS44VjQ4LjIzYzAsMi4xLDEuOCwzLjYsMy42LDMuNiwuOSwwLDEuOC0uOSwxLjgtMS44VjkuNTNjLjMtMi4xLTEuNS0zLjktMy42LTMuOVoiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTQyMC45NCwzNC4xM2wxMC4yLTE1LjljLjYtLjksMC0yLjQtMS4yLTIuNGgwYy0yLjQsMC00LjgsMS4yLTYsMy4zbC02LDkuNi02LTkuNmMtMS4yLTIuMS0zLjYtMy4zLTYtMy4zLTEuMiwwLTEuOCwxLjItMS4yLDIuNGwxMC4yLDE1LjktMTAuMiwxNS45Yy0uNiwuOSwwLDIuNCwxLjIsMi40aDBjMi40LDAsNC44LTEuMiw2LTMuM2w2LTkuNiw2LDkuNmMxLjIsMi4xLDMuNiwzLjMsNiwzLjNoMGMxLjIsMCwxLjgtMS4yLDEuMi0yLjRsLTEwLjItMTUuOVoiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTI0Mi43NCwxMy43M2MyLjEsMCwzLjYtMS44LDMuNi0zLjYsMC0uOS0uOS0xLjgtMS44LTEuOGgtMjcuM2MtMi4xLDAtMy42LDEuOC0zLjYsMy42LDAsLjksLjksMS44LDEuOCwxLjhoMTEuN1Y0OC41M2MwLDIuMSwxLjgsMy42LDMuNiwzLjYsLjksMCwxLjgtLjksMS44LTEuOFYxMy43M2gxMC4yWiIvPjwvZz48cGF0aCBjbGFzcz0iYyIgZD0iTTk2LjY0LDQ5LjEzbC0yMS4zLTIxLjMsMTcuNy0xNy43Yy45LS45LC4zLTIuNC0uOS0yLjRoLTMuM2MtMi43LDAtNS40LDEuMi03LjIsM2wtMTUuOSwxNS45VjEyLjgzYzAtMy0yLjQtNS4xLTUuMS01LjFoMGMtMS4yLDAtMi40LDEuMi0yLjQsMi40VjQ2LjczYzAsMywyLjQsNS4xLDUuMSw1LjFoMGMxLjIsMCwyLjQtMS4yLDIuNC0yLjR2LTcuOGMwLTIuNCwuOS00LjUsMi40LTZsMi4xLTIuMSwxNSwxNWMxLjgsMS44LDQuNSwzLDcuMiwzaDMuM2MxLjIsMCwxLjgtMS41LC45LTIuNFoiLz48Zz48cGF0aCBjbGFzcz0iYiIgZD0iTTMxLjU0LDM4LjAzdjUuMWMwLDMtMi40LDUuMS01LjEsNS4xSDE1LjM0Yy0zLDAtNS4xLTIuNC01LjEtNS4xVjE0LjMzYzAtMywyLjQtNS4xLDUuMS01LjFoMTEuMWMzLDAsNS4xLDIuNCw1LjEsNS4xdi42Yy45LS4zLDIuMS0uNiwzLS42LDEuMiwwLDIuNCwuMywzLjYsLjZ2LS45YzAtNi42LTUuNC0xMi0xMi0xMkgxNS4wNEM4LjQ0LDIuMDMsMy4wNCw3LjQzLDMuMDQsMTQuMDN2MjguOGMwLDYuNiw1LjQsMTIsMTIsMTJoMTEuMWM2LjYsMCwxMi01LjQsMTItMTJ2LTEyLjlsLTYuNiw4LjFaIi8+PHBhdGggY2xhc3M9ImIiIGQ9Ik0yNi4xNCwzNi4yM2wxMi0xMy44Yy4zLS4zLC4zLTEuMiwwLTEuNWgwYy0yLjEtMS44LTUuNC0xLjUtNy4yLC42bC03LjUsOC43Yy0uMywuMy0uNiwuMy0uNiwwbC0yLjctMy4zYy0uNi0uNi0xLjUtLjYtMi4xLDBoMGMtMS44LDEuOC0xLjgsNC44LS4zLDYuOWwyLjQsMi43YzEuMiwxLjgsNC4yLDEuOCw2LS4zWiIvPjwvZz48L3N2Zz4K",
+        "href": ""
+    },
     "payload limit": "1mb",
     "text field character limit": 1000000,
     "maps": [

templates/kobo-env/envfiles/databases.txt.tpl

@@ -59,3 +59,4 @@ REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_
 REDIS_PASSWORD=${REDIS_PASSWORD}
 CACHE_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/5
 REDIS_CACHE_MAX_MEMORY=${REDIS_CACHE_MAX_MEMORY}
+SERVICE_ACCOUNT_BACKEND_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/6

templates/kobo-env/envfiles/kobocat.txt.tpl

@@ -14,3 +14,6 @@ ${USE_MEDIA_BACKUP}KOBOCAT_MEDIA_BACKUP_SCHEDULE=${KOBOCAT_MEDIA_BACKUP_SCHEDULE
 # Dev: One or more mappings from PyDev remote debugging machine file paths to `kobocat` container
 #   file paths (see https://github.com/kobotoolbox/kobocat/blob/master/docker/setup_pydev.bash).
 #KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS=~/devel/kobocat -> /srv/src/kobocat | ~/.virtualenvs/kobocat/lib/python2.7/site-packages -> /usr/local/lib/python2.7/dist-packages
+
+# Comma separated domains
+${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOCAT_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}

templates/kobo-env/envfiles/kpi.txt.tpl

@@ -7,3 +7,6 @@ KPI_PREFIX=/
 KPI_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1
 
 DJANGO_LANGUAGE_CODES=ar cs de-DE en es fr hi ku pl pt tr zh-hans
+
+# Comma separated domains
+${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}

templates/kobo-env/postgres/primary/postgres.conf.tpl

@@ -7,7 +7,7 @@
 #------------------------------------------------------------------------------------
 # These settings are based on server configuration
 # https://www.pgconfig.org/#/tuning
-# DB Version: 9.5
+# DB Version: 14
 # OS Type: linux
 # App profile: ${POSTGRES_APP_PROFILE}
 # Hard-drive: SSD

templates/kobo-env/postgres/secondary/postgres.conf.tpl

@@ -7,7 +7,7 @@
 #------------------------------------------------------------------------------------
 # These settings are based on server configuration
 # https://www.pgconfig.org/#/tuning
-# DB Version: 9.5
+# DB Version: 14
 # OS Type: linux
 # App profile: ${POSTGRES_APP_PROFILE}
 # Hard-drive: SSD