diff --git a/helpers/config.py b/helpers/config.py index 77ad797..376594b 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -140,6 +140,7 @@ def build(self): self.__questions_google() self.__questions_raven() self.__questions_uwsgi() + self.__questions_service_account() self.__questions_custom_yml() @@ -364,6 +365,7 @@ def get_template(cls): 'multi': False, 'nginx_proxy_port': Config.DEFAULT_PROXY_PORT, 'npm_container': True, + 'postgres_cpus': '1', 'postgres_backup_schedule': '0 2 * * 0', 'postgres_hard_drive_type': 'hdd', 'postgres_max_connections': '100', @@ -373,21 +375,36 @@ def get_template(cls): 'postgres_replication_password': Config.generate_password(), 'postgres_settings': False, 'postgres_settings_content': '\n'.join([ + '# Generated by PGConfig 3.0.0 (3af8ea764b2fd9ea8401acfdc69f90cb825cdee9)', + '# ', + 'https://api.pgconfig.org/v1/tuning/get-config?environment_name=Desktop&format=conf&include_pgbadger=false&cpus=1&max_connections=100&pg_version=14&total_ram=2GB&drive_type=HDD', + '', '# Memory Configuration', - 'shared_buffers = 512MB', - 'effective_cache_size = 2GB', - 'work_mem = 10MB', - 'maintenance_work_mem = 128MB', + 'shared_buffers = 128MB', + 'effective_cache_size = 307MB', + 'work_mem = 419KB', + 'maintenance_work_mem = 20MB', '', '# Checkpoint Related Configuration', - 'min_wal_size = 512MB', - 'max_wal_size = 2GB', + 'min_wal_size = 2GB', + 'max_wal_size = 3GB', 'checkpoint_completion_target = 0.9', - 'wal_buffers = 15MB', + 'wal_buffers = -1', '', '# Network Related Configuration', "listen_addresses = '*'", 'max_connections = 100', + '', + '# Storage Configuration', + 'random_page_cost = 4.0', + 'effective_io_concurrency = 2', + '', + '# Worker Processes Configuration', + 'max_worker_processes = 8', + 'max_parallel_workers_per_gather = 2', + 'max_parallel_workers = 2', + '', + '' ]), 'postgres_user': 'kobo', 'postgresql_port': '5432', @@ -404,6 +421,7 @@ def get_template(cls): 'review_host': True, 'run_redis_containers': True, 'server_role': 'frontend', + 'service_account_whitelisted_hosts': True, 'smtp_host': '', 'smtp_password': '', 'smtp_port': '25', @@ -1580,12 +1598,19 @@ def __questions_postgres(self): break open_port += 1 - # Start pgconfig.org API docker image - docker_command = ['docker', 'run', '--rm', '-p', - f'127.0.0.1:{open_port}:8080', - '-d', '--name', 'pgconfig_container', - 'sebastianwebber/pgconfig-api'] - CLI.run_command(docker_command) + # Unfortunately, the docker image is old and does not support + # PostgreSQL 14. The author has rewritten the code base in Go + # (instead of Python), so chances are pretty low that the image + # gets updated someday. + # ToDo Keep the code below for few months just in case and + # get rid of it if it never happens. + + # # Start pgconfig.org API docker image + # docker_command = ['docker', 'run', '--rm', '-p', + # f'127.0.0.1:{open_port}:8080', + # '-d', '--name', 'pgconfig_container', + # 'sebastianwebber/pgconfig-api'] + # CLI.run_command(docker_command) # From https://docs.pgconfig.org/api/#available-parameters # Parameters are case-sensitive, for example @@ -1597,6 +1622,11 @@ def __questions_postgres(self): # - `Desktop` # It's case-sensitive. + CLI.colored_print('Number of CPUs?', CLI.COLOR_QUESTION) + self.__dict['postgres_cpus'] = CLI.get_response( + r'~^\d+$', + self.__dict['postgres_cpus']) + CLI.colored_print('Total Memory in GB?', CLI.COLOR_QUESTION) self.__dict['postgres_ram'] = CLI.get_response( r'~^\d+$', @@ -1639,17 +1669,24 @@ def __questions_postgres(self): else: self.__dict['postgres_profile'] = 'Mixed' - endpoint = 'http://127.0.0.1:{open_port}/v1/tuning/get-config' \ - '?environment_name={profile}&format=conf' \ - '&include_pgbadger=false' \ - '&max_connections={max_connections}' \ - '&pg_version=9.5' \ - '&total_ram={ram}GB' \ - '&drive_type={drive_type}' + # Instead of using the API locally with the docker image, let's + # use the (unreliable) public API. + # Notes: It has failed several times in the past. + endpoint = ( + 'https://api.pgconfig.org/v1/tuning/get-config' + '?environment_name={profile}&format=conf' + '&include_pgbadger=false' + '&cpus={cpus}' + '&max_connections={max_connections}' + '&pg_version=14' + '&total_ram={ram}GB' + '&drive_type={drive_type}' + ) endpoint = endpoint.format( open_port=open_port, profile=self.__dict['postgres_profile'], ram=self.__dict['postgres_ram'], + cpus=self.__dict['postgres_cpus'], max_connections=self.__dict['postgres_max_connections'], drive_type=self.__dict['postgres_hard_drive_type'].upper() ) @@ -1662,12 +1699,14 @@ def __questions_postgres(self): 'PostgreSQL settings will be used', CLI.COLOR_INFO) - # Stop container - docker_command = ['docker', 'stop', '-t', '0', - 'pgconfig_container'] - CLI.run_command(docker_command) - CLI.colored_print('pgconfig.org API container has been stopped!', - CLI.COLOR_INFO) + # ToDo Keep the code below for few months just in case and + # get rid of it if it never happens. + # # Stop container + # docker_command = ['docker', 'stop', '-t', '0', + # 'pgconfig_container'] + # CLI.run_command(docker_command) + # CLI.colored_print('pgconfig.org API container has been stopped!', + # CLI.COLOR_INFO) else: # Forcing the default settings to remain even if there # is an existing value in .run.conf. Without this, @@ -2037,6 +2076,16 @@ def __questions_secret_keys(self): to_lower=False, error_msg='Too short. 10 characters minimum.') + def __questions_service_account(self): + if not self.local_install: + self.__dict['service_account_whitelisted_hosts'] = CLI.yes_no_question( + 'Do you want to restrict API calls between KPI and KoBoCAT ' + 'to their internal domain names?', + default=self.__dict['service_account_whitelisted_hosts'] + ) + else: + self.__dict['service_account_whitelisted_hosts'] = False + def __questions_smtp(self): self.__dict['smtp_host'] = CLI.colored_input('SMTP server?', CLI.COLOR_QUESTION, diff --git a/helpers/template.py b/helpers/template.py index 1d24411..6c975e9 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -311,6 +311,11 @@ def _get_value(property_, true_value='', false_value='#', comparison_value='', ), 'USE_LETSENSCRYPT': '#' if config.use_letsencrypt else '', + 'USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS': ( + '#' + if config.local_install + else _get_value('service_account_whitelisted_hosts') + ), } @staticmethod diff --git a/templates/kobo-env/enketo_express/config.json.tpl b/templates/kobo-env/enketo_express/config.json.tpl index f13fe9b..e862086 100644 --- a/templates/kobo-env/enketo_express/config.json.tpl +++ b/templates/kobo-env/enketo_express/config.json.tpl @@ -72,6 +72,10 @@ "domain": "${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}" } }, + "logo": { + "source": "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz48c3ZnIGlkPSJhIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjBweCIgeT0iMHB4IiB3aWR0aD0iMjQzLjYyNCIgaGVpZ2h0PSI2MS44MDYiIHZpZXdCb3g9IjAgMCA0MzQuNDEgNTYuODUiPjxkZWZzPjxzdHlsZT4uYntmaWxsOiMyMDk1ZjM7fS5je2ZpbGw6IzU2NWU3Njt9PC9zdHlsZT48L2RlZnM+PHBhdGggY2xhc3M9ImMiIGQ9Ik0xMTIuODQsMTMuNzNjLTEwLjgsMC0xOS4yLDguNy0xOS4yLDE5LjJzOC43LDE5LjIsMTkuMiwxOS4yLDE5LjItOC43LDE5LjItMTkuMmMuMy0xMC44LTguNC0xOS4yLTE5LjItMTkuMlptMCwzMC45Yy02LjMsMC0xMS40LTUuMS0xMS40LTExLjRzNS4xLTExLjQsMTEuNC0xMS40LDExLjQsNS4xLDExLjQsMTEuNGMuMyw2LTUuMSwxMS40LTExLjQsMTEuNFoiLz48cGF0aCBjbGFzcz0iYyIgZD0iTTE1Ny44NCwxMy43M2MtNS40LS45LTEwLjUsLjYtMTQuNCwzLjZWNy40M2MwLTMtMi40LTUuMS01LjEtNS4xaDBjLTEuNSwwLTIuNCwxLjItMi40LDIuNFY0Ni43M2MwLDMsMi40LDUuMSw1LjEsNS4xaDBjMS41LDAsMi40LTEuMiwyLjQtMi40di0uOWMzLjMsMi40LDcuMiwzLjksMTEuNCwzLjksMTEuNCwwLDIwLjQtOS45LDE5LjItMjEuNi0uOS04LjctNy44LTE1LjYtMTYuMi0xNy4xWm0tNS40LDMwLjZjLTUuNC0xLjItOS02LjMtOS0xMS43aDBjMC00LjIsMi4xLTcuOCw1LjctOS42LDguNy00LjIsMTcuMSwyLjEsMTcuMSwxMC4yLC4zLDYuOS02LjMsMTIuNi0xMy44LDExLjFaIi8+PHBhdGggY2xhc3M9ImMiIGQ9Ik0xOTYuNTQsMTMuNzNjLTEwLjgsMC0xOS4yLDguNy0xOS4yLDE5LjJzOC43LDE5LjIsMTkuMiwxOS4yLDE5LjItOC43LDE5LjItMTkuMmMuMy0xMC44LTguNC0xOS4yLTE5LjItMTkuMlptMCwzMC45Yy02LjMsMC0xMS40LTUuMS0xMS40LTExLjRzNS4xLTExLjQsMTEuNC0xMS40LDExLjQsNS4xLDExLjQsMTEuNGMuMyw2LTUuMSwxMS40LTExLjQsMTEuNFoiLz48Zz48cGF0aCBjbGFzcz0iYiIgZD0iTTI1Ni44NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTI5NS41NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTM0Ni44NCwxNi4xM2MtNC44LDAtOS4zLDEuOC0xMi42LDUuMVY5LjUzYzAtMi4xLTEuOC0zLjYtMy42LTMuNi0uOSwwLTEuOCwuOS0xLjgsMS44VjQ4LjUzYzAsMi4xLDEuOCwzLjYsMy42LDMuNiwuOSwwLDEuOC0uOSwxLjgtMS44di0zYzMuMywzLDcuNSw1LjEsMTIuNiw1LjEsOS45LDAsMTgtOC4xLDE4LTE4cy03LjgtMTguMy0xOC0xOC4zWm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTM4NS44NCwxNi4xM2MtOS45LDAtMTgsOC4xLTE4LDE4czguMSwxOCwxOCwxOCwxOC04LjEsMTgtMTgtOC4xLTE4LTE4LTE4Wm0wLDMwLjZjLTYuOSwwLTEyLjYtNS43LTEyLjYtMTIuNnM1LjctMTIuNiwxMi42LTEyLjYsMTIuNiw1LjcsMTIuNiwxMi42LTUuNywxMi42LTEyLjYsMTIuNloiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTMxOS4yNCw1LjYzYy0uOSwwLTEuOCwuOS0xLjgsMS44VjQ4LjIzYzAsMi4xLDEuOCwzLjYsMy42LDMuNiwuOSwwLDEuOC0uOSwxLjgtMS44VjkuNTNjLjMtMi4xLTEuNS0zLjktMy42LTMuOVoiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTQyMC45NCwzNC4xM2wxMC4yLTE1LjljLjYtLjksMC0yLjQtMS4yLTIuNGgwYy0yLjQsMC00LjgsMS4yLTYsMy4zbC02LDkuNi02LTkuNmMtMS4yLTIuMS0zLjYtMy4zLTYtMy4zLTEuMiwwLTEuOCwxLjItMS4yLDIuNGwxMC4yLDE1LjktMTAuMiwxNS45Yy0uNiwuOSwwLDIuNCwxLjIsMi40aDBjMi40LDAsNC44LTEuMiw2LTMuM2w2LTkuNiw2LDkuNmMxLjIsMi4xLDMuNiwzLjMsNiwzLjNoMGMxLjIsMCwxLjgtMS4yLDEuMi0yLjRsLTEwLjItMTUuOVoiLz48cGF0aCBjbGFzcz0iYiIgZD0iTTI0Mi43NCwxMy43M2MyLjEsMCwzLjYtMS44LDMuNi0zLjYsMC0uOS0uOS0xLjgtMS44LTEuOGgtMjcuM2MtMi4xLDAtMy42LDEuOC0zLjYsMy42LDAsLjksLjksMS44LDEuOCwxLjhoMTEuN1Y0OC41M2MwLDIuMSwxLjgsMy42LDMuNiwzLjYsLjksMCwxLjgtLjksMS44LTEuOFYxMy43M2gxMC4yWiIvPjwvZz48cGF0aCBjbGFzcz0iYyIgZD0iTTk2LjY0LDQ5LjEzbC0yMS4zLTIxLjMsMTcuNy0xNy43Yy45LS45LC4zLTIuNC0uOS0yLjRoLTMuM2MtMi43LDAtNS40LDEuMi03LjIsM2wtMTUuOSwxNS45VjEyLjgzYzAtMy0yLjQtNS4xLTUuMS01LjFoMGMtMS4yLDAtMi40LDEuMi0yLjQsMi40VjQ2LjczYzAsMywyLjQsNS4xLDUuMSw1LjFoMGMxLjIsMCwyLjQtMS4yLDIuNC0yLjR2LTcuOGMwLTIuNCwuOS00LjUsMi40LTZsMi4xLTIuMSwxNSwxNWMxLjgsMS44LDQuNSwzLDcuMiwzaDMuM2MxLjIsMCwxLjgtMS41LC45LTIuNFoiLz48Zz48cGF0aCBjbGFzcz0iYiIgZD0iTTMxLjU0LDM4LjAzdjUuMWMwLDMtMi40LDUuMS01LjEsNS4xSDE1LjM0Yy0zLDAtNS4xLTIuNC01LjEtNS4xVjE0LjMzYzAtMywyLjQtNS4xLDUuMS01LjFoMTEuMWMzLDAsNS4xLDIuNCw1LjEsNS4xdi42Yy45LS4zLDIuMS0uNiwzLS42LDEuMiwwLDIuNCwuMywzLjYsLjZ2LS45YzAtNi42LTUuNC0xMi0xMi0xMkgxNS4wNEM4LjQ0LDIuMDMsMy4wNCw3LjQzLDMuMDQsMTQuMDN2MjguOGMwLDYuNiw1LjQsMTIsMTIsMTJoMTEuMWM2LjYsMCwxMi01LjQsMTItMTJ2LTEyLjlsLTYuNiw4LjFaIi8+PHBhdGggY2xhc3M9ImIiIGQ9Ik0yNi4xNCwzNi4yM2wxMi0xMy44Yy4zLS4zLC4zLTEuMiwwLTEuNWgwYy0yLjEtMS44LTUuNC0xLjUtNy4yLC42bC03LjUsOC43Yy0uMywuMy0uNiwuMy0uNiwwbC0yLjctMy4zYy0uNi0uNi0xLjUtLjYtMi4xLDBoMGMtMS44LDEuOC0xLjgsNC44LS4zLDYuOWwyLjQsMi43YzEuMiwxLjgsNC4yLDEuOCw2LS4zWiIvPjwvZz48L3N2Zz4K", + "href": "" + }, "payload limit": "1mb", "text field character limit": 1000000, "maps": [ diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index b207e58..6c4ed23 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -59,3 +59,4 @@ REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_ REDIS_PASSWORD=${REDIS_PASSWORD} CACHE_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/5 REDIS_CACHE_MAX_MEMORY=${REDIS_CACHE_MAX_MEMORY} +SERVICE_ACCOUNT_BACKEND_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/6 diff --git a/templates/kobo-env/envfiles/kobocat.txt.tpl b/templates/kobo-env/envfiles/kobocat.txt.tpl index 0915d3c..b0fe7d7 100644 --- a/templates/kobo-env/envfiles/kobocat.txt.tpl +++ b/templates/kobo-env/envfiles/kobocat.txt.tpl @@ -14,3 +14,6 @@ ${USE_MEDIA_BACKUP}KOBOCAT_MEDIA_BACKUP_SCHEDULE=${KOBOCAT_MEDIA_BACKUP_SCHEDULE # Dev: One or more mappings from PyDev remote debugging machine file paths to `kobocat` container # file paths (see https://github.com/kobotoolbox/kobocat/blob/master/docker/setup_pydev.bash). #KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS=~/devel/kobocat -> /srv/src/kobocat | ~/.virtualenvs/kobocat/lib/python2.7/site-packages -> /usr/local/lib/python2.7/dist-packages + +# Comma separated domains +${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOCAT_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} diff --git a/templates/kobo-env/envfiles/kpi.txt.tpl b/templates/kobo-env/envfiles/kpi.txt.tpl index 611189b..107c877 100644 --- a/templates/kobo-env/envfiles/kpi.txt.tpl +++ b/templates/kobo-env/envfiles/kpi.txt.tpl @@ -7,3 +7,6 @@ KPI_PREFIX=/ KPI_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1 DJANGO_LANGUAGE_CODES=ar cs de-DE en es fr hi ku pl pt tr zh-hans + +# Comma separated domains +${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/primary/postgres.conf.tpl index 141c8df..bd23009 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/primary/postgres.conf.tpl @@ -7,7 +7,7 @@ #------------------------------------------------------------------------------------ # These settings are based on server configuration # https://www.pgconfig.org/#/tuning -# DB Version: 9.5 +# DB Version: 14 # OS Type: linux # App profile: ${POSTGRES_APP_PROFILE} # Hard-drive: SSD diff --git a/templates/kobo-env/postgres/secondary/postgres.conf.tpl b/templates/kobo-env/postgres/secondary/postgres.conf.tpl index aad98af..e2c8493 100644 --- a/templates/kobo-env/postgres/secondary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/secondary/postgres.conf.tpl @@ -7,7 +7,7 @@ #------------------------------------------------------------------------------------ # These settings are based on server configuration # https://www.pgconfig.org/#/tuning -# DB Version: 9.5 +# DB Version: 14 # OS Type: linux # App profile: ${POSTGRES_APP_PROFILE} # Hard-drive: SSD