From 2bb1df6e3d0961040585f67ee0c78678e3f628a8 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 27 Aug 2021 05:20:24 -0400 Subject: [PATCH 01/23] `--update` stays on the current branch and update tip when no version is specified --- helpers/config.py | 20 ++++++++++---------- helpers/updater.py | 13 +++++++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 889859f..ab21133 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -288,6 +288,16 @@ def generate_password(cls): def get_dict(self): return self.__dict + def get_service_names(self): + service_list_command = ['docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + 'config', '--services'] + + services = CLI.run_command(service_list_command, + self.__dict['kobodocker_path']) + return services.strip().split('\n') + @classmethod def get_template(cls): @@ -432,16 +442,6 @@ def get_template(cls): } # Keep properties sorted alphabetically - def get_service_names(self): - service_list_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - 'config', '--services'] - - services = CLI.run_command(service_list_command, - self.__dict['kobodocker_path']) - return services.strip().split('\n') - @property def is_secure(self): return self.__dict['https'] is True diff --git a/helpers/updater.py b/helpers/updater.py index c60c5b6..f1142d7 100644 --- a/helpers/updater.py +++ b/helpers/updater.py @@ -2,8 +2,8 @@ import os import sys -from helpers.setup import Setup from helpers.cli import CLI +from helpers.setup import Setup class Updater: @@ -14,10 +14,19 @@ class Updater: NO_UPDATE_SELF_OPTION = '--no-update-self' @classmethod - def run(cls, version='master', cron=False, update_self=True): + def run(cls, version=None, cron=False, update_self=True): # Validate kobo-docker already exists and is valid Setup.validate_already_run() + if version is None: + git_commit_version_command = [ + 'git', + 'rev-parse', + '--abbrev-ref', + 'HEAD', + ] + version = CLI.run_command(git_commit_version_command).strip() + if update_self: # Update kobo-install first Setup.update_koboinstall(version) From 768bc0f171e7b29d2d7d360570ad241130500a15 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 27 Aug 2021 06:00:26 -0400 Subject: [PATCH 02/23] Bump version to pre6.0.0 --- helpers/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/config.py b/helpers/config.py index ab21133..24147ea 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -31,7 +31,7 @@ class Config(metaclass=Singleton): DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' KOBO_DOCKER_BRANCH = '2.021.30' - KOBO_INSTALL_VERSION = '5.0.0' + KOBO_INSTALL_VERSION = 'pre-6.0.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): From e0e1d77cfc977a8d49ed5350aabbe2be05dbf267 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 Sep 2021 14:09:19 -0400 Subject: [PATCH 03/23] Add more question to redis section to move containers secondary back end and cap the memory of the cache container --- helpers/config.py | 27 ++++++++++++++--- helpers/template.py | 16 ++++++++-- ...r-compose.backend.primary.override.yml.tpl | 29 ++++++++++--------- ...compose.backend.secondary.override.yml.tpl | 12 ++++++-- templates/kobo-env/envfiles/databases.txt.tpl | 2 ++ 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 0125679..ddc846a 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -30,8 +30,8 @@ class Config(metaclass=Singleton): DEFAULT_PROXY_PORT = '8080' DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' - KOBO_DOCKER_BRANCH = '2.021.34a' - KOBO_INSTALL_VERSION = '5.0.1' + KOBO_DOCKER_BRANCH = '317-customize-redis-cache-maxmemory' + KOBO_INSTALL_VERSION = '5.1.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): @@ -293,6 +293,7 @@ def get_template(cls): primary_ip = Network.get_primary_ip() + # Keep properties sorted alphabetically return { 'advanced': False, 'aws_access_key': '', @@ -402,10 +403,12 @@ def get_template(cls): 'public_domain_name': 'kobo.local', 'raven_settings': False, 'redis_backup_schedule': '0 3 * * 0', + 'redis_cache_max_memory': '', 'redis_cache_port': '6380', 'redis_main_port': '6379', 'redis_password': Config.generate_password(), 'review_host': True, + 'run_redis_containers': True, 'server_role': 'frontend', 'smtp_host': '', 'smtp_password': '', @@ -430,7 +433,7 @@ def get_template(cls): 'uwsgi_workers_max': '2', 'uwsgi_workers_start': '1', } - # Keep properties sorted alphabetically + def get_service_names(self): service_list_command = ['docker-compose', @@ -1721,7 +1724,15 @@ def __questions_redis(self): - primary back end - single server installation """ - if self.primary_backend or not self.multi_servers: + if self.multi_servers: + self.__dict['run_redis_containers'] = CLI.yes_no_question( + 'Do you want to run the redis containers from this server?', + default=self.__dict['run_redis_containers'] + ) + else: + self.__dict['run_redis_containers'] = True + + if self.__dict['run_redis_containers']: CLI.colored_print('Redis password?', CLI.COLOR_QUESTION) self.__dict['redis_password'] = CLI.get_response( r'~^.{8,}|$', @@ -1743,6 +1754,14 @@ def __questions_redis(self): if response is False: self.__questions_redis() + CLI.colored_print('Max memory for redis cache container (in MB)?', + CLI.COLOR_QUESTION) + CLI.colored_print('Leave empty for no limits', + CLI.COLOR_INFO) + self.__dict['redis_cache_max_memory'] = CLI.get_response( + r'~^(\d+|-)?$', + self.__dict['redis_cache_max_memory']) + def __questions_reverse_proxy(self): if self.is_secure: diff --git a/helpers/template.py b/helpers/template.py index 46c4cc9..f8a9c00 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -145,9 +145,11 @@ def __get_template_variables(config): def _get_value(property_, true_value='', false_value='#', comparison_value=True): - return true_value \ - if dict_[property_] == comparison_value \ + return ( + true_value + if dict_[property_] == comparison_value else false_value + ) if config.proxy: nginx_port = dict_['nginx_proxy_port'] @@ -232,6 +234,7 @@ def _get_value(property_, true_value='', false_value='#', 'MONGO_PORT': dict_['mongo_port'], 'REDIS_MAIN_PORT': dict_['redis_main_port'], 'REDIS_CACHE_PORT': dict_['redis_cache_port'], + 'REDIS_CACHE_MAX_MEMORY': dict_['redis_cache_max_memory'], 'USE_BACKUP': '' if dict_['use_backup'] else '#', 'USE_WAL_E': _get_value('use_wal_e'), 'USE_AWS_BACKUP': '' if (config.aws and @@ -296,7 +299,14 @@ def _get_value(property_, true_value='', false_value='#', 'local_installation', true_value='true', false_value='false' - ) + ), + 'RUN_REDIS_CONTAINERS': _get_value('run_redis_containers'), + 'USE_REDIS_CACHE_MAX_MEMORY': _get_value( + 'redis_cache_max_memory', + true_value='#', + false_value='', + comparison_value='', + ), } @staticmethod diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl index f7a8277..9653d36 100644 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl @@ -23,18 +23,19 @@ services: ${USE_BACKEND_NETWORK} aliases: ${USE_BACKEND_NETWORK} - mongo.${PRIVATE_DOMAIN_NAME} - redis_main: - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} + ${RUN_REDIS_CONTAINERS}redis_main: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} - redis_cache: - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} + ${RUN_REDIS_CONTAINERS}redis_cache: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 + ${RUN_REDIS_CONTAINERS} ${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index d839393..9c09d18 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -3,9 +3,6 @@ version: '2.2' services: postgres: - extends: - file: docker-compose.backend.template.yml - service: postgres volumes: - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf ports: @@ -13,3 +10,12 @@ services: ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${ADD_BACKEND_EXTRA_HOSTS}- primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + + ${RUN_REDIS_CONTAINERS}redis_main: + ${RUN_REDIS_CONTAINERS}ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_MAIN_PORT}:6379 + + ${RUN_REDIS_CONTAINERS}redis_cache: + ${RUN_REDIS_CONTAINERS}ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 + ${RUN_REDIS_CONTAINERS}${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index 789bfa4..7204e6a 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -55,3 +55,5 @@ ${USE_BACKUP}REDIS_BACKUP_SCHEDULE=${REDIS_BACKUP_SCHEDULE} REDIS_SESSION_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/2 REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/3 REDIS_PASSWORD=${REDIS_PASSWORD} + +REDIS_CACHE_MAX_MEMORY=${REDIS_CACHE_MAX_MEMORY} From 3b1b16941f6213f6b5383f70e7387dff0fc148be Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 24 Sep 2021 11:01:10 -0400 Subject: [PATCH 04/23] Fixed: letsencrypt process is triggered when env is back end but should not --- helpers/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 4012bfa..e41c939 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -444,13 +444,12 @@ def get_template(cls): 'uwsgi_workers_start': '1', } - @property def is_secure(self): return self.__dict['https'] is True def init_letsencrypt(self): - if self.use_letsencrypt: + if self.frontend and self.use_letsencrypt: reverse_proxy_path = self.get_letsencrypt_repo_path() reverse_proxy_command = [ '/bin/bash', From e4cf637b84bac6db07570060dca90b0c2014b9ee Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 24 Sep 2021 16:12:26 -0400 Subject: [PATCH 05/23] Activate Redis containers from overrides --- ...ker-compose.backend.primary.override.yml.tpl | 12 +++++++++++- ...r-compose.backend.secondary.override.yml.tpl | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl index 9653d36..00f5fd5 100644 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl @@ -1,4 +1,4 @@ -# For public, HTTPS servers. +# Override for primary back-end server version: '2.2' services: @@ -24,6 +24,9 @@ services: ${USE_BACKEND_NETWORK} - mongo.${PRIVATE_DOMAIN_NAME} ${RUN_REDIS_CONTAINERS}redis_main: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_main ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: @@ -32,6 +35,9 @@ services: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} ${RUN_REDIS_CONTAINERS}redis_cache: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_cache ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 ${RUN_REDIS_CONTAINERS} ${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M @@ -39,3 +45,7 @@ services: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} + +${USE_BACKEND_NETWORK}networks: +${USE_BACKEND_NETWORK} kobo-be-network: +${USE_BACKEND_NETWORK} driver: bridge diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index 9c09d18..acbdcdd 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -12,10 +12,17 @@ services: ${ADD_BACKEND_EXTRA_HOSTS}- primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${RUN_REDIS_CONTAINERS}redis_main: - ${RUN_REDIS_CONTAINERS}ports: - ${RUN_REDIS_CONTAINERS} - ${REDIS_MAIN_PORT}:6379 + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_main + ${RUN_REDIS_CONTAINERS} ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_MAIN_PORT}:6379 ${RUN_REDIS_CONTAINERS}redis_cache: - ${RUN_REDIS_CONTAINERS}ports: - ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 - ${RUN_REDIS_CONTAINERS}${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_cache + ${RUN_REDIS_CONTAINERS} ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 + ${RUN_REDIS_CONTAINERS} ${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M + From dafc38bb52305fb3c5a3da71b92e0c9750835a99 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 24 Sep 2021 16:16:26 -0400 Subject: [PATCH 06/23] Give Redis a capital R --- helpers/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index e41c939..1bf94eb 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -1595,7 +1595,7 @@ def reset_ports(): if not self.multi_servers: self.__dict['expose_backend_ports'] = CLI.yes_no_question( 'Do you want to expose back-end container ports ' - '(`PostgreSQL`, `MongoDB`, `redis`)?', + '(`PostgreSQL`, `MongoDB`, `Redis`)?', default=self.__dict['expose_backend_ports'] ) else: @@ -1719,13 +1719,13 @@ def __questions_raven(self): def __questions_redis(self): """ - Ask for redis password only when server is for: + Ask for Redis password only when server is for: - primary back end - single server installation """ if self.multi_servers: self.__dict['run_redis_containers'] = CLI.yes_no_question( - 'Do you want to run the redis containers from this server?', + 'Do you want to run the Redis containers from this server?', default=self.__dict['run_redis_containers'] ) else: @@ -1753,7 +1753,7 @@ def __questions_redis(self): if response is False: self.__questions_redis() - CLI.colored_print('Max memory for redis cache container (in MB)?', + CLI.colored_print('Max memory (MB) for Redis cache container?', CLI.COLOR_QUESTION) CLI.colored_print('Leave empty for no limits', CLI.COLOR_INFO) From 0c8630b7526abf5e681948d31b5645ea2a286ed6 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 24 Sep 2021 16:16:59 -0400 Subject: [PATCH 07/23] Make indentation consistent --- ...compose.backend.secondary.override.yml.tpl | 4 +- .../docker-compose.frontend.override.yml.tpl | 80 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index acbdcdd..743e9eb 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -8,8 +8,8 @@ services: ports: - ${POSTGRES_PORT}:5432 ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${RUN_REDIS_CONTAINERS}redis_main: ${RUN_REDIS_CONTAINERS} extends: diff --git a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl index 84c135a..677896c 100644 --- a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl @@ -3,10 +3,10 @@ version: '3' services: kobocat: - ${USE_KC_DEV_MODE}build: ${KC_PATH} - ${USE_KC_DEV_MODE}image: kobocat:dev.${KC_DEV_BUILD_ID} - ${USE_KC_DEV_MODE}volumes: - ${USE_KC_DEV_MODE} - ${KC_PATH}:/srv/src/kobocat + ${USE_KC_DEV_MODE} build: ${KC_PATH} + ${USE_KC_DEV_MODE} image: kobocat:dev.${KC_DEV_BUILD_ID} + ${USE_KC_DEV_MODE} volumes: + ${USE_KC_DEV_MODE} - ${KC_PATH}:/srv/src/kobocat environment: - ENKETO_PROTOCOL=${PROTOCOL} - KC_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} @@ -19,13 +19,13 @@ services: ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=onadata.settings.dev ${USE_CELERY}- SKIP_CELERY=True ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: @@ -33,10 +33,10 @@ services: ${USE_BACKEND_NETWORK} - kobocat.docker.container kpi: - ${USE_KPI_DEV_MODE}build: ${KPI_PATH} - ${USE_KPI_DEV_MODE}image: kpi:dev.${KPI_DEV_BUILD_ID} - ${USE_KPI_DEV_MODE}volumes: - ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi environment: - KPI_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} - KPI_UWSGI_CHEAPER_WORKERS_COUNT=${UWSGI_WORKERS_START} @@ -45,18 +45,18 @@ services: - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT} - KPI_UWSGI_HARAKIRI=${UWSGI_HARAKIRI} - KPI_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY} - ${USE_CELERY}- SKIP_CELERY=True - ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=kobo.settings.dev - ${USE_HTTPS}- SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO, https - ${USE_NPM_FROM_HOST}- FRONTEND_DEV_MODE=host + ${USE_CELERY} - SKIP_CELERY=True + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO, https + ${USE_NPM_FROM_HOST} - FRONTEND_DEV_MODE=host ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: @@ -70,13 +70,13 @@ services: ports: - ${NGINX_EXPOSED_PORT}:80 ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} networks: kobo-fe-network: aliases: @@ -91,13 +91,13 @@ services: environment: - DUMMY_ENV=True ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: From ec849a081700d5af92ee46405189b819753fdb4f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 5 Oct 2021 11:21:29 -0400 Subject: [PATCH 08/23] Remove Redis docker container memory limit setting --- .../kobo-docker/docker-compose.backend.primary.override.yml.tpl | 1 - .../docker-compose.backend.secondary.override.yml.tpl | 1 - 2 files changed, 2 deletions(-) diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl index 00f5fd5..72d37da 100644 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl @@ -40,7 +40,6 @@ services: ${RUN_REDIS_CONTAINERS} service: redis_cache ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 - ${RUN_REDIS_CONTAINERS} ${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index 743e9eb..2fb7992 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -24,5 +24,4 @@ services: ${RUN_REDIS_CONTAINERS} service: redis_cache ${RUN_REDIS_CONTAINERS} ports: ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 - ${RUN_REDIS_CONTAINERS} ${USE_REDIS_CACHE_MAX_MEMORY}mem_limit: ${REDIS_CACHE_MAX_MEMORY}M From b8fd5898127fd155a655e2b3265a46af36c442fc Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 5 Oct 2021 11:22:54 -0400 Subject: [PATCH 09/23] Use kobo-docker `beta` branch --- helpers/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/config.py b/helpers/config.py index 1bf94eb..78caf08 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -30,7 +30,7 @@ class Config(metaclass=Singleton): DEFAULT_PROXY_PORT = '8080' DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' - KOBO_DOCKER_BRANCH = '317-customize-redis-cache-maxmemory' + KOBO_DOCKER_BRANCH = 'beta' KOBO_INSTALL_VERSION = '6.1.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 From 3191cbf3239aa19808b0eaa670ffb8dd20e6efb2 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 8 Oct 2021 15:00:56 -0400 Subject: [PATCH 10/23] Add questions for custom docker configuration --- helpers/config.py | 69 ++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 78caf08..172210c 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -31,7 +31,7 @@ class Config(metaclass=Singleton): DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' KOBO_DOCKER_BRANCH = 'beta' - KOBO_INSTALL_VERSION = '6.1.0' + KOBO_INSTALL_VERSION = '6.2.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): @@ -76,8 +76,7 @@ def aws(self): @property def backend(self): - return not self.multi_servers or self.primary_backend or \ - self.secondary_backend + return not self.multi_servers or not self.frontend @property def block_common_http_ports(self): @@ -154,16 +153,6 @@ def get_upgraded_dict(self): return upgraded_dict - @property - def backend_questions(self): - """ - Checks whether questions are back end only - - Returns: - bool - """ - return not self.multi_servers or not self.frontend - def build(self): """ Build configuration based on user's answers @@ -199,12 +188,12 @@ def build(self): else: self.__reset(fake_dns=True) - if self.frontend_questions: + if self.frontend: self.__questions_public_routes() self.__questions_https() self.__questions_reverse_proxy() - if self.frontend_questions: + if self.frontend: self.__questions_smtp() self.__questions_super_user_credentials() @@ -216,12 +205,15 @@ def build(self): self.__questions_redis() self.__questions_ports() - if self.frontend_questions: + if self.frontend: self.__questions_secret_keys() self.__questions_aws() self.__questions_google() self.__questions_raven() self.__questions_uwsgi() + + self.__questions_custom_yml() + else: self.__secure_mongo() @@ -256,18 +248,9 @@ def frontend(self): dict: all values from user's responses needed to create configuration files """ - return not self.multi_servers or \ - self.__dict['server_role'] == 'frontend' - - @property - def frontend_questions(self): - """ - Checks whether questions are front-end only - - Returns: - bool - """ - return not self.multi_servers or self.frontend + return ( + not self.multi_servers or self.__dict['server_role'] == 'frontend' + ) @classmethod def generate_password(cls): @@ -431,7 +414,9 @@ def get_template(cls): 'two_databases': True, 'use_aws': False, 'use_backup': False, + 'use_backend_custom_yml': False, 'use_celery': True, + 'use_frontend_custom_yml': False, 'use_letsencrypt': True, 'use_private_dns': False, 'use_wal_e': False, @@ -927,7 +912,7 @@ def __questions_backup(self): """ Asks all questions about backups. """ - if self.backend_questions or (self.frontend_questions and not self.aws): + if self.backend or (self.frontend and not self.aws): self.__dict['use_backup'] = CLI.yes_no_question( 'Do you want to activate backups?', @@ -936,7 +921,7 @@ def __questions_backup(self): if self.__dict['use_backup']: if self.advanced_options: - if self.backend_questions and not self.frontend_questions: + if self.backend and not self.frontend: self.__questions_aws() # Prompting user whether they want to use WAL-E for @@ -972,7 +957,7 @@ def __questions_backup(self): ) CLI.framed_print(message, color=CLI.COLOR_INFO) - if self.frontend_questions and not self.aws: + if self.frontend and not self.aws: CLI.colored_print('KoBoCat media backup schedule?', CLI.COLOR_QUESTION) self.__dict[ @@ -980,7 +965,7 @@ def __questions_backup(self): '~{}'.format(schedule_regex_pattern), self.__dict['kobocat_media_backup_schedule']) - if self.backend_questions: + if self.backend: if self.__dict['use_wal_e'] is True: self.__dict['backup_from_primary'] = True else: @@ -1035,6 +1020,22 @@ def __questions_backup(self): else: self.__reset(no_backups=True) + def __questions_custom_yml(self): + + if self.frontend: + self.__dict['use_frontend_custom_yml'] = CLI.yes_no_question( + 'Do you want to add additional settings to the front-end ' + 'docker containers?', + default=self.__dict['use_frontend_custom_yml'], + ) + + if self.backend: + self.__dict['use_backend_custom_yml'] = CLI.yes_no_question( + 'Do you want to add additional settings to the backend-end ' + 'docker containers?', + default=self.__dict['use_backend_custom_yml'] + ) + def __questions_dev_mode(self): """ Asks for developer/staging mode. @@ -1045,7 +1046,7 @@ def __questions_dev_mode(self): Reset to default in case of No """ - if self.frontend_questions: + if self.frontend: if self.local_install: # NGINX different port @@ -1447,7 +1448,7 @@ def __questions_postgres(self): self.__write_upsert_db_users_trigger_file(content, 'postgres') - if self.backend_questions: + if self.backend: # Postgres settings self.__dict['postgres_settings'] = CLI.yes_no_question( 'Do you want to tweak PostgreSQL settings?', From e8c2d149b098c5d448109d090355f69b8e250ed1 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 8 Oct 2021 15:01:58 -0400 Subject: [PATCH 11/23] Reorder Config.build() alphabetically --- helpers/config.py | 140 +++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 172210c..f998e3e 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -78,6 +78,76 @@ def aws(self): def backend(self): return not self.multi_servers or not self.frontend + def build(self): + """ + Build configuration based on user's answers + + Returns: + dict: all values from user's responses needed to create + configuration files + """ + if not Network.get_primary_ip(): + message = ( + 'No valid networks detected. Can not continue!\n' + 'Please connect to a network and re-run the command.' + ) + CLI.framed_print(message, color=CLI.COLOR_ERROR) + sys.exit(1) + else: + + self.__welcome() + self.__dict = self.get_upgraded_dict() + + self.__create_directory() + self.__questions_advanced_options() + self.__questions_installation_type() + self.__detect_network() + + if not self.local_install: + if self.advanced_options: + self.__questions_multi_servers() + if self.multi_servers: + self.__questions_roles() + if self.frontend or self.secondary_backend: + self.__questions_private_routes() + else: + self.__reset(fake_dns=True) + + if self.frontend: + self.__questions_public_routes() + self.__questions_https() + self.__questions_reverse_proxy() + + if self.frontend: + self.__questions_smtp() + self.__questions_super_user_credentials() + + if self.advanced_options: + self.__questions_docker_prefix() + self.__questions_dev_mode() + self.__questions_postgres() + self.__questions_mongo() + self.__questions_redis() + self.__questions_ports() + + if self.frontend: + self.__questions_secret_keys() + self.__questions_aws() + self.__questions_google() + self.__questions_raven() + self.__questions_uwsgi() + + self.__questions_custom_yml() + + else: + self.__secure_mongo() + + self.__questions_backup() + + self.write_config() + + return self.__dict + @property def block_common_http_ports(self): return self.use_letsencrypt or self.__dict['block_common_http_ports'] @@ -153,76 +223,6 @@ def get_upgraded_dict(self): return upgraded_dict - def build(self): - """ - Build configuration based on user's answers - - Returns: - dict: all values from user's responses needed to create - configuration files - """ - if not Network.get_primary_ip(): - message = ( - 'No valid networks detected. Can not continue!\n' - 'Please connect to a network and re-run the command.' - ) - CLI.framed_print(message, color=CLI.COLOR_ERROR) - sys.exit(1) - else: - - self.__welcome() - self.__dict = self.get_upgraded_dict() - - self.__create_directory() - self.__questions_advanced_options() - self.__questions_installation_type() - self.__detect_network() - - if not self.local_install: - if self.advanced_options: - self.__questions_multi_servers() - if self.multi_servers: - self.__questions_roles() - if self.frontend or self.secondary_backend: - self.__questions_private_routes() - else: - self.__reset(fake_dns=True) - - if self.frontend: - self.__questions_public_routes() - self.__questions_https() - self.__questions_reverse_proxy() - - if self.frontend: - self.__questions_smtp() - self.__questions_super_user_credentials() - - if self.advanced_options: - self.__questions_docker_prefix() - self.__questions_dev_mode() - self.__questions_postgres() - self.__questions_mongo() - self.__questions_redis() - self.__questions_ports() - - if self.frontend: - self.__questions_secret_keys() - self.__questions_aws() - self.__questions_google() - self.__questions_raven() - self.__questions_uwsgi() - - self.__questions_custom_yml() - - else: - self.__secure_mongo() - - self.__questions_backup() - - self.write_config() - - return self.__dict - @property def dev_mode(self): return self.__dict['dev_mode'] is True From 4bdc351ef29b46226127e34b7edc67be1053e2d7 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 8 Oct 2021 15:05:39 -0400 Subject: [PATCH 12/23] Missing replacements after refactor frontend and backend properties of class Config --- tests/test_config.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 46ad0aa..5a0c679 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -116,8 +116,8 @@ def test_dev_mode(): def test_server_roles_questions(): config = read_config() - assert config.frontend_questions - assert config.backend_questions + assert config.frontend + assert config.backend with patch('helpers.cli.CLI.colored_input') as mock_colored_input: mock_colored_input.side_effect = iter( @@ -126,12 +126,12 @@ def test_server_roles_questions(): config._Config__questions_multi_servers() config._Config__questions_roles() - assert config.frontend_questions - assert not config.backend_questions + assert config.frontend + assert not config.backend config._Config__questions_roles() - assert not config.frontend_questions - assert config.backend_questions + assert not config.frontend + assert config.backend assert config.secondary_backend From bf91a64602316531c487962b91daf00defd71733 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 8 Oct 2021 15:06:48 -0400 Subject: [PATCH 13/23] Validate existence of custom yml file and inject it before starting containers --- helpers/command.py | 207 ++++++++++++++++++++++++++++++--------------- 1 file changed, 137 insertions(+), 70 deletions(-) diff --git a/helpers/command.py b/helpers/command.py index fea6dff..1048ebe 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import os import sys import time import subprocess @@ -109,6 +109,8 @@ def compose_frontend(cls, args): '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend')] + + cls.__validate_custom_yml(config, command) command.extend(args) subprocess.call(command, cwd=dict_['kobodocker_path']) @@ -123,6 +125,7 @@ def compose_backend(cls, args): '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend') ] + cls.__validate_custom_yml(config, command) command.extend(args) subprocess.call(command, cwd=dict_['kobodocker_path']) @@ -225,31 +228,27 @@ def logs(cls): if config.primary_backend or config.secondary_backend: backend_role = dict_['backend_server_role'] - backend_command = ['docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format( - backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format( - backend_role), - '-p', - config.get_prefix('backend'), - 'logs', - '-f' - ] - CLI.run_command(backend_command, - dict_['kobodocker_path'], - True) + backend_command = [ + 'docker-compose', + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-p', config.get_prefix('backend'), + 'logs', '-f', + ] + cls.__validate_custom_yml(config, backend_command) + CLI.run_command(backend_command, dict_['kobodocker_path'], True) if config.frontend: - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'logs', '-f'] - CLI.run_command(frontend_command, - dict_['kobodocker_path'], - True) + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'logs', '-f', + ] + + cls.__validate_custom_yml(config, frontend_command) + CLI.run_command(frontend_command, dict_['kobodocker_path'], True) @classmethod def configure_maintenance(cls): @@ -270,12 +269,15 @@ def stop_nginx(cls): config = Config() dict_ = config.get_dict() - nginx_stop_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'stop', 'nginx'] + nginx_stop_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'stop', 'nginx', + ] + cls.__validate_custom_yml(config, nginx_stop_command) CLI.run_command(nginx_stop_command, dict_['kobodocker_path']) @classmethod @@ -283,11 +285,13 @@ def start_maintenance(cls): config = Config() dict_ = config.get_dict() - frontend_command = ['docker-compose', - '-f', 'docker-compose.maintenance.yml', - '-f', 'docker-compose.maintenance.override.yml', - '-p', config.get_prefix('maintenance'), - 'up', '-d'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.maintenance.yml', + '-f', 'docker-compose.maintenance.override.yml', + '-p', config.get_prefix('maintenance'), + 'up', '-d', + ] CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('Maintenance mode has been started', @@ -341,15 +345,13 @@ def start(cls, frontend_only=False): backend_command = [ 'docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format(backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format(backend_role), - '-p', - config.get_prefix('backend'), - 'up', - '-d' + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-p', config.get_prefix('backend'), + 'up', '-d' ] + + cls.__validate_custom_yml(config, backend_command) CLI.run_command(backend_command, dict_['kobodocker_path']) # Start the front-end containers @@ -359,11 +361,13 @@ def start(cls, frontend_only=False): # separate databases for KPI and KoBoCAT Upgrading.migrate_single_to_two_databases(config) - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'up', '-d'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'up', '-d', + ] if dict_['maintenance_enabled']: cls.start_maintenance() @@ -372,12 +376,12 @@ def start(cls, frontend_only=False): s for s in config.get_service_names() if s != 'nginx' ]) + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) # Start reverse proxy if user uses it. if config.use_letsencrypt: - proxy_command = ['docker-compose', - 'up', '-d'] + proxy_command = ['docker-compose', 'up', '-d'] CLI.run_command(proxy_command, config.get_letsencrypt_repo_path()) @@ -421,32 +425,35 @@ def stop(cls, output=True, frontend_only=False): dict_['kobodocker_path']) # Shut down front-end containers - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'down'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'down', + ] + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) # Stop reverse proxy if user uses it. if config.use_letsencrypt: - proxy_command = ['docker-compose', - 'down'] - CLI.run_command(proxy_command, - config.get_letsencrypt_repo_path()) + proxy_command = ['docker-compose', 'down'] + CLI.run_command( + proxy_command, config.get_letsencrypt_repo_path() + ) if not frontend_only and config.backend: backend_role = dict_['backend_server_role'] backend_command = [ 'docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format(backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend'), 'down' ] + + cls.__validate_custom_yml(config, backend_command) CLI.run_command(backend_command, dict_['kobodocker_path']) if output: @@ -467,17 +474,22 @@ def stop_maintenance(cls): '-f', 'docker-compose.maintenance.yml', '-f', 'docker-compose.maintenance.override.yml', '-p', config.get_prefix('maintenance'), - 'down'] + 'down' + ] - CLI.run_command(maintenance_down_command, - dict_['kobodocker_path']) + CLI.run_command(maintenance_down_command, dict_['kobodocker_path']) # Create and start NGINX container - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'up', '-d', 'nginx'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'up', '-d', + 'nginx', + ] + + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('Maintenance mode has been stopped', @@ -495,3 +507,58 @@ def version(cls): Config.KOBO_INSTALL_VERSION, stdout.strip()[0:7], ), CLI.COLOR_SUCCESS) + + @staticmethod + def __validate_custom_yml(config, command): + """ + Validate whether docker-compose must starts the containers with a user's + custom YML file in addition to already existing ones and especially if + this file already exists. If it does not, kobo-install is paused until + the user resumes it manually. + + If user has chosen to use a custom YML file, it is injected in `command` + before being sent to shell. + """ + dict_ = config.get_dict() + frontend_command = True + # Detect if it's a front-end command or back-end command + for part in command: + if 'backend' in part: + frontend_command = False + break + + if frontend_command and dict_['use_frontend_custom_yml']: + custom_file = '{}/docker-compose.frontend.custom.yml'.format( + dict_['kobodocker_path'] + ) + if not os.path.exists(custom_file): + message = ( + 'Please create your custom configuration in\n' + '`{custom_file}`.' + ).format(custom_file=custom_file) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=75) + input('Press any key when it is done...') + + # Add custom file to docker-compose command + command.insert(5, '-f') + command.insert(6, 'docker-compose.frontend.custom.yml') + + if not frontend_command and dict_['use_backend_custom_yml']: + backend_server_role = dict_['backend_server_role'] + custom_file = '{}/docker-compose.backend.{}.custom.yml'.format( + dict_['kobodocker_path'], + backend_server_role + ) + if not os.path.exists(custom_file): + message = ( + 'Please create your custom configuration in\n' + '`{custom_file}`.' + ).format(custom_file=custom_file) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=75) + input('Press any key when it is done...') + + # Add custom file to docker-compose command + command.insert(5, '-f') + command.insert( + 6, 'docker-compose.backend.{}.custom.yml'.format(backend_server_role) + ) From 1de1783601c2b251c5858949f5595887099e8481 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 8 Oct 2021 18:53:45 -0400 Subject: [PATCH 14/23] Apply requested changes for PR#172 --- helpers/command.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/helpers/command.py b/helpers/command.py index 1048ebe..5bb9df3 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -511,13 +511,12 @@ def version(cls): @staticmethod def __validate_custom_yml(config, command): """ - Validate whether docker-compose must starts the containers with a user's - custom YML file in addition to already existing ones and especially if - this file already exists. If it does not, kobo-install is paused until - the user resumes it manually. + Validate whether docker-compose must start the containers with a + custom YML file in addition to the default. If the file does not yet exist, + kobo-install is paused until the user creates it and resumes the setup manually. - If user has chosen to use a custom YML file, it is injected in `command` - before being sent to shell. + If user has chosen to use a custom YML file, it is injected into `command` + before being executed. """ dict_ = config.get_dict() frontend_command = True @@ -531,13 +530,16 @@ def __validate_custom_yml(config, command): custom_file = '{}/docker-compose.frontend.custom.yml'.format( dict_['kobodocker_path'] ) - if not os.path.exists(custom_file): + + does_custom_file_exist = os.path.exists(custom_file) + while not does_custom_file_exist: message = ( 'Please create your custom configuration in\n' '`{custom_file}`.' ).format(custom_file=custom_file) - CLI.framed_print(message, color=CLI.COLOR_INFO, columns=75) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') + does_custom_file_exist = os.path.exists(custom_file) # Add custom file to docker-compose command command.insert(5, '-f') @@ -549,13 +551,17 @@ def __validate_custom_yml(config, command): dict_['kobodocker_path'], backend_server_role ) - if not os.path.exists(custom_file): + + does_custom_file_exist = os.path.exists(custom_file) + while not does_custom_file_exist: message = ( 'Please create your custom configuration in\n' '`{custom_file}`.' ).format(custom_file=custom_file) - CLI.framed_print(message, color=CLI.COLOR_INFO, columns=75) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') + does_custom_file_exist = os.path.exists(custom_file) + # Add custom file to docker-compose command command.insert(5, '-f') From 86d70319c7c5ada5eca24d097b6b25ebcaebb211 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Sun, 10 Oct 2021 09:17:00 -0400 Subject: [PATCH 15/23] Use format() instead of f-strings --- helpers/aws_validation.py | 6 +++++- helpers/command.py | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/helpers/aws_validation.py b/helpers/aws_validation.py index 965f568..fbdd421 100644 --- a/helpers/aws_validation.py +++ b/helpers/aws_validation.py @@ -48,7 +48,11 @@ def _get_request_url_and_headers(self): canonical_querystring = self.REQUEST_PARAMETERS canonical_headers = '\n'.join( - [f'host:{self.HOST}', f'x-amz-date:{amzdate}', ''] + [ + 'host:{host}'.format(host=self.HOST), + 'x-amz-date:{amzdate}'.format(amzdate=amzdate), + '', + ] ) canonical_request = '\n'.join( diff --git a/helpers/command.py b/helpers/command.py index 5bb9df3..24da0d9 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -562,7 +562,6 @@ def __validate_custom_yml(config, command): input('Press any key when it is done...') does_custom_file_exist = os.path.exists(custom_file) - # Add custom file to docker-compose command command.insert(5, '-f') command.insert( From 107e2e75c527577261580905ba7cbcfb068ceaff Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Oct 2021 14:44:24 -0400 Subject: [PATCH 16/23] Allow empty schedule cron (to deactivate backup of specific service), fix schedule backup logic for secondary back end) --- helpers/config.py | 86 ++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index a26bd2d..a6b4320 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -331,7 +331,6 @@ def get_template(cls): 'kc_postgres_db': 'kobocat', 'kc_subdomain': 'kc', 'kobocat_media_backup_schedule': '0 0 * * 0', - 'kobocat_media_schedule': '0 0 * * 0', 'kobocat_raven': '', 'kobodocker_path': os.path.realpath(os.path.normpath(os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -463,9 +462,11 @@ def primary_backend(self): Returns: bool """ - return self.multi_servers and \ - self.__dict['server_role'] == 'backend' and \ - self.__dict['backend_server_role'] == 'primary' + return ( + self.multi_servers + and self.__dict['server_role'] == 'backend' + and self.__dict['backend_server_role'] == 'primary' + ) @property def multi_servers(self): @@ -550,9 +551,11 @@ def secondary_backend(self): Returns: bool """ - return self.multi_servers and \ - self.__dict['server_role'] == 'backend' and \ - self.__dict['backend_server_role'] == 'secondary' + return ( + self.multi_servers + and self.__dict['server_role'] == 'backend' + and self.__dict['backend_server_role'] == 'secondary' + ) def set_config(self, value): self.__dict = value @@ -944,8 +947,9 @@ def __questions_backup(self): self.__dict['use_wal_e'] = False schedule_regex_pattern = ( - r'^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))' - r'(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$') + r'^\-|((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))' + r'(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})?$' + ) message = ( 'Schedules use linux cron syntax with UTC datetimes.\n' 'For example, schedule at 12:00 AM E.S.T every Sunday ' @@ -956,7 +960,11 @@ def __questions_backup(self): 'cron schedule.' ) CLI.framed_print(message, color=CLI.COLOR_INFO) - + CLI.colored_print( + 'Leave empty (or use `-` to empty) to deactivate backups' + ' for a specific\nservice.', + color=CLI.COLOR_WARNING + ) if self.frontend and not self.aws: CLI.colored_print('KoBoCat media backup schedule?', CLI.COLOR_QUESTION) @@ -966,34 +974,38 @@ def __questions_backup(self): self.__dict['kobocat_media_backup_schedule']) if self.backend: - if self.__dict['use_wal_e'] is True: + if self.__dict['use_wal_e'] or not self.multi_servers: + # We are on primary back-end server self.__dict['backup_from_primary'] = True + backup_postgres = True else: if self.primary_backend: - response = CLI.yes_no_question( - 'Run PostgreSQL backup from primary ' - 'backend server?', - default=self.__dict['backup_from_primary'] - ) - self.__dict['backup_from_primary'] = response - elif self.secondary_backend: - self.__dict['backup_from_primary'] = False + default_response = self.__dict['backup_from_primary'] else: - self.__dict['backup_from_primary'] = True + default_response = not self.__dict[ + 'backup_from_primary'] - backup_from_primary = self.__dict['backup_from_primary'] + backup_postgres = CLI.yes_no_question( + 'Run PostgreSQL backup from this server?', + default=default_response + ) - if (not self.multi_servers or - (self.primary_backend and backup_from_primary) - or - (self.secondary_backend and - not backup_from_primary)): + if self.primary_backend: + self.__dict['backup_from_primary'] = backup_postgres + else: + self.__dict['backup_from_primary'] = not backup_postgres + + print('Run PostgreSQL backup from this server?', backup_postgres) + + if backup_postgres: CLI.colored_print('PostgreSQL backup schedule?', CLI.COLOR_QUESTION) self.__dict[ 'postgres_backup_schedule'] = CLI.get_response( '~{}'.format(schedule_regex_pattern), self.__dict['postgres_backup_schedule']) + else: + self.__dict['postgres_backup_schedule'] = '' if self.primary_backend or not self.multi_servers: CLI.colored_print('MongoDB backup schedule?', @@ -1003,12 +1015,15 @@ def __questions_backup(self): '~{}'.format(schedule_regex_pattern), self.__dict['mongo_backup_schedule']) + if self.__dict['run_redis_containers']: CLI.colored_print('Redis backup schedule?', CLI.COLOR_QUESTION) self.__dict[ 'redis_backup_schedule'] = CLI.get_response( '~{}'.format(schedule_regex_pattern), self.__dict['redis_backup_schedule']) + else: + self.__dict['redis_backup_schedule'] = '' if self.aws: self.__questions_aws_backup_settings() @@ -1031,7 +1046,7 @@ def __questions_custom_yml(self): if self.backend: self.__dict['use_backend_custom_yml'] = CLI.yes_no_question( - 'Do you want to add additional settings to the backend-end ' + 'Do you want to add additional settings to the back-end ' 'docker containers?', default=self.__dict['use_backend_custom_yml'] ) @@ -1724,7 +1739,7 @@ def __questions_redis(self): - primary back end - single server installation """ - if self.multi_servers: + if self.backend: self.__dict['run_redis_containers'] = CLI.yes_no_question( 'Do you want to run the Redis containers from this server?', default=self.__dict['run_redis_containers'] @@ -1754,13 +1769,14 @@ def __questions_redis(self): if response is False: self.__questions_redis() - CLI.colored_print('Max memory (MB) for Redis cache container?', - CLI.COLOR_QUESTION) - CLI.colored_print('Leave empty for no limits', - CLI.COLOR_INFO) - self.__dict['redis_cache_max_memory'] = CLI.get_response( - r'~^(\d+|-)?$', - self.__dict['redis_cache_max_memory']) + if self.backend: + CLI.colored_print('Max memory (MB) for Redis cache container?', + CLI.COLOR_QUESTION) + CLI.colored_print('Leave empty for no limits', + CLI.COLOR_INFO) + self.__dict['redis_cache_max_memory'] = CLI.get_response( + r'~^(\d+|-)?$', + self.__dict['redis_cache_max_memory']) def __questions_reverse_proxy(self): From 215626d8095c6cf1dcc1a01847d819628d4dc389 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Oct 2021 14:44:51 -0400 Subject: [PATCH 17/23] Add tests to validate backup schedule logic --- tests/test_config.py | 189 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 5a0c679..7523b9f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -152,6 +152,7 @@ def test_use_https(): assert config.local_install assert not config.is_secure + def _aws_validation_setup(): config = read_config() @@ -160,6 +161,7 @@ def _aws_validation_setup(): return config + def test_aws_credentials_invalid_with_no_configuration(): config = _aws_validation_setup() @@ -168,6 +170,7 @@ def test_aws_credentials_invalid_with_no_configuration(): assert not config._Config__dict['use_aws'] assert not config._Config__dict['aws_credentials_valid'] + def test_aws_validation_fails_with_system_exit(): config = _aws_validation_setup() @@ -181,6 +184,7 @@ def test_aws_validation_fails_with_system_exit(): pass assert not config._Config__dict['aws_credentials_valid'] + def test_aws_invalid_credentials_continue_without_validation(): config = _aws_validation_setup() @@ -189,6 +193,7 @@ def test_aws_invalid_credentials_continue_without_validation(): config._Config__questions_aws() assert not config._Config__dict['aws_credentials_valid'] + @patch('helpers.aws_validation.AWSValidation.validate_credentials', new=MockAWSValidation.validate_credentials) def test_aws_validation_passes_with_valid_credentials(): @@ -265,6 +270,7 @@ def test_aws_validation_passes_with_valid_credentials(): config._Config__questions_aws() assert config._Config__dict['aws_credentials_valid'] + @patch('helpers.config.Config._Config__clone_repo', MagicMock(return_value=True)) def test_proxy_letsencrypt(): @@ -761,3 +767,186 @@ def test_use_boolean(): for property_ in boolean_properties: assert dict_[property_] == expected_dict[property_] + +def test_backup_schedules_from_single_instance(): + config = read_config() + # Force advanced options and single instance + config._Config__dict['advanced'] = True + config._Config__dict['multi'] = False + # Force `False` to validate it becomes `True` at the end + config._Config__dict['backup_from_primary'] = False + + assert config._Config__dict['kobocat_media_backup_schedule'] == '0 0 * * 0' + assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + '1 1 1 1 1', # KoBoCAT media + '2 2 2 2 2', # PostgreSQL + '3 3 3 3 3', # Mongo + '4 4 4 4 4', # Redis + ]) + config._Config__questions_backup() + assert config._Config__dict['kobocat_media_backup_schedule'] == '1 1 1 1 1' + assert config._Config__dict['postgres_backup_schedule'] == '2 2 2 2 2' + assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' + assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' + assert config._Config__dict['backup_from_primary'] is True + + +def test_backup_schedules_from_frontend_instance(): + config = read_config() + # Force advanced options + config._Config__dict['advanced'] = True + + assert config._Config__dict['kobocat_media_backup_schedule'] == '0 0 * * 0' + + with patch('helpers.cli.CLI.colored_input') as mock_colored_input: + mock_colored_input.side_effect = iter( + [CHOICE_YES, 'frontend'] + ) + config._Config__questions_multi_servers() + config._Config__questions_roles() + + assert config.frontend + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + '1 1 1 1 1', # KoBoCAT media + ]) + config._Config__questions_backup() + assert config._Config__dict['kobocat_media_backup_schedule'] == '1 1 1 1 1' + + +def test_backup_schedules_from_primary_backend_with_redis_and_no_postgres(): + config = read_config() + # Force advanced options + config._Config__dict['advanced'] = True + + assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + assert config._Config__dict['run_redis_containers'] is True + assert config._Config__dict['backup_from_primary'] is True + + with patch('helpers.cli.CLI.colored_input') as mock_colored_input: + mock_colored_input.side_effect = iter( + [CHOICE_YES, 'backend', 'primary']) + config._Config__questions_multi_servers() + config._Config__questions_roles() + assert config.backend and config.primary_backend + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + CHOICE_NO, # Choose AWS + CHOICE_NO, # Run postgres backup from current server + '3 3 3 3 3', # Mongo + '4 4 4 4 4', # Redis + ]) + config._Config__questions_backup() + + assert config._Config__dict['postgres_backup_schedule'] == '' + assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' + assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' + assert config._Config__dict['backup_from_primary'] is False + + +def test_backup_schedules_from_primary_backend_with_no_redis_and_no_postgres(): + config = read_config() + # Force advanced options + config._Config__dict['advanced'] = True + config._Config__dict['run_redis_containers'] = False + + assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + assert config._Config__dict['backup_from_primary'] is True + + with patch('helpers.cli.CLI.colored_input') as mock_colored_input: + mock_colored_input.side_effect = iter( + [CHOICE_YES, 'backend', 'primary']) + config._Config__questions_multi_servers() + config._Config__questions_roles() + assert config.backend and config.primary_backend + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + CHOICE_NO, # Choose AWS + CHOICE_NO, # Run postgres backup from current server + '3 3 3 3 3', # Mongo + ]) + config._Config__questions_backup() + + assert config._Config__dict['postgres_backup_schedule'] == '' + assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' + assert config._Config__dict['redis_backup_schedule'] == '' + assert config._Config__dict['backup_from_primary'] is False + + +def test_backup_schedules_from_secondary_backend_with_redis_and_postgres(): + config = read_config() + # Force advanced options + config._Config__dict['advanced'] = True + config._Config__dict['run_redis_containers'] = True + + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + assert config._Config__dict['backup_from_primary'] is True + + with patch('helpers.cli.CLI.colored_input') as mock_colored_input: + mock_colored_input.side_effect = iter( + [CHOICE_YES, 'backend', 'secondary']) + config._Config__questions_multi_servers() + config._Config__questions_roles() + assert config.backend and config.secondary_backend + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + CHOICE_NO, # Choose AWS + CHOICE_YES, # Run postgres backup from current server + '2 2 2 2 2', # PostgreSQL + '4 4 4 4 4', # Redis + ]) + config._Config__questions_backup() + + assert config._Config__dict['postgres_backup_schedule'] == '2 2 2 2 2' + assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' + assert config._Config__dict['backup_from_primary'] is False + + +def test_backup_schedules_from_secondary_backend_with_redis_and_no_postgres(): + config = read_config() + # Force advanced options + config._Config__dict['advanced'] = True + config._Config__dict['run_redis_containers'] = True + + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + + with patch('helpers.cli.CLI.colored_input') as mock_colored_input: + mock_colored_input.side_effect = iter( + [CHOICE_YES, 'backend', 'secondary']) + config._Config__questions_multi_servers() + config._Config__questions_roles() + assert config.backend and config.secondary_backend + + with patch('helpers.cli.CLI.colored_input') as mock_ci: + mock_ci.side_effect = iter([ + CHOICE_YES, # Activate backup + CHOICE_NO, # Choose AWS + CHOICE_NO, # Run postgres backup from current server + '4 4 4 4 4', # Redis + ]) + config._Config__questions_backup() + + assert config._Config__dict['postgres_backup_schedule'] == '' + assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' + assert config._Config__dict['backup_from_primary'] is True + assert config._Config__dict['run_redis_containers'] is True From 27775cad423b87671315320f33c46482aade99f1 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Oct 2021 15:32:30 -0400 Subject: [PATCH 18/23] Bump version to 6.3.0 --- helpers/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/config.py b/helpers/config.py index a6b4320..8f7b5ff 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -31,7 +31,7 @@ class Config(metaclass=Singleton): DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' KOBO_DOCKER_BRANCH = '2.021.41a' - KOBO_INSTALL_VERSION = '6.2.0' + KOBO_INSTALL_VERSION = '6.3.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): From c93ede09be70bb87382c8bcfe2bf716971dd137f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Oct 2021 15:49:31 -0400 Subject: [PATCH 19/23] Remove set of characters to avoid issues with PostgreSQL, Mongo, redis configuration --- helpers/config.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 8f7b5ff..637ee87 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -253,17 +253,18 @@ def frontend(self): ) @classmethod - def generate_password(cls): + def generate_password(cls, required_chars_count=20): """ - Generate 12 characters long random password + Generate n characters long random password Returns: str """ - characters = string.ascii_letters \ - + '!$%+-_^~@#{}[]()/\'\'`~,;:.<>' \ - + string.digits - required_chars_count = 12 + characters = ( + string.ascii_letters + + string.digits + + '!$+-_^~#`~' + ) return ''.join(choice(characters) for _ in range(required_chars_count)) From 200e696b8e5016bc089971d48e60afcc0ae57f6d Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 27 Oct 2021 16:03:37 -0400 Subject: [PATCH 20/23] Bump version to 6.3.1 --- helpers/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/config.py b/helpers/config.py index 637ee87..549dcdd 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -31,7 +31,7 @@ class Config(metaclass=Singleton): DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' KOBO_DOCKER_BRANCH = '2.021.41a' - KOBO_INSTALL_VERSION = '6.3.0' + KOBO_INSTALL_VERSION = '6.3.1' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): From 68db8b9ea85532aa217f2752a0df62c393529a5e Mon Sep 17 00:00:00 2001 From: JacquelineMorrissette Date: Mon, 8 Nov 2021 13:47:48 -0500 Subject: [PATCH 21/23] Updated list to include languages with reviewed content --- templates/kobo-env/envfiles/kpi.txt.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/kobo-env/envfiles/kpi.txt.tpl b/templates/kobo-env/envfiles/kpi.txt.tpl index 1d01252..afae503 100644 --- a/templates/kobo-env/envfiles/kpi.txt.tpl +++ b/templates/kobo-env/envfiles/kpi.txt.tpl @@ -12,4 +12,4 @@ KPI_MONGO_NAME=formhub KPI_MONGO_USER=${MONGO_USER_USERNAME} KPI_MONGO_PASS=${MONGO_USER_PASSWORD} -DJANGO_LANGUAGE_CODES=en ar es fr hi ku pl pt zh-hans +DJANGO_LANGUAGE_CODES=en ar es de-DE fr hi ku pl pt tr zh-hans From 2cd21e743a9a4dc97bfb6583bdd2f498e07cc633 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 12 Nov 2021 17:08:59 -0500 Subject: [PATCH 22/23] Add test to deactivate backups with `-` --- tests/test_config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 7523b9f..97a9dca 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -950,3 +950,30 @@ def test_backup_schedules_from_secondary_backend_with_redis_and_no_postgres(): assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' assert config._Config__dict['backup_from_primary'] is True assert config._Config__dict['run_redis_containers'] is True + +def test_activate_only_postgres_backup(): + config = read_config() + # Force advanced options and single instance + config._Config__dict['advanced'] = True + config._Config__dict['multi'] = False + # Force `False` to validate it becomes `True` at the end + config._Config__dict['backup_from_primary'] = False + + assert config._Config__dict['kobocat_media_backup_schedule'] == '0 0 * * 0' + assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' + assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' + assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' + + with patch('builtins.input') as mock_input: + mock_input.side_effect = iter([ + CHOICE_YES, # Activate backup + '-', # Deactivate KoBoCAT media + '2 2 2 2 2', # Modify PostgreSQL + '-', # Deactivate Mongo + '-', # Deactivate Redis + ]) + config._Config__questions_backup() + assert config._Config__dict['kobocat_media_backup_schedule'] == '' + assert config._Config__dict['postgres_backup_schedule'] == '2 2 2 2 2' + assert config._Config__dict['mongo_backup_schedule'] == '' + assert config._Config__dict['redis_backup_schedule'] == '' From eb2c36d6fbb8c1b2380f68092dd456095ef9837f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Mon, 15 Nov 2021 15:19:07 -0500 Subject: [PATCH 23/23] Remove debug print message --- helpers/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 617143b..58d348f 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -996,8 +996,6 @@ def __questions_backup(self): else: self.__dict['backup_from_primary'] = not backup_postgres - print('Run PostgreSQL backup from this server?', backup_postgres) - if backup_postgres: CLI.colored_print('PostgreSQL backup schedule?', CLI.COLOR_QUESTION)