diff --git a/helpers/aws_validation.py b/helpers/aws_validation.py index 965f568..fbdd421 100644 --- a/helpers/aws_validation.py +++ b/helpers/aws_validation.py @@ -48,7 +48,11 @@ def _get_request_url_and_headers(self): canonical_querystring = self.REQUEST_PARAMETERS canonical_headers = '\n'.join( - [f'host:{self.HOST}', f'x-amz-date:{amzdate}', ''] + [ + 'host:{host}'.format(host=self.HOST), + 'x-amz-date:{amzdate}'.format(amzdate=amzdate), + '', + ] ) canonical_request = '\n'.join( diff --git a/helpers/command.py b/helpers/command.py index fea6dff..24da0d9 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import os import sys import time import subprocess @@ -109,6 +109,8 @@ def compose_frontend(cls, args): '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend')] + + cls.__validate_custom_yml(config, command) command.extend(args) subprocess.call(command, cwd=dict_['kobodocker_path']) @@ -123,6 +125,7 @@ def compose_backend(cls, args): '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend') ] + cls.__validate_custom_yml(config, command) command.extend(args) subprocess.call(command, cwd=dict_['kobodocker_path']) @@ -225,31 +228,27 @@ def logs(cls): if config.primary_backend or config.secondary_backend: backend_role = dict_['backend_server_role'] - backend_command = ['docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format( - backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format( - backend_role), - '-p', - config.get_prefix('backend'), - 'logs', - '-f' - ] - CLI.run_command(backend_command, - dict_['kobodocker_path'], - True) + backend_command = [ + 'docker-compose', + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-p', config.get_prefix('backend'), + 'logs', '-f', + ] + cls.__validate_custom_yml(config, backend_command) + CLI.run_command(backend_command, dict_['kobodocker_path'], True) if config.frontend: - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'logs', '-f'] - CLI.run_command(frontend_command, - dict_['kobodocker_path'], - True) + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'logs', '-f', + ] + + cls.__validate_custom_yml(config, frontend_command) + CLI.run_command(frontend_command, dict_['kobodocker_path'], True) @classmethod def configure_maintenance(cls): @@ -270,12 +269,15 @@ def stop_nginx(cls): config = Config() dict_ = config.get_dict() - nginx_stop_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'stop', 'nginx'] + nginx_stop_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'stop', 'nginx', + ] + cls.__validate_custom_yml(config, nginx_stop_command) CLI.run_command(nginx_stop_command, dict_['kobodocker_path']) @classmethod @@ -283,11 +285,13 @@ def start_maintenance(cls): config = Config() dict_ = config.get_dict() - frontend_command = ['docker-compose', - '-f', 'docker-compose.maintenance.yml', - '-f', 'docker-compose.maintenance.override.yml', - '-p', config.get_prefix('maintenance'), - 'up', '-d'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.maintenance.yml', + '-f', 'docker-compose.maintenance.override.yml', + '-p', config.get_prefix('maintenance'), + 'up', '-d', + ] CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('Maintenance mode has been started', @@ -341,15 +345,13 @@ def start(cls, frontend_only=False): backend_command = [ 'docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format(backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format(backend_role), - '-p', - config.get_prefix('backend'), - 'up', - '-d' + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-p', config.get_prefix('backend'), + 'up', '-d' ] + + cls.__validate_custom_yml(config, backend_command) CLI.run_command(backend_command, dict_['kobodocker_path']) # Start the front-end containers @@ -359,11 +361,13 @@ def start(cls, frontend_only=False): # separate databases for KPI and KoBoCAT Upgrading.migrate_single_to_two_databases(config) - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'up', '-d'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'up', '-d', + ] if dict_['maintenance_enabled']: cls.start_maintenance() @@ -372,12 +376,12 @@ def start(cls, frontend_only=False): s for s in config.get_service_names() if s != 'nginx' ]) + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) # Start reverse proxy if user uses it. if config.use_letsencrypt: - proxy_command = ['docker-compose', - 'up', '-d'] + proxy_command = ['docker-compose', 'up', '-d'] CLI.run_command(proxy_command, config.get_letsencrypt_repo_path()) @@ -421,32 +425,35 @@ def stop(cls, output=True, frontend_only=False): dict_['kobodocker_path']) # Shut down front-end containers - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'down'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'down', + ] + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) # Stop reverse proxy if user uses it. if config.use_letsencrypt: - proxy_command = ['docker-compose', - 'down'] - CLI.run_command(proxy_command, - config.get_letsencrypt_repo_path()) + proxy_command = ['docker-compose', 'down'] + CLI.run_command( + proxy_command, config.get_letsencrypt_repo_path() + ) if not frontend_only and config.backend: backend_role = dict_['backend_server_role'] backend_command = [ 'docker-compose', - '-f', - 'docker-compose.backend.{}.yml'.format(backend_role), - '-f', - 'docker-compose.backend.{}.override.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.yml'.format(backend_role), + '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend'), 'down' ] + + cls.__validate_custom_yml(config, backend_command) CLI.run_command(backend_command, dict_['kobodocker_path']) if output: @@ -467,17 +474,22 @@ def stop_maintenance(cls): '-f', 'docker-compose.maintenance.yml', '-f', 'docker-compose.maintenance.override.yml', '-p', config.get_prefix('maintenance'), - 'down'] + 'down' + ] - CLI.run_command(maintenance_down_command, - dict_['kobodocker_path']) + CLI.run_command(maintenance_down_command, dict_['kobodocker_path']) # Create and start NGINX container - frontend_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'up', '-d', 'nginx'] + frontend_command = [ + 'docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'up', '-d', + 'nginx', + ] + + cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('Maintenance mode has been stopped', @@ -495,3 +507,63 @@ def version(cls): Config.KOBO_INSTALL_VERSION, stdout.strip()[0:7], ), CLI.COLOR_SUCCESS) + + @staticmethod + def __validate_custom_yml(config, command): + """ + Validate whether docker-compose must start the containers with a + custom YML file in addition to the default. If the file does not yet exist, + kobo-install is paused until the user creates it and resumes the setup manually. + + If user has chosen to use a custom YML file, it is injected into `command` + before being executed. + """ + dict_ = config.get_dict() + frontend_command = True + # Detect if it's a front-end command or back-end command + for part in command: + if 'backend' in part: + frontend_command = False + break + + if frontend_command and dict_['use_frontend_custom_yml']: + custom_file = '{}/docker-compose.frontend.custom.yml'.format( + dict_['kobodocker_path'] + ) + + does_custom_file_exist = os.path.exists(custom_file) + while not does_custom_file_exist: + message = ( + 'Please create your custom configuration in\n' + '`{custom_file}`.' + ).format(custom_file=custom_file) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) + input('Press any key when it is done...') + does_custom_file_exist = os.path.exists(custom_file) + + # Add custom file to docker-compose command + command.insert(5, '-f') + command.insert(6, 'docker-compose.frontend.custom.yml') + + if not frontend_command and dict_['use_backend_custom_yml']: + backend_server_role = dict_['backend_server_role'] + custom_file = '{}/docker-compose.backend.{}.custom.yml'.format( + dict_['kobodocker_path'], + backend_server_role + ) + + does_custom_file_exist = os.path.exists(custom_file) + while not does_custom_file_exist: + message = ( + 'Please create your custom configuration in\n' + '`{custom_file}`.' + ).format(custom_file=custom_file) + CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) + input('Press any key when it is done...') + does_custom_file_exist = os.path.exists(custom_file) + + # Add custom file to docker-compose command + command.insert(5, '-f') + command.insert( + 6, 'docker-compose.backend.{}.custom.yml'.format(backend_server_role) + ) diff --git a/helpers/config.py b/helpers/config.py index c7072cb..3ca1196 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -30,8 +30,8 @@ class Config(metaclass=Singleton): DEFAULT_PROXY_PORT = '8080' DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' - KOBO_DOCKER_BRANCH = '2.021.34d' - KOBO_INSTALL_VERSION = '5.0.1' + KOBO_DOCKER_BRANCH = '2.021.41' + KOBO_INSTALL_VERSION = '6.2.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 def __init__(self): @@ -76,8 +76,77 @@ def aws(self): @property def backend(self): - return not self.multi_servers or self.primary_backend or \ - self.secondary_backend + return not self.multi_servers or not self.frontend + + def build(self): + """ + Build configuration based on user's answers + + Returns: + dict: all values from user's responses needed to create + configuration files + """ + if not Network.get_primary_ip(): + message = ( + 'No valid networks detected. Can not continue!\n' + 'Please connect to a network and re-run the command.' + ) + CLI.framed_print(message, color=CLI.COLOR_ERROR) + sys.exit(1) + else: + + self.__welcome() + self.__dict = self.get_upgraded_dict() + + self.__create_directory() + self.__questions_advanced_options() + self.__questions_installation_type() + self.__detect_network() + + if not self.local_install: + if self.advanced_options: + self.__questions_multi_servers() + if self.multi_servers: + self.__questions_roles() + if self.frontend or self.secondary_backend: + self.__questions_private_routes() + else: + self.__reset(fake_dns=True) + + if self.frontend: + self.__questions_public_routes() + self.__questions_https() + self.__questions_reverse_proxy() + + if self.frontend: + self.__questions_smtp() + self.__questions_super_user_credentials() + + if self.advanced_options: + self.__questions_docker_prefix() + self.__questions_dev_mode() + self.__questions_postgres() + self.__questions_mongo() + self.__questions_redis() + self.__questions_ports() + + if self.frontend: + self.__questions_secret_keys() + self.__questions_aws() + self.__questions_google() + self.__questions_raven() + self.__questions_uwsgi() + + self.__questions_custom_yml() + + else: + self.__secure_mongo() + + self.__questions_backup() + + self.write_config() + + return self.__dict @property def block_common_http_ports(self): @@ -154,83 +223,6 @@ def get_upgraded_dict(self): return upgraded_dict - @property - def backend_questions(self): - """ - Checks whether questions are back end only - - Returns: - bool - """ - return not self.multi_servers or not self.frontend - - def build(self): - """ - Build configuration based on user's answers - - Returns: - dict: all values from user's responses needed to create - configuration files - """ - if not Network.get_primary_ip(): - message = ( - 'No valid networks detected. Can not continue!\n' - 'Please connect to a network and re-run the command.' - ) - CLI.framed_print(message, color=CLI.COLOR_ERROR) - sys.exit(1) - else: - - self.__welcome() - self.__dict = self.get_upgraded_dict() - - self.__create_directory() - self.__questions_advanced_options() - self.__questions_installation_type() - self.__detect_network() - - if not self.local_install: - if self.advanced_options: - self.__questions_multi_servers() - if self.multi_servers: - self.__questions_roles() - if self.frontend or self.secondary_backend: - self.__questions_private_routes() - else: - self.__reset(fake_dns=True) - - if self.frontend_questions: - self.__questions_public_routes() - self.__questions_https() - self.__questions_reverse_proxy() - - if self.frontend_questions: - self.__questions_smtp() - self.__questions_super_user_credentials() - - if self.advanced_options: - self.__questions_docker_prefix() - self.__questions_dev_mode() - self.__questions_postgres() - self.__questions_mongo() - self.__questions_redis() - self.__questions_ports() - - if self.frontend_questions: - self.__questions_secret_keys() - self.__questions_aws() - self.__questions_google() - self.__questions_raven() - self.__questions_uwsgi() - else: - self.__secure_mongo() - - self.__questions_backup() - - self.write_config() - - return self.__dict - @property def dev_mode(self): return self.__dict['dev_mode'] is True @@ -256,18 +248,9 @@ def frontend(self): dict: all values from user's responses needed to create configuration files """ - return not self.multi_servers or \ - self.__dict['server_role'] == 'frontend' - - @property - def frontend_questions(self): - """ - Checks whether questions are front-end only - - Returns: - bool - """ - return not self.multi_servers or self.frontend + return ( + not self.multi_servers or self.__dict['server_role'] == 'frontend' + ) @classmethod def generate_password(cls): @@ -288,11 +271,22 @@ def generate_password(cls): def get_dict(self): return self.__dict + def get_service_names(self): + service_list_command = ['docker-compose', + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + 'config', '--services'] + + services = CLI.run_command(service_list_command, + self.__dict['kobodocker_path']) + return services.strip().split('\n') + @classmethod def get_template(cls): primary_ip = Network.get_primary_ip() + # Keep properties sorted alphabetically return { 'advanced': False, 'aws_access_key': '', @@ -402,10 +396,12 @@ def get_template(cls): 'public_domain_name': 'kobo.local', 'raven_settings': False, 'redis_backup_schedule': '0 3 * * 0', + 'redis_cache_max_memory': '', 'redis_cache_port': '6380', 'redis_main_port': '6379', 'redis_password': Config.generate_password(), 'review_host': True, + 'run_redis_containers': True, 'server_role': 'frontend', 'smtp_host': '', 'smtp_password': '', @@ -418,7 +414,9 @@ def get_template(cls): 'two_databases': True, 'use_aws': False, 'use_backup': False, + 'use_backend_custom_yml': False, 'use_celery': True, + 'use_frontend_custom_yml': False, 'use_letsencrypt': True, 'use_private_dns': False, 'use_wal_e': False, @@ -430,24 +428,13 @@ def get_template(cls): 'uwsgi_workers_max': '2', 'uwsgi_workers_start': '1', } - # Keep properties sorted alphabetically - - def get_service_names(self): - service_list_command = ['docker-compose', - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - 'config', '--services'] - - services = CLI.run_command(service_list_command, - self.__dict['kobodocker_path']) - return services.strip().split('\n') @property def is_secure(self): return self.__dict['https'] is True def init_letsencrypt(self): - if self.use_letsencrypt: + if self.frontend and self.use_letsencrypt: reverse_proxy_path = self.get_letsencrypt_repo_path() reverse_proxy_command = [ '/bin/bash', @@ -925,7 +912,7 @@ def __questions_backup(self): """ Asks all questions about backups. """ - if self.backend_questions or (self.frontend_questions and not self.aws): + if self.backend or (self.frontend and not self.aws): self.__dict['use_backup'] = CLI.yes_no_question( 'Do you want to activate backups?', @@ -934,7 +921,7 @@ def __questions_backup(self): if self.__dict['use_backup']: if self.advanced_options: - if self.backend_questions and not self.frontend_questions: + if self.backend and not self.frontend: self.__questions_aws() # Prompting user whether they want to use WAL-E for @@ -970,7 +957,7 @@ def __questions_backup(self): ) CLI.framed_print(message, color=CLI.COLOR_INFO) - if self.frontend_questions and not self.aws: + if self.frontend and not self.aws: CLI.colored_print('KoBoCat media backup schedule?', CLI.COLOR_QUESTION) self.__dict[ @@ -978,7 +965,7 @@ def __questions_backup(self): '~{}'.format(schedule_regex_pattern), self.__dict['kobocat_media_backup_schedule']) - if self.backend_questions: + if self.backend: if self.__dict['use_wal_e'] is True: self.__dict['backup_from_primary'] = True else: @@ -1033,6 +1020,22 @@ def __questions_backup(self): else: self.__reset(no_backups=True) + def __questions_custom_yml(self): + + if self.frontend: + self.__dict['use_frontend_custom_yml'] = CLI.yes_no_question( + 'Do you want to add additional settings to the front-end ' + 'docker containers?', + default=self.__dict['use_frontend_custom_yml'], + ) + + if self.backend: + self.__dict['use_backend_custom_yml'] = CLI.yes_no_question( + 'Do you want to add additional settings to the backend-end ' + 'docker containers?', + default=self.__dict['use_backend_custom_yml'] + ) + def __questions_dev_mode(self): """ Asks for developer/staging mode. @@ -1043,7 +1046,7 @@ def __questions_dev_mode(self): Reset to default in case of No """ - if self.frontend_questions: + if self.frontend: if self.local_install: # NGINX different port @@ -1445,7 +1448,7 @@ def __questions_postgres(self): self.__write_upsert_db_users_trigger_file(content, 'postgres') - if self.backend_questions: + if self.backend: # Postgres settings self.__dict['postgres_settings'] = CLI.yes_no_question( 'Do you want to tweak PostgreSQL settings?', @@ -1593,7 +1596,7 @@ def reset_ports(): if not self.multi_servers: self.__dict['expose_backend_ports'] = CLI.yes_no_question( 'Do you want to expose back-end container ports ' - '(`PostgreSQL`, `MongoDB`, `redis`)?', + '(`PostgreSQL`, `MongoDB`, `Redis`)?', default=self.__dict['expose_backend_ports'] ) else: @@ -1717,11 +1720,19 @@ def __questions_raven(self): def __questions_redis(self): """ - Ask for redis password only when server is for: + Ask for Redis password only when server is for: - primary back end - single server installation """ - if self.primary_backend or not self.multi_servers: + if self.multi_servers: + self.__dict['run_redis_containers'] = CLI.yes_no_question( + 'Do you want to run the Redis containers from this server?', + default=self.__dict['run_redis_containers'] + ) + else: + self.__dict['run_redis_containers'] = True + + if self.__dict['run_redis_containers']: CLI.colored_print('Redis password?', CLI.COLOR_QUESTION) self.__dict['redis_password'] = CLI.get_response( r'~^.{8,}|$', @@ -1743,6 +1754,14 @@ def __questions_redis(self): if response is False: self.__questions_redis() + CLI.colored_print('Max memory (MB) for Redis cache container?', + CLI.COLOR_QUESTION) + CLI.colored_print('Leave empty for no limits', + CLI.COLOR_INFO) + self.__dict['redis_cache_max_memory'] = CLI.get_response( + r'~^(\d+|-)?$', + self.__dict['redis_cache_max_memory']) + def __questions_reverse_proxy(self): if self.is_secure: diff --git a/helpers/template.py b/helpers/template.py index 46c4cc9..f8a9c00 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -145,9 +145,11 @@ def __get_template_variables(config): def _get_value(property_, true_value='', false_value='#', comparison_value=True): - return true_value \ - if dict_[property_] == comparison_value \ + return ( + true_value + if dict_[property_] == comparison_value else false_value + ) if config.proxy: nginx_port = dict_['nginx_proxy_port'] @@ -232,6 +234,7 @@ def _get_value(property_, true_value='', false_value='#', 'MONGO_PORT': dict_['mongo_port'], 'REDIS_MAIN_PORT': dict_['redis_main_port'], 'REDIS_CACHE_PORT': dict_['redis_cache_port'], + 'REDIS_CACHE_MAX_MEMORY': dict_['redis_cache_max_memory'], 'USE_BACKUP': '' if dict_['use_backup'] else '#', 'USE_WAL_E': _get_value('use_wal_e'), 'USE_AWS_BACKUP': '' if (config.aws and @@ -296,7 +299,14 @@ def _get_value(property_, true_value='', false_value='#', 'local_installation', true_value='true', false_value='false' - ) + ), + 'RUN_REDIS_CONTAINERS': _get_value('run_redis_containers'), + 'USE_REDIS_CACHE_MAX_MEMORY': _get_value( + 'redis_cache_max_memory', + true_value='#', + false_value='', + comparison_value='', + ), } @staticmethod diff --git a/helpers/updater.py b/helpers/updater.py index c60c5b6..f1142d7 100644 --- a/helpers/updater.py +++ b/helpers/updater.py @@ -2,8 +2,8 @@ import os import sys -from helpers.setup import Setup from helpers.cli import CLI +from helpers.setup import Setup class Updater: @@ -14,10 +14,19 @@ class Updater: NO_UPDATE_SELF_OPTION = '--no-update-self' @classmethod - def run(cls, version='master', cron=False, update_self=True): + def run(cls, version=None, cron=False, update_self=True): # Validate kobo-docker already exists and is valid Setup.validate_already_run() + if version is None: + git_commit_version_command = [ + 'git', + 'rev-parse', + '--abbrev-ref', + 'HEAD', + ] + version = CLI.run_command(git_commit_version_command).strip() + if update_self: # Update kobo-install first Setup.update_koboinstall(version) diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl index f7a8277..72d37da 100644 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl @@ -1,4 +1,4 @@ -# For public, HTTPS servers. +# Override for primary back-end server version: '2.2' services: @@ -23,18 +23,28 @@ services: ${USE_BACKEND_NETWORK} aliases: ${USE_BACKEND_NETWORK} - mongo.${PRIVATE_DOMAIN_NAME} - redis_main: - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} + ${RUN_REDIS_CONTAINERS}redis_main: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_main + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} - redis_cache: - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} + ${RUN_REDIS_CONTAINERS}redis_cache: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_cache + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: + ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: + ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} + +${USE_BACKEND_NETWORK}networks: +${USE_BACKEND_NETWORK} kobo-be-network: +${USE_BACKEND_NETWORK} driver: bridge diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index d839393..2fb7992 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -3,13 +3,25 @@ version: '2.2' services: postgres: - extends: - file: docker-compose.backend.template.yml - service: postgres volumes: - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf ports: - ${POSTGRES_PORT}:5432 ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + + ${RUN_REDIS_CONTAINERS}redis_main: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_main + ${RUN_REDIS_CONTAINERS} ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_MAIN_PORT}:6379 + + ${RUN_REDIS_CONTAINERS}redis_cache: + ${RUN_REDIS_CONTAINERS} extends: + ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml + ${RUN_REDIS_CONTAINERS} service: redis_cache + ${RUN_REDIS_CONTAINERS} ports: + ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 + diff --git a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl index 84c135a..677896c 100644 --- a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl @@ -3,10 +3,10 @@ version: '3' services: kobocat: - ${USE_KC_DEV_MODE}build: ${KC_PATH} - ${USE_KC_DEV_MODE}image: kobocat:dev.${KC_DEV_BUILD_ID} - ${USE_KC_DEV_MODE}volumes: - ${USE_KC_DEV_MODE} - ${KC_PATH}:/srv/src/kobocat + ${USE_KC_DEV_MODE} build: ${KC_PATH} + ${USE_KC_DEV_MODE} image: kobocat:dev.${KC_DEV_BUILD_ID} + ${USE_KC_DEV_MODE} volumes: + ${USE_KC_DEV_MODE} - ${KC_PATH}:/srv/src/kobocat environment: - ENKETO_PROTOCOL=${PROTOCOL} - KC_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} @@ -19,13 +19,13 @@ services: ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=onadata.settings.dev ${USE_CELERY}- SKIP_CELERY=True ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: @@ -33,10 +33,10 @@ services: ${USE_BACKEND_NETWORK} - kobocat.docker.container kpi: - ${USE_KPI_DEV_MODE}build: ${KPI_PATH} - ${USE_KPI_DEV_MODE}image: kpi:dev.${KPI_DEV_BUILD_ID} - ${USE_KPI_DEV_MODE}volumes: - ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi environment: - KPI_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} - KPI_UWSGI_CHEAPER_WORKERS_COUNT=${UWSGI_WORKERS_START} @@ -45,18 +45,18 @@ services: - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT} - KPI_UWSGI_HARAKIRI=${UWSGI_HARAKIRI} - KPI_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY} - ${USE_CELERY}- SKIP_CELERY=True - ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=kobo.settings.dev - ${USE_HTTPS}- SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO, https - ${USE_NPM_FROM_HOST}- FRONTEND_DEV_MODE=host + ${USE_CELERY} - SKIP_CELERY=True + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO, https + ${USE_NPM_FROM_HOST} - FRONTEND_DEV_MODE=host ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: @@ -70,13 +70,13 @@ services: ports: - ${NGINX_EXPOSED_PORT}:80 ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} networks: kobo-fe-network: aliases: @@ -91,13 +91,13 @@ services: environment: - DUMMY_ENV=True ${USE_EXTRA_HOSTS}extra_hosts: - ${USE_FAKE_DNS}- ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${USE_FAKE_DNS}- ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS}- redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index 789bfa4..7204e6a 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -55,3 +55,5 @@ ${USE_BACKUP}REDIS_BACKUP_SCHEDULE=${REDIS_BACKUP_SCHEDULE} REDIS_SESSION_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/2 REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/3 REDIS_PASSWORD=${REDIS_PASSWORD} + +REDIS_CACHE_MAX_MEMORY=${REDIS_CACHE_MAX_MEMORY} diff --git a/tests/test_config.py b/tests/test_config.py index 46ad0aa..5a0c679 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -116,8 +116,8 @@ def test_dev_mode(): def test_server_roles_questions(): config = read_config() - assert config.frontend_questions - assert config.backend_questions + assert config.frontend + assert config.backend with patch('helpers.cli.CLI.colored_input') as mock_colored_input: mock_colored_input.side_effect = iter( @@ -126,12 +126,12 @@ def test_server_roles_questions(): config._Config__questions_multi_servers() config._Config__questions_roles() - assert config.frontend_questions - assert not config.backend_questions + assert config.frontend + assert not config.backend config._Config__questions_roles() - assert not config.frontend_questions - assert config.backend_questions + assert not config.frontend + assert config.backend assert config.secondary_backend