Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeffreyaven
jeffreyaven committed Sep 2, 2024
1 parent 3477058 commit f25ace5
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 100 deletions.
6 changes: 3 additions & 3 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# Changelog

## v1.7.0 (2024-09-05)
## v1.7.0 (2024-09-03)

* changed `preflight` to `exists` and `postdeploy` to `statecheck`
* changed `preflight` to `exists` and `postdeploy` to `statecheck`, maintaining backwards compatibility
* enhanced `multi` resource support

## v1.6.5 (2024-08-31)

* added `multi` type
Expand Down
1 change: 1 addition & 0 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ Model driven resource provisioning and deployment framework using StackQL.
:alt: PyPI

.. image:: https://img.shields.io/pypi/dm/stackql-deploy
:target: https://pypi.org/project/stackql-deploy/
:alt: PyPI - Downloads

==============
Expand Down
18 changes: 0 additions & 18 deletions stackql_deploy/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,6 @@ def cli(ctx, custom_registry, download_dir):
@click.argument('stack_dir')
@click.argument('stack_env')
@add_common_options
# @click.option('--log-level', default='INFO', help='set the logging level.')
# @click.option('--env-file', default='.env', help='environment variables file.')
# @click.option('-e', '--env', multiple=True, callback=parse_env_var, help='set additional environment variables.')
# @click.option('--dry-run', is_flag=True, help='perform a dry run of the operation.')
# @click.option('--show-queries', is_flag=True, help='show queries run in the output logs.')
# @click.option('--on-failure', type=click.Choice(['rollback', 'ignore', 'error']), default='error', help='action on failure.')
@click.pass_context
def build(ctx, stack_dir, stack_env, log_level, env_file, env, dry_run, show_queries, on_failure):
"""Create or update resources."""
Expand Down Expand Up @@ -151,12 +145,6 @@ def build(ctx, stack_dir, stack_env, log_level, env_file, env, dry_run, show_que
@click.argument('stack_dir')
@click.argument('stack_env')
@add_common_options
# @click.option('--log-level', default='INFO', help='set the logging level.')
# @click.option('--env-file', default='.env', help='environment variables file.')
# @click.option('-e', '--env', multiple=True, callback=parse_env_var, help='set additional environment variables.')
# @click.option('--dry-run', is_flag=True, help='perform a dry run of the operation.')
# @click.option('--show-queries', is_flag=True, help='show queries run in the output logs.')
# @click.option('--on-failure', type=click.Choice(['rollback', 'ignore', 'error']), default='error', help='action on failure.')
@click.pass_context
def teardown(ctx, stack_dir, stack_env, log_level, env_file, env, dry_run, show_queries, on_failure):
"""Teardown a provisioned stack defined in the `{STACK_DIR}/stackql_manifest.yml` file."""
Expand Down Expand Up @@ -184,12 +172,6 @@ def teardown(ctx, stack_dir, stack_env, log_level, env_file, env, dry_run, show_
@click.argument('stack_dir')
@click.argument('stack_env')
@add_common_options
# @click.option('--log-level', default='INFO', help='set the logging level.')
# @click.option('--env-file', default='.env', help='environment variables file.')
# @click.option('-e', '--env', multiple=True, callback=parse_env_var, help='set additional environment variables.')
# @click.option('--dry-run', is_flag=True, help='perform a dry run of the operation.')
# @click.option('--show-queries', is_flag=True, help='show queries run in the output logs.')
# @click.option('--on-failure', type=click.Choice(['rollback', 'ignore', 'error']), default='error', help='action on failure.')
@click.pass_context
def test(ctx, stack_dir, stack_env, log_level, env_file, env, dry_run, show_queries, on_failure):
"""Run test queries to ensure desired state resources and configuration for the stack defined in the `{STACK_DIR}/stackql_manifest.yml` file."""
Expand Down
82 changes: 82 additions & 0 deletions stackql_deploy/cmd/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,85 @@ def process_exports(self, resource, exports_query, exports_retries, exports_retr
export_data[key] = export.get(key, '')

export_vars(self, resource, export_data, expected_exports, protected_exports)

def check_if_resource_exists(self, resource_exists, resource, exists_query, exists_retries, exists_retry_delay, dry_run, show_queries, delete_test=False):
check_type = 'exists'
if delete_test:
check_type = 'post-delete'
if exists_query:
if dry_run:
self.logger.info(f"🔎 dry run {check_type} check for [{resource['name']}]:\n\n/* exists query */\n{exists_query}\n")
else:
self.logger.info(f"🔎 running {check_type} check for [{resource['name']}]...")
show_query(show_queries, exists_query, self.logger)
resource_exists = perform_retries(resource, exists_query, exists_retries, exists_retry_delay, self.stackql, self.logger, delete_test)
else:
self.logger.info(f"{check_type} check not configured for [{resource['name']}]")
return resource_exists

# if dry_run:
# self.logger.info(f"🔎 dry run post-delete check for [{resource['name']}]:\n\n{exists_query}\n")
# else:
# self.logger.info(f"🔎 checking if [{resource['name']}] exists...")
# show_query(show_queries, exists_query, self.logger)
# resource_deleted = perform_retries(resource, exists_query, postdelete_exists_retries, postdelete_exists_retry_delay, self.stackql, self.logger, delete_test=True)
# if resource_deleted:
# self.logger.info(f"✅ successfully deleted {resource['name']}")
# else:
# catch_error_and_exit(f"❌ failed to delete {resource['name']}.", self.logger)



def check_if_resource_is_correct_state(self, is_correct_state, resource, statecheck_query, statecheck_retries, statecheck_retry_delay, dry_run, show_queries):
if statecheck_query:
if dry_run:
self.logger.info(f"🔎 dry run state check for [{resource['name']}]:\n\n/* state check query */\n{statecheck_query}\n")
else:
self.logger.info(f"🔎 running state check for [{resource['name']}]...")
show_query(show_queries, statecheck_query, self.logger)
is_correct_state = perform_retries(resource, statecheck_query, statecheck_retries, statecheck_retry_delay, self.stackql, self.logger)
if is_correct_state:
self.logger.info(f"👍 [{resource['name']}] is in the desired state")
else:
self.logger.info(f"👎 [{resource['name']}] is not in the desired state")
else:
self.logger.info(f"state check not configured for [{resource['name']}]")
is_correct_state = True
return is_correct_state

def create_resource(self, is_created_or_updated, resource, create_query, create_retries, create_retry_delay, dry_run, show_queries, ignore_errors=False):
if dry_run:
self.logger.info(f"🚧 dry run create for [{resource['name']}]:\n\n/* insert (create) query */\n{create_query}\n")
else:
self.logger.info(f"[{resource['name']}] does not exist, creating 🚧...")
show_query(show_queries, create_query, self.logger)
msg = run_stackql_command(create_query, self.stackql, self.logger, ignore_errors=ignore_errors, retries=create_retries, retry_delay=create_retry_delay)
self.logger.debug(f"create response: {msg}")
is_created_or_updated = True
return is_created_or_updated

def update_resource(self, is_created_or_updated, resource, update_query, update_retries, update_retry_delay, dry_run, show_queries, ignore_errors=False):
if update_query:
if dry_run:
self.logger.info(f"🚧 dry run update for [{resource['name']}]:\n\n/* update query */\n{update_query}\n")
else:
self.logger.info(f"🔧 updating [{resource['name']}]...")
show_query(show_queries, update_query, self.logger)
msg = run_stackql_command(update_query, self.stackql, self.logger, ignore_errors=ignore_errors, retries=update_retries, retry_delay=update_retry_delay)
self.logger.debug(f"update response: {msg}")
is_created_or_updated = True
else:
self.logger.info(f"update query not configured for [{resource['name']}], skipping update...")
return is_created_or_updated

def delete_resource(self, resource, delete_query, delete_retries, delete_retry_delay, dry_run, show_queries, ignore_errors=False):
if delete_query:
if dry_run:
self.logger.info(f"🚧 dry run delete for [{resource['name']}]:\n\n{delete_query}\n")
else:
self.logger.info(f"🚧 deleting [{resource['name']}]...")
show_query(show_queries, delete_query, self.logger)
msg = run_stackql_command(delete_query, self.stackql, self.logger, ignore_errors=ignore_errors, retries=delete_retries, retry_delay=delete_retry_delay)
self.logger.debug(f"delete response: {msg}")
else:
self.logger.info(f"delete query not configured for [{resource['name']}], skipping delete...")
55 changes: 16 additions & 39 deletions stackql_deploy/cmd/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,69 +100,46 @@ def run(self, dry_run, show_queries, on_failure):

if type in ('resource', 'multi'):

ignore_errors = False
resource_exists = False
is_correct_state = False
if type == 'multi':
# multi resources ignore errors on create or update
ignore_errors = True

#
# run exists check (check if resource exists)
#
if exists_query:
if dry_run:
self.logger.info(f"🔎 dry run exists check for [{resource['name']}]:\n\n/* exists query */\n{exists_query}\n")
else:
self.logger.info(f"🔎 running exists check for [{resource['name']}]...")
show_query(show_queries, exists_query, self.logger)
resource_exists = perform_retries(resource, exists_query, exists_retries, exists_retry_delay, self.stackql, self.logger)
else:
self.logger.info(f"exists check not configured for [{resource['name']}]")
resource_exists = False

resource_exists = self.check_if_resource_exists(resource_exists, resource, exists_query, exists_retries, exists_retry_delay, dry_run, show_queries)

#
# initial state check (if resource exists)
#
if resource_exists:
if not statecheck_query:
self.logger.info(f"state check not configured for [{resource['name']}], state check bypassed...")
is_correct_state = True
else:
self.logger.info(f"🔎 [{resource['name']}] exists, running state check...")
show_query(show_queries, statecheck_query, self.logger)
is_correct_state = perform_retries(resource, statecheck_query, statecheck_retries, statecheck_retry_delay, self.stackql, self.logger)
is_correct_state = self.check_if_resource_is_correct_state(is_correct_state, resource, statecheck_query, statecheck_retries, statecheck_retry_delay, dry_run, show_queries)

# if exists and correct state, skip deploy
if resource_exists and is_correct_state:
self.logger.info(f"👍 [{resource['name']}] is in the desired state, skipping create or update...")
self.logger.info(f"skipping create or update for {resource['name']}...")

#
# resource does not exist
#
is_created_or_updated = False
if not resource_exists:
if dry_run:
self.logger.info(f"🚧 dry run create for [{resource['name']}]:\n\n/* insert (create) query */\n{create_query}\n")
else:
self.logger.info(f"[{resource['name']}] does not exist, creating 🚧...")
show_query(show_queries, create_query, self.logger)
msg = run_stackql_command(create_query, self.stackql, self.logger, ignore_errors=False, retries=create_retries, retry_delay=create_retry_delay)
self.logger.debug(f"create response: {msg}")
is_created_or_updated = self.create_resource(is_created_or_updated, resource, create_query, create_retries, create_retry_delay, dry_run, show_queries, ignore_errors)

#
# resource exists but is not in the correct state
#
if resource_exists and not is_correct_state:
if dry_run:
self.logger.info(f"🚧 dry run update for [{resource['name']}]:\n\n/* update query */\n{update_query}\n")
else:
self.logger.info(f"🔧 updating [{resource['name']}]...")
show_query(show_queries, update_query, self.logger)
msg = run_stackql_command(update_query, self.stackql, self.logger, ignore_errors=False, retries=update_retries, retry_delay=update_retry_delay)
self.logger.debug(f"update response: {msg}")
is_created_or_updated = self.update_resource(is_created_or_updated, resource, update_query, update_retries, update_retry_delay, dry_run, show_queries, ignore_errors)

#
# check state again after create or update
#
if dry_run:
self.logger.info(f"🔎 dry run post create/update state check for [{resource['name']}]:\n\n/* state check query */\n{statecheck_query}\n")
else:
self.logger.info(f"🔎 running post create/update state check for [{resource['name']}]...")
show_query(show_queries, statecheck_query, self.logger)
is_correct_state = perform_retries(resource, statecheck_query, statecheck_retries, statecheck_retry_delay, self.stackql, self.logger)
if is_created_or_updated:
is_correct_state = self.check_if_resource_is_correct_state(is_correct_state, resource, statecheck_query, statecheck_retries, statecheck_retry_delay, dry_run, show_queries)

#
# statecheck check complete
Expand Down
41 changes: 12 additions & 29 deletions stackql_deploy/cmd/teardown.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,46 +73,29 @@ def run(self, dry_run, show_queries, on_failure):
#
# pre-delete check
#
resource_exists = False
ignore_errors = False
resource_exists = True # assume exists
if type == 'multi':
self.logger.info(f"pre-delete check not supported for multi resources, skipping...")
resource_exists = True
elif exists_query:
if dry_run:
self.logger.info(f"🔎 dry run pre-delete check for [{resource['name']}]:\n\n{exists_query}\n")
else:
self.logger.info(f"🔎 checking if [{resource['name']}] exists...")
show_query(show_queries, exists_query, self.logger)
resource_exists = perform_retries(resource, exists_query, exists_retries, exists_retry_delay, self.stackql, self.logger)
else:
self.logger.info(f"no exists query defined for [{resource['name']}], skipping pre-delete check...")

ignore_errors = True # multi resources ignore errors on create or update
elif type == 'resource':
resource_exists = self.check_if_resource_exists(resource_exists, resource, exists_query, exists_retries, exists_retry_delay, dry_run, show_queries)

#
# delete
#
if resource_exists:
if dry_run:
self.logger.info(f"🚧 dry run delete for [{resource['name']}]:\n\n{delete_query}\n")
else:
self.logger.info(f"🚧 deleting [{resource['name']}]...")
show_query(show_queries, delete_query, self.logger)
msg = run_stackql_command(delete_query, self.stackql, self.logger, ignore_errors=False, retries=delete_retries, retry_delay=delete_retry_delay)
self.logger.debug(f"delete response: {msg}")
self.delete_resource(resource, delete_query, delete_retries, delete_retry_delay, dry_run, show_queries, ignore_errors)
else:
self.logger.info(f"resource [{resource['name']}] does not exist, skipping delete")
continue

#
# confirm deletion
#
resource_deleted = False
if dry_run:
self.logger.info(f"🔎 dry run post-delete check for [{resource['name']}]:\n\n{exists_query}\n")
resource_deleted = self.check_if_resource_exists(False, resource, exists_query, postdelete_exists_retries, postdelete_exists_retry_delay, dry_run, show_queries, delete_test=True)

if resource_deleted:
self.logger.info(f"✅ successfully deleted {resource['name']}")
else:
self.logger.info(f"🔎 checking if [{resource['name']}] exists...")
show_query(show_queries, exists_query, self.logger)
resource_deleted = perform_retries(resource, exists_query, postdelete_exists_retries, postdelete_exists_retry_delay, self.stackql, self.logger, delete_test=True)
if resource_deleted:
self.logger.info(f"✅ successfully deleted {resource['name']}")
else:
catch_error_and_exit(f"❌ failed to delete {resource['name']}.", self.logger)
catch_error_and_exit(f"❌ failed to delete {resource['name']}.", self.logger)
12 changes: 2 additions & 10 deletions stackql_deploy/cmd/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,8 @@ def run(self, dry_run, show_queries, on_failure):
# statecheck check
#
if type in ('resource', 'multi'):
if not statecheck_query:
is_correct_state = True
self.logger.info(f"❓ test not configured for [{resource['name']}]")
elif dry_run:
is_correct_state = True
self.logger.info(f"test query for [{resource['name']}]:\n\n{statecheck_query}\n")
else:
self.logger.info(f"🔎 checking state for [{resource['name']}]...")
show_query(show_queries, statecheck_query, self.logger)
is_correct_state = perform_retries(resource, statecheck_query, statecheck_retries, statecheck_retry_delay, self.stackql, self.logger)

is_correct_state = self.check_if_resource_is_correct_state(False, resource, statecheck_query, statecheck_retries, statecheck_retry_delay, dry_run, show_queries)

if not is_correct_state:
catch_error_and_exit(f"❌ test failed for {resource['name']}.", self.logger)
Expand Down
1 change: 0 additions & 1 deletion stackql_deploy/lib/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ def run_stackql_query(query, stackql, suppress_errors, logger, retries=0, delay=

def error_detected(result):
"""parse stdout for known error conditions"""
# aws cloud control hack...
if result['message'].startswith('http response status code: 4') or result['message'].startswith('http response status code: 5'):
return True
if result['message'].startswith('error:'):
Expand Down

0 comments on commit f25ace5

Please sign in to comment.