From 63445e57934fe3ef0e9eb4a2b48cb8cb3889a845 Mon Sep 17 00:00:00 2001 From: Anton Boritskiy Date: Wed, 9 Jun 2021 09:18:06 +0200 Subject: [PATCH 1/2] ADHOC feat async-import: make things faster through parallel run if task is imported asyncronously - playbook can proceed with other tasks, in the use cases here, I don't think we need to check back if task has finished successfully, if this is a concern - don't allow async execution see: * https://www.treitos.com/blog/2020/improving-ansible-performance.html * https://docs.ansible.com/ansible/2.9/user_guide/playbooks_async.html * https://docs.ansible.com/ansible/latest/user_guide/playbooks_handlers.html --- defaults/main.yml | 10 ++++++++++ tasks/cleanup-old-clones.yml | 13 +++++++++++-- tasks/installation.yml | 12 ++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index c30c169..0df719a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -326,3 +326,13 @@ magento_force_reindex_on_deploy: False # and more # and more # and more + +# allows to run deletion of old clones folders asynchronously, +# this allows pipeline to finish faster due to "parallel" execution of tasks +# when clones are cleaned up asynchronously, role is not checking successful execution +magento_allow_async_cleanup_old_clones: false + +# allows to run magento reindexing asynchronously, +# this allows pipeline to finish faster due to "parallel" execution of tasks +# when reindexing is done asynchronously, role is not checking successful execution +magento_allow_async_reindex: false \ No newline at end of file diff --git a/tasks/cleanup-old-clones.yml b/tasks/cleanup-old-clones.yml index 0b7ead3..822a785 100644 --- a/tasks/cleanup-old-clones.yml +++ b/tasks/cleanup-old-clones.yml @@ -7,9 +7,18 @@ age_stamp: "ctime" register: magento_clones_for_cleanup -- name: "Remove old clones/releases" +- name: "Remove old clones/releases asynchronously" file: path: "{{ item.path }}" state: absent - when: magento_cleanup_old_clones == True + when: magento_cleanup_old_clones and magento_allow_async_cleanup_old_clones with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" + async: 600 + poll: 0 + +- name: "Remove old clones/releases synchronously" + file: + path: "{{ item.path }}" + state: absent + when: magento_cleanup_old_clones and (not magento_allow_async_cleanup_old_clones) + with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" \ No newline at end of file diff --git a/tasks/installation.yml b/tasks/installation.yml index 3a83e50..991b2cb 100644 --- a/tasks/installation.yml +++ b/tasks/installation.yml @@ -216,11 +216,19 @@ become_user: "{{ magento_user }}" command: "/usr/bin/php {{ magento_app_root }}/bin/magento cache:flush" -- name: "Let Magento indexer run" +- name: "Let Magento indexer run asynchronously" become: "yes" become_user: "{{ magento_user }}" command: "/usr/bin/php {{ magento_app_root }}/bin/magento indexer:reindex" - when: magento_db_status.stdout_lines|length == 0 or magento_force_reindex_on_deploy == True + when: (magento_db_status.stdout_lines|length == 0 or magento_force_reindex_on_deploy == True) and magento_allow_async_reindex + async: 600 + poll: 0 + +- name: "Let Magento indexer run synchronously" + become: "yes" + become_user: "{{ magento_user }}" + command: "/usr/bin/php {{ magento_app_root }}/bin/magento indexer:reindex" + when: (magento_db_status.stdout_lines|length == 0 or magento_force_reindex_on_deploy == True) and (not magento_allow_async_reindex) - name: "Disable maintenance mode" file: From cae79d1c177afc85d28eaab53defb5f8269a29db Mon Sep 17 00:00:00 2001 From: Anton Boritskiy Date: Thu, 10 Jun 2021 03:09:32 +0200 Subject: [PATCH 2/2] ADHOC feat async-import: control async timeouts there are project where reindex takes way longer than 10 minutes, those projects should have a possibility to specify a higher timeout. --- defaults/main.yml | 8 +++++++- tasks/cleanup-old-clones.yml | 4 ++-- tasks/installation.yml | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 0df719a..71065c2 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -332,7 +332,13 @@ magento_force_reindex_on_deploy: False # when clones are cleaned up asynchronously, role is not checking successful execution magento_allow_async_cleanup_old_clones: false +# timeout for async old clones cleanup, seconds +magento_async_cleanup_old_clones_timeout: 600 + # allows to run magento reindexing asynchronously, # this allows pipeline to finish faster due to "parallel" execution of tasks # when reindexing is done asynchronously, role is not checking successful execution -magento_allow_async_reindex: false \ No newline at end of file +magento_allow_async_reindex: false + +# timeout for async reindex execution, seconds +magento_async_reindex_timeout: 600 diff --git a/tasks/cleanup-old-clones.yml b/tasks/cleanup-old-clones.yml index 822a785..6d4886e 100644 --- a/tasks/cleanup-old-clones.yml +++ b/tasks/cleanup-old-clones.yml @@ -13,7 +13,7 @@ state: absent when: magento_cleanup_old_clones and magento_allow_async_cleanup_old_clones with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" - async: 600 + async: "{{ magento_async_cleanup_old_clones_timeout }}" poll: 0 - name: "Remove old clones/releases synchronously" @@ -21,4 +21,4 @@ path: "{{ item.path }}" state: absent when: magento_cleanup_old_clones and (not magento_allow_async_cleanup_old_clones) - with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" \ No newline at end of file + with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" diff --git a/tasks/installation.yml b/tasks/installation.yml index 991b2cb..a0266b6 100644 --- a/tasks/installation.yml +++ b/tasks/installation.yml @@ -221,7 +221,7 @@ become_user: "{{ magento_user }}" command: "/usr/bin/php {{ magento_app_root }}/bin/magento indexer:reindex" when: (magento_db_status.stdout_lines|length == 0 or magento_force_reindex_on_deploy == True) and magento_allow_async_reindex - async: 600 + async: "{{ magento_async_reindex_timeout }}" poll: 0 - name: "Let Magento indexer run synchronously"