From a718b188f1bd3760449500d80d841b5afeb3e2aa Mon Sep 17 00:00:00 2001 From: Anton Boritskiy Date: Tue, 20 Aug 2019 13:39:00 +0100 Subject: [PATCH 1/3] ADHOC feat (cleaup): auto-cleanup automatic cleanup of older clones to keep the disk usage under control. Those clones/releases are usually around 1GB in size, having 30 of them on the machine is easy within a week if project is atively developed. That means that most of this space is pretty much wasted, cause all those clones are anyways stored in AWS/Azure we don't need to keep them on the machine for backup purposes or whatsoever. We only need few recent ones to ensure fast roll-back in case of failed deploy. The roll back is expected to be performed manually as of now. There is no automation prepared or planned for it at the moment. Using `shell` and not `command` here cause `command` is not able to interpret `\(` correctly and pass that to the actual shell. It is an element of `find` parameter syntax, we are using a construct there to do a negation. The actual one-liner command was borrowed from other project where release activation/deployment is done with a shell script, so the command itself is battle-proven. --- defaults/main.yml | 6 ++++++ tasks/cleanup-old-clones.yml | 7 +++++++ tasks/main.yml | 1 + 3 files changed, 14 insertions(+) create mode 100644 tasks/cleanup-old-clones.yml diff --git a/defaults/main.yml b/defaults/main.yml index 3b7e964..251be57 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -290,3 +290,9 @@ magento_upgrade_commands: magento_run_message_queue_consumers_separately: false magento_queue_consumers_cron_template: "/usr/bin/php {{ magento_app_root }}/bin/magento queue:consumers:start {{ item.name }}" + +## Deletes all installed deployable magento packages except the +## %magento_clones_to_keep% amount of latest ones + the one that is being +## deployed right now. +magento_cleanup_old_clones: False +magento_clones_to_keep: 5 diff --git a/tasks/cleanup-old-clones.yml b/tasks/cleanup-old-clones.yml new file mode 100644 index 0000000..0d4d63e --- /dev/null +++ b/tasks/cleanup-old-clones.yml @@ -0,0 +1,7 @@ +--- +- name: "Cleanup old clones" + become: "yes" + become_user: "{{ magento_user }}" + shell: "find {{ magento_release_folder }} -maxdepth 1 -mindepth 1 ! \\( -name \"{{ magento_release_version }}\" \\) -type d -printf '%T@:%p\n' | sort -r | tail --lines=+{{ magento_clones_to_keep }} | awk -F: '{ print $2 }' | xargs -t rm -rf ;" + when: + - magento_cleanup_old_clones == True diff --git a/tasks/main.yml b/tasks/main.yml index 8f327bd..76f8ab9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -3,4 +3,5 @@ - include: "setup.yml" - include: "installation.yml" - include: "message-queue-consumers.yml" +- include: "cleanup-old-clones.yml" - include: "admin.yml" From cb60e4c4292ffaebeb9f475caddd7bbea658b5e4 Mon Sep 17 00:00:00 2001 From: Anton Boritskiy Date: Wed, 21 Aug 2019 18:05:11 +0100 Subject: [PATCH 2/3] ADHOC feat (cleaup): auto-cleanup automatic cleanup of older clones to keep the disk usage under control. Those clones/releases are usually around 1GB in size, having 30 of them on the machine is easy within a week if project is atively developed. That means that most of this space is pretty much wasted, cause all those clones are anyways stored in AWS/Azure we don't need to keep them on the machine for backup purposes or whatsoever. We only need few recent ones to ensure fast roll-back in case of failed deploy. The roll back is expected to be performed manually as of now. There is no automation prepared or planned for it at the moment. This is yet another implementation for the problem explained above. This implementaion is also backported from another project, so it is also battle-prven so to say, at the same time it ansible native which makes it a bit more readable. Author leaves both implementation stay in the history and not squash/rebase to show possible alternative if later on additional constainst will be discovered. Using ctime as the actual file creation date and also non-fakeable attribute. http://lists.gnu.org/archive/html/coreutils/2010-08/msg00010.html ctime cannot be faked (at least it's not intended to be fakeable): POSIX says that atime and mtime are user-settable to arbitrary times via the utimensat() family of syscalls, but that ctime must unfakeably track the current time of any action that changes a file's metadata or contents. --- defaults/main.yml | 5 +++-- tasks/cleanup-old-clones.yml | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 251be57..48f5214 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -292,7 +292,8 @@ magento_run_message_queue_consumers_separately: false magento_queue_consumers_cron_template: "/usr/bin/php {{ magento_app_root }}/bin/magento queue:consumers:start {{ item.name }}" ## Deletes all installed deployable magento packages except the -## %magento_clones_to_keep% amount of latest ones + the one that is being -## deployed right now. +## %magento_clones_to_keep% amount of latest ones + all that are newer than +## %magento_clones_cleanup_min_age% magento_cleanup_old_clones: False magento_clones_to_keep: 5 +magento_clones_cleanup_min_age: "1d" diff --git a/tasks/cleanup-old-clones.yml b/tasks/cleanup-old-clones.yml index 0d4d63e..0b7ead3 100644 --- a/tasks/cleanup-old-clones.yml +++ b/tasks/cleanup-old-clones.yml @@ -1,7 +1,15 @@ --- -- name: "Cleanup old clones" - become: "yes" - become_user: "{{ magento_user }}" - shell: "find {{ magento_release_folder }} -maxdepth 1 -mindepth 1 ! \\( -name \"{{ magento_release_version }}\" \\) -type d -printf '%T@:%p\n' | sort -r | tail --lines=+{{ magento_clones_to_keep }} | awk -F: '{ print $2 }' | xargs -t rm -rf ;" - when: - - magento_cleanup_old_clones == True +- name: "Discover old clone/release directories which need to be cleaned" + find: + paths: "{{ magento_release_folder }}" + age: "{{ magento_clones_cleanup_min_age }}" + file_type: directory + age_stamp: "ctime" + register: magento_clones_for_cleanup + +- name: "Remove old clones/releases" + file: + path: "{{ item.path }}" + state: absent + when: magento_cleanup_old_clones == True + with_items: "{{ (magento_clones_for_cleanup.files | sort(attribute='ctime', reverse=True))[magento_clones_to_keep:] | list }}" From 5a09b7d6df03777e09a6782737f2fe1913e75edf Mon Sep 17 00:00:00 2001 From: Anton Boritskiy Date: Wed, 21 Aug 2019 18:15:14 +0100 Subject: [PATCH 3/3] ADHOC fix (cleaup): auto-cleanup by default enable auto-cleanup by default, to cover the conern highlighted on code-review: Looks good. But would clean up by default. I am not aware of any project that needs to keep the old copies. So I vote for setting to something that makes sense. Instead of using this role and then need to apply this config to get a behavior that should be the default. Author accept the concern and applies corresponding change --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 48f5214..e333fe5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -294,6 +294,6 @@ magento_queue_consumers_cron_template: "/usr/bin/php {{ magento_app_root }}/bin/ ## Deletes all installed deployable magento packages except the ## %magento_clones_to_keep% amount of latest ones + all that are newer than ## %magento_clones_cleanup_min_age% -magento_cleanup_old_clones: False +magento_cleanup_old_clones: True magento_clones_to_keep: 5 magento_clones_cleanup_min_age: "1d"