Merge pull request #19 from weseek/imprv/add-time-to-filename

Improvement: Add time string to filename

bin/backup.sh

@@ -17,7 +17,7 @@ CWD=`/usr/bin/dirname $0`
 cd $CWD
 
 . ./functions.sh
-TODAY=`create_today_yyyymmdd`
+NOW=`create_current_yyyymmddhhmmss`
 
 echo "=== $0 started at `/bin/date "+%Y/%m/%d %H:%M:%S"` ==="
 
@@ -29,7 +29,7 @@ TAR_OPTS="jcvf"
 
 DIRNAME=`/usr/bin/dirname ${TARGET}`
 BASENAME=`/usr/bin/basename ${TARGET}`
-TARBALL="${BACKUPFILE_PREFIX}-${TODAY}.tar.bz2"
+TARBALL="${BACKUPFILE_PREFIX}-${NOW}.tar.bz2"
 TARBALL_FULLPATH="${TMPDIR}/${TARBALL}"
 
 

bin/functions.sh

@@ -64,9 +64,9 @@ gs_copy_file() {
 	${GCPCLI} ${GCPCLIOPT} ${GCPCLI_COPY_OPT} $1 $2
 }
 
-# Create today's date string(YYYYmmdd)
-create_today_yyyymmdd() {
-	create_past_yyyymmdd 0
+# Create current datetime string(YYYYmmddHHMMSS)
+create_current_yyyymmddhhmmss() {
+	echo `/bin/date +%Y%m%d%H%M%S`
 }
 
 # Create date string of n days ago

test/gcs/Dockerfile.init_test

@@ -16,7 +16,7 @@ RUN curl -SL https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_V
       | tar -xz -C /usr/local/bin
 
 # dummy backup data for restore
-COPY dummy-backup-20180622.tar.bz2 /
+COPY dummy-backup-20180622000000.tar.bz2 /
 
 # copy shell script for entrypoint
 COPY entrypoint.init_test.sh /root

test/gcs/docker-compose.yml

@@ -26,7 +26,7 @@ services:
     <<: *app_default
     environment:
       <<: *base_env
-      TARGET_FILE: dummy-backup-20180622.tar.bz2
+      TARGET_FILE: dummy-backup-20180622000000.tar.bz2
     command:
       - restore
 
@@ -53,7 +53,7 @@ services:
     <<: *app_with_dot_boto
     environment:
       <<: *with_boto_env
-      TARGET_FILE: dummy-backup-20180622.tar.bz2
+      TARGET_FILE: dummy-backup-20180622000000.tar.bz2
     command:
       - restore
 

test/gcs/e2e.sh

@@ -87,7 +87,7 @@ execute_default_commands_and_assert_file_exists_on_gcs() {
   SERVICE_NAME=$1
   docker-compose up --build $SERVICE_NAME
   # Expect for app
-  assert_file_exists_on_gcs "backup-${TODAY}.tar.bz2"
+  assert_file_exists_on_gcs "backup-${TODAY}"
   # Exit test for app
   echo "Finished test for $SERVICE_NAME: OK"
 
@@ -112,7 +112,7 @@ execute_default_command_in_cron_mode_and_assert_file_exists_on_gcs() {
   sleep 65 # wait for the network of docker-compose to be ready, and wait until test backup is executed at least once.
   docker-compose stop $SERVICE_NAME
   # Expect for app_backup_cronmode
-  assert_file_exists_on_gcs "backup-${TODAY}.tar.bz2"
+  assert_file_exists_on_gcs "backup-${TODAY}"
   # Exit test for app_restore
   echo "Finished test for $SERVICE_NAME: OK"
 
@@ -152,7 +152,7 @@ TODAY=`/bin/date +%Y%m%d` # It is used to generate file name to restore
 echo "=== $0 started at `/bin/date "+%Y/%m/%d %H:%M:%S"` ==="
 
 # Validate environment variables
-REQUIRED_ENVS=("GCP_ACCESS_KEY_ID" "GCP_SECRET_ACCESS_KEY" "GCP_PROJECT_ID" "TARGET_BUCKET_URL" "DOT_BOTO_OAUTH")
+REQUIRED_ENVS=("GCP_ACCESS_KEY_ID" "GCP_SECRET_ACCESS_KEY" "GCP_PROJECT_ID" "TARGET_BUCKET_URL")
 SATISFY=1
 for ((i = 0; i < ${#REQUIRED_ENVS[@]}; i++)) {
   ENV=$(eval echo "\$${REQUIRED_ENVS[i]}")

test/gcs/entrypoint.init_test.sh

@@ -5,7 +5,7 @@ envsubst < /tmp/.boto > /root/.boto
 
 # REMOVE ALL OBJECTS in GCS bucket
 gsutil ls ${TARGET_BUCKET_URL}
-gsutil rm -rf ${TARGET_BUCKET_URL}
+gsutil rm -rf ${TARGET_BUCKET_URL}**
 
 # Copy fixture file which is used in test restoring
-gsutil cp dummy-backup-20180622.tar.bz2 ${TARGET_BUCKET_URL}
+gsutil cp dummy-backup-20180622000000.tar.bz2 ${TARGET_BUCKET_URL}

test/s3/Dockerfile.init_test

@@ -17,7 +17,7 @@ RUN curl -SL https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_V
       | tar -xz -C /usr/local/bin
 
 # dummy backup data for restore
-COPY dummy-backup-20180622.tar.bz2 /
+COPY dummy-backup-20180622000000.tar.bz2 /
 
 # copy shell script for entrypoint
 COPY entrypoint.init_test.sh /root

test/s3/docker-compose.yml

@@ -28,7 +28,7 @@ services:
     environment:
       <<: *base_env
       TARGET_BUCKET_URL: s3://app_restore/
-      TARGET_FILE: dummy-backup-20180622.tar.bz2
+      TARGET_FILE: dummy-backup-20180622000000.tar.bz2
     command:
       - restore
 

test/s3/e2e.sh

@@ -17,7 +17,7 @@ assert_file_exists_on_s3() {
   if [ $# -ne 1 ]; then exit 1; fi
 
   S3_FILE_PATH=$1
-  docker-compose exec s3proxy sh -c "test -f /data/${S3_FILE_PATH}"
+  docker-compose exec s3proxy sh -c "ls /data/${S3_FILE_PATH} >/dev/null 2>&1"
   if [ $? -ne 0 ]; then
     echo "assert_file_exists_on_s3 FAILED";
     echo "could not find /data/${S3_FILE_PATH} in s3proxy.";
@@ -52,7 +52,8 @@ docker-compose up --build init
 # Execute app_default
 docker-compose up --build app_default
 # Expect for app_default
-assert_file_exists_on_s3 "app_default/backup-${TODAY}.tar.bz2"
+# Use wildcard since the time field of the filename is changed frequently.
+assert_file_exists_on_s3 "app_default/backup-${TODAY}*.tar.bz2"
 # Exit test for app_default
 echo 'Finished test for app_default: OK'
 
@@ -68,7 +69,8 @@ docker-compose up --build app_backup_cronmode &
 sleep 65 # wait for the network of docker-compose to be ready, and wait until test backup is executed at least once.
 docker-compose stop app_backup_cronmode
 # Expect for app_backup_cronmode
-assert_file_exists_on_s3 "app_backup_cronmode/backup-${TODAY}.tar.bz2"
+# Use wildcard since the time field of the filename is changed frequently.
+assert_file_exists_on_s3 "app_backup_cronmode/backup-${TODAY}*.tar.bz2"
 # Exit test for app_restore
 echo 'Finished test for app_backup_cronmode: OK'
 

test/s3/entrypoint.init_test.sh

@@ -6,4 +6,4 @@
 /usr/bin/aws --endpoint-url=http://s3proxy:80/ s3 mb s3://app_restore
 
 # Copy fixture files which are used in test restoring
-/usr/bin/aws --endpoint-url=http://s3proxy:80/ s3 cp /dummy-backup-20180622.tar.bz2 s3://app_restore/
+/usr/bin/aws --endpoint-url=http://s3proxy:80/ s3 cp /dummy-backup-20180622000000.tar.bz2 s3://app_restore/