From 6a46d8e5e6f24f05e70664e3940dec735cd75705 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 10:54:39 -0700 Subject: [PATCH 01/14] Refactor load_mongodump.sh script to support better error handling, debugging information, and integration with Docker Compose configuration file for database host configuration. --- viz_scripts/docker/load_mongodump.sh | 72 ++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index d69854ed..b85d10e1 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -1,9 +1,71 @@ +#!/bin/bash + +# Directory of the script +SCRIPT_DIR="$(dirname "$0")" + +# Path to the configuration file (one level up) +CONFIG_FILE="$SCRIPT_DIR/../../docker-compose.dev.yml" + +# Check if the correct number of arguments is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " : The path to the MongoDB dump file to be restored." + echo " run git add -f after using this command" + exit 1 +fi + MONGODUMP_FILE=$1 -echo "Copying file to docker container" -docker cp $MONGODUMP_FILE em-public-dashboard-db-1:/tmp +# Print debug information +echo "Script Directory: $SCRIPT_DIR" +echo "Configuration File Path: $CONFIG_FILE" +echo "MongoDump File Path: $MONGODUMP_FILE" + +# Check if the provided file exists +if [ ! -f "$MONGODUMP_FILE" ]; then + echo "Error: File '$MONGODUMP_FILE' does not exist." + exit 1 +fi + +# Check if the configuration file exists +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: Configuration file '$CONFIG_FILE' does not exist." + exit 1 +fi + +# Print details about the configuration file +echo "Configuration file details:" +ls -l "$CONFIG_FILE" + +# Extract the database name from the mongodump file +DB_NAME=$(tar -tf "$MONGODUMP_FILE" | grep '^dump/' | sed 's|^dump/||' | awk -F'/' '{if (NF > 0) {print $1; exit}}') + +# Output the database name +echo "$DB_NAME" + +if [ -z "$DB_NAME" ]; then + echo "Error: Failed to extract database name from mongodump." + exit 1 +fi + +echo "Database Name: $DB_NAME" + +# Update the docker-compose configuration file with the actual DB_HOST +DB_HOST="mongodb://db/$DB_NAME" +sed -i.bak "s|DB_HOST:.*|DB_HOST: $DB_HOST|" "$CONFIG_FILE" + +echo "Updated docker-compose file:" +cat "$CONFIG_FILE" + +echo "Copying file to Docker container" +docker cp "$MONGODUMP_FILE" em-public-dashboard-db-1:/tmp + +FILE_NAME=$(basename "$MONGODUMP_FILE") + +echo "Clearing existing database" +docker exec em-public-dashboard-db-1 bash -c "mongo $DB_NAME --eval 'db.dropDatabase()'" -FILE_NAME=`basename $MONGODUMP_FILE` +echo "Restoring the dump from $FILE_NAME to database $DB_NAME" +docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard-db-1 bash -c "cd /tmp && tar xvf $FILE_NAME && mongorestore -d $DB_NAME dump/$DB_NAME" -echo "Restoring the dump from $FILE_NAME" -docker exec -e MONGODUMP_FILE=$FILE_NAME em-public-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore' +echo "Database restore complete." From df28c818edf9673bc514be2513f358085cb8cc42 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 11:10:10 -0700 Subject: [PATCH 02/14] Update the docker-compose.dev.yml's environment DB_HOST to mongo://db/DB_NAME. Add docker-compose.dev.yml into gitignore. Update README.md file with information about not to make changes into docker-compose.dev.yml and update about forced push if changes are made to docker-compose.dev.yml. --- .gitignore | 3 +++ README.md | 18 ++++++++++++++++++ docker-compose.dev.yml | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa7a8ff4..c1c6d967 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ dmypy.json # Pyre type checker .pyre/ + +# docker-compose-dev.yml +docker-compose.yml diff --git a/README.md b/README.md index 378c165e..1bf3771b 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,24 @@ Note that this expects a standard setup with: - this repository checked out under the `em-public-dashboard` directory, which makes the database name `em-public-dashboard_db_1` - the incoming mongodump is in tar gz format. This should be true of all canbikeco dumps, you may need to change the `tar xvf` to `unzip` otherwise. The mongo container typically doesn't have zip installed, so using tar is more portable. +## Working with `docker compose` and `.gitignore` + +### Using `docker compose` + +When working with `docker compose`, it's generally recommended to avoid committing changes to the `docker-compose-dev.yml` file, especially if you're running the `./load_mongodump ` script. This file is typically configured to work in a specific way for your development environment, and changes might not be applicable or useful for others working on the same project. + +### `.gitignore` Configuration + +To streamline your workflow, we have added the `docker-compose-dev.yml` file to the `.gitignore` file. This means that by default, changes to `docker-compose-dev.yml` will not be tracked by Git. This setup helps to avoid unnecessary commits and ensures that your `docker-compose-dev.yml` remains consistent with the intended configuration for the project. + +### Committing Changes to `docker-compose-dev.yml` + +If you do need to make changes to `docker-compose-dev.yml` and want to commit those changes, you can override the ignore settings by using the following Git command: + +```bash +git add -f docker-compose-dev.yml +``` + **If you have a non-standard setup, please use your expertise to change the script appropriately.** #### Happy visualizations! diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1906101e..d8cbd2e7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -25,7 +25,7 @@ services: depends_on: - db environment: - - DB_HOST=db + - DB_HOST=mongo://db/DB_NAME - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - STUDY_CONFIG=stage-program From 20a2c6f847af004654bf9e04f05ef31f33a9293f Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 11:16:39 -0700 Subject: [PATCH 03/14] Update docker-compose-dev.yml to docker-compose.dev.yml. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1bf3771b..bf502f4b 100644 --- a/README.md +++ b/README.md @@ -74,18 +74,18 @@ Note that this expects a standard setup with: ### Using `docker compose` -When working with `docker compose`, it's generally recommended to avoid committing changes to the `docker-compose-dev.yml` file, especially if you're running the `./load_mongodump ` script. This file is typically configured to work in a specific way for your development environment, and changes might not be applicable or useful for others working on the same project. +When working with `docker compose`, it's generally recommended to avoid committing changes to the `docker-compose.dev.yml` file, especially if you're running the `./load_mongodump ` script. This file is typically configured to work in a specific way for your development environment, and changes might not be applicable or useful for others working on the same project. ### `.gitignore` Configuration -To streamline your workflow, we have added the `docker-compose-dev.yml` file to the `.gitignore` file. This means that by default, changes to `docker-compose-dev.yml` will not be tracked by Git. This setup helps to avoid unnecessary commits and ensures that your `docker-compose-dev.yml` remains consistent with the intended configuration for the project. +To streamline your workflow, we have added the `docker-compose.dev.yml` file to the `.gitignore` file. This means that by default, changes to `docker-compose.dev.yml` will not be tracked by Git. This setup helps to avoid unnecessary commits and ensures that your `docker-compose.dev.yml` remains consistent with the intended configuration for the project. -### Committing Changes to `docker-compose-dev.yml` +### Committing Changes to `docker-compose.dev.yml` -If you do need to make changes to `docker-compose-dev.yml` and want to commit those changes, you can override the ignore settings by using the following Git command: +If you do need to make changes to `docker-compose.dev.yml` and want to commit those changes, you can override the ignore settings by using the following Git command: ```bash -git add -f docker-compose-dev.yml +git add -f docker-compose.dev.yml ``` **If you have a non-standard setup, please use your expertise to change the script appropriately.** From 3a156c1d114a422f79b4acc6dc2e368c57208b66 Mon Sep 17 00:00:00 2001 From: iantei Date: Tue, 10 Sep 2024 11:17:48 -0700 Subject: [PATCH 04/14] Remove stupid prefix on dashboard's title. Remove extra bullet point. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index bf502f4b..c2934257 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# A simple and stupid dashboard for e-mission +# A simple dashboard for e-mission Issues: Since this repository is part of a larger project, all issues are tracked in the central docs repository. If you have a question, as suggested by the open source guide, please file an issue instead of sending an email. Since issues are public, other contributors can try to answer the question and benefit from the answer. @@ -139,7 +139,6 @@ You may need to increase the resources avaliable to Docker if: - you believe you've loaded the data but there is none when running the notebooks - the notebook can't connect to the database - when you try and start the container for the database it exits with code 14 -- ## Large Dataset Workaround From f36706aa17e248aecefcbe922bae1ec46f942676 Mon Sep 17 00:00:00 2001 From: iantei Date: Sun, 15 Sep 2024 08:45:48 -0700 Subject: [PATCH 05/14] Update DB_HOST to mongodb from mongo --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d8cbd2e7..d165ff15 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -25,7 +25,7 @@ services: depends_on: - db environment: - - DB_HOST=mongo://db/DB_NAME + - DB_HOST=mongodb://db/DB_NAME - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - STUDY_CONFIG=stage-program From 00e03470207a053b5617628c705c7bf6aa0194f4 Mon Sep 17 00:00:00 2001 From: iantei Date: Sun, 15 Sep 2024 08:47:17 -0700 Subject: [PATCH 06/14] Include both docker-compose.dev.yml and docker-compose.yml in git ignore. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c1c6d967..fd088c9b 100644 --- a/.gitignore +++ b/.gitignore @@ -133,5 +133,6 @@ dmypy.json # Pyre type checker .pyre/ -# docker-compose-dev.yml +# docker-compose yml docker-compose.yml +docker-compose.dev.yml \ No newline at end of file From 960b44904b03dd973862b2cb9b5a787379a99826 Mon Sep 17 00:00:00 2001 From: iantei Date: Sun, 15 Sep 2024 09:01:35 -0700 Subject: [PATCH 07/14] Resolve merge conflict in .gitignore with addition of .DS_Store --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 14507d0a..84f64f09 100644 --- a/.gitignore +++ b/.gitignore @@ -133,10 +133,8 @@ dmypy.json # Pyre type checker .pyre/ -<<<<<<< HEAD # docker-compose yml docker-compose.yml docker-compose.dev.yml -======= + .DS_Store ->>>>>>> main From 9b90990262299fc90c47ab338c1ca468c5ab8fb9 Mon Sep 17 00:00:00 2001 From: iantei Date: Mon, 16 Sep 2024 09:14:40 -0700 Subject: [PATCH 08/14] Remove comment for docker compose yml in gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 84f64f09..7b6ab784 100644 --- a/.gitignore +++ b/.gitignore @@ -133,7 +133,6 @@ dmypy.json # Pyre type checker .pyre/ -# docker-compose yml docker-compose.yml docker-compose.dev.yml From 1da78549a6b3a35c2a66de033bea4126780cd789 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 18 Sep 2024 09:28:06 -0700 Subject: [PATCH 09/14] Remove docker-compose.yml from .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7b6ab784..f8801269 100644 --- a/.gitignore +++ b/.gitignore @@ -133,7 +133,6 @@ dmypy.json # Pyre type checker .pyre/ -docker-compose.yml docker-compose.dev.yml .DS_Store From 82dfe2e12e67e52b1370f221205af91bcf26c308 Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 18 Sep 2024 09:29:46 -0700 Subject: [PATCH 10/14] Revert to DB_HOST=db for docker-compose.dev.yml. --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d165ff15..1906101e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -25,7 +25,7 @@ services: depends_on: - db environment: - - DB_HOST=mongodb://db/DB_NAME + - DB_HOST=db - WEB_SERVER_HOST=0.0.0.0 - CRON_MODE= - STUDY_CONFIG=stage-program From ce28e3e085e203dc110ea21f767d00c2d8f0160d Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 18 Sep 2024 09:35:04 -0700 Subject: [PATCH 11/14] Remove the check for correct number of args in load_mongodump.sh --- viz_scripts/docker/load_mongodump.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index b85d10e1..a65117d9 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -6,14 +6,6 @@ SCRIPT_DIR="$(dirname "$0")" # Path to the configuration file (one level up) CONFIG_FILE="$SCRIPT_DIR/../../docker-compose.dev.yml" -# Check if the correct number of arguments is provided -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - echo " : The path to the MongoDB dump file to be restored." - echo " run git add -f after using this command" - exit 1 -fi - MONGODUMP_FILE=$1 # Print debug information From 8d25f439c3979c400fca3f17a89795ad611a424f Mon Sep 17 00:00:00 2001 From: iantei Date: Wed, 18 Sep 2024 09:36:26 -0700 Subject: [PATCH 12/14] Update path to config file commentto two levels up instead of one level up in load_mongodump.sh --- viz_scripts/docker/load_mongodump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index a65117d9..9dbff43c 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -3,7 +3,7 @@ # Directory of the script SCRIPT_DIR="$(dirname "$0")" -# Path to the configuration file (one level up) +# Path to the configuration file (two levels up) CONFIG_FILE="$SCRIPT_DIR/../../docker-compose.dev.yml" MONGODUMP_FILE=$1 From fea748bf71479427c8e4260d7817fbbe397f61ad Mon Sep 17 00:00:00 2001 From: iantei Date: Sat, 21 Sep 2024 09:38:00 -0700 Subject: [PATCH 13/14] Re-introduce check for correct number of args --- viz_scripts/docker/load_mongodump.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 9dbff43c..780680bb 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -6,6 +6,13 @@ SCRIPT_DIR="$(dirname "$0")" # Path to the configuration file (two levels up) CONFIG_FILE="$SCRIPT_DIR/../../docker-compose.dev.yml" +# Check if the correct number of arguments is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " : The path to the MongoDB dump file to be restored." + exit 1 +fi + MONGODUMP_FILE=$1 # Print debug information From 458e0f88df7dd9acc96241fd728a421c2335bdfc Mon Sep 17 00:00:00 2001 From: iantei Date: Sat, 21 Sep 2024 10:24:32 -0700 Subject: [PATCH 14/14] Update the sed command from DB_HOST:xx to DB_HOST=xx --- viz_scripts/docker/load_mongodump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/docker/load_mongodump.sh b/viz_scripts/docker/load_mongodump.sh index 780680bb..64fc71a6 100644 --- a/viz_scripts/docker/load_mongodump.sh +++ b/viz_scripts/docker/load_mongodump.sh @@ -51,7 +51,7 @@ echo "Database Name: $DB_NAME" # Update the docker-compose configuration file with the actual DB_HOST DB_HOST="mongodb://db/$DB_NAME" -sed -i.bak "s|DB_HOST:.*|DB_HOST: $DB_HOST|" "$CONFIG_FILE" +sed -i.bak "s|DB_HOST=.*|DB_HOST=$DB_HOST|" "$CONFIG_FILE" echo "Updated docker-compose file:" cat "$CONFIG_FILE"