Added maintainance.md file and updated fixtures for spammer editor

metabrainz · Aug 23, 2024 · c8c59ba · c8c59ba
1 parent 9269994
commit c8c59ba
Show file tree

Hide file tree

Showing 9 changed files with 402 additions and 380 deletions.
diff --git a/docs/maintainance.md b/docs/maintainance.md
@@ -0,0 +1,3 @@
+# Maintaining the project
+
+- The project depends on `musicbrainz_db`, therefore, make sure all the `CREATE TABLE musicbrainz.*` instructions, present in `scripts/sql` scripts are in sync with MusicBrainz database schema.
diff --git a/scripts/generate_fixtures.sh b/scripts/generate_fixtures.sh
@@ -16,33 +16,35 @@ backup_tables=()
 
 # Function to create, verify, dump a new table
 dump_table() {
-    local table=$1
-    local schema_name=$(echo $table | awk -F '.' '{print $1}')
-    local base_name=$(echo $table | awk -F '.' '{print $2}')
+    local table="$1"
+    local schema_name
+    schema_name=$(echo "$table" | awk -F '.' '{print $1}')
+    local base_name
+    base_name=$(echo "$table" | awk -F '.' '{print $2}')
     local new_table_name="backup_${base_name}"
     local filepath="$dump_dir/${base_name}_dump.sql"
 
     echo "Attempting to create new table $schema_name.$new_table_name for $table..."
 
     # Create the new table in the correct schema
     if [[ "$table" == "musicbrainz.edit_data" ]]; then
-        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+        PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
         CREATE TABLE $schema_name.$new_table_name AS
         SELECT * FROM $table WHERE edit >= 111450838 ORDER BY edit LIMIT 100;
         "
     elif [[ "$table" == "musicbrainz.edit_note" ]]; then
-        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+        PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
         CREATE TABLE $schema_name.$new_table_name AS
         SELECT * FROM $table WHERE id >= 71024901 ORDER BY id LIMIT 100;
         "
     elif [[ "$table" == "musicbrainz.edit" ]]; then
-        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+        PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
         CREATE TABLE $schema_name.$new_table_name AS
         SELECT * FROM $table WHERE id IN (SELECT edit FROM $schema_name.backup_edit_data);
         "
     elif [[ "$table" == "musicbrainz.editor" ]]; then
         echo "Selected editor IDs for $table:"
-        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+        PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
         CREATE TABLE $schema_name.$new_table_name AS
         SELECT * FROM $table WHERE id IN (
             SELECT editor FROM $schema_name.backup_edit_note
@@ -51,13 +53,13 @@ dump_table() {
         );
         "
     else
-        PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+        PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
         CREATE TABLE $schema_name.$new_table_name AS
         SELECT * FROM $table LIMIT 100;
         "
     fi
 
-    if [ $? -ne 0 ]; then
+    if ! PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1;" > /dev/null 2>&1; then
         echo "Failed to create table $schema_name.$new_table_name for $table"
         exit 1
     fi
@@ -70,9 +72,9 @@ dump_table() {
     # Dump the new table
     touch "$filepath"
     echo "Dumping table $schema_name.$new_table_name to $filepath..."
-    PGPASSWORD=$PGPASSWORD pg_dump -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE --data-only --table=$schema_name.$new_table_name --inserts --column-inserts --no-owner --no-privileges -f "$filepath"
+    PGPASSWORD="$PGPASSWORD" pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" --data-only --table="$schema_name.$new_table_name" --inserts --column-inserts --no-owner --no-privileges -f "$filepath"
 
-    if [ $? -ne 0 ]; then
+    if ! PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1;" > /dev/null 2>&1; then
         echo "Failed to dump table $schema_name.$new_table_name"
         exit 1
     fi
@@ -85,17 +87,17 @@ dump_table() {
 
 # Dump the specific tables first
 for table in "${tables[@]}"; do
-    dump_table $table
+    dump_table "$table"
 done
 
 # Drop all backup tables after dumping
 for backup_table in "${backup_tables[@]}"; do
     echo "Dropping table $backup_table..."
-    PGPASSWORD=$PGPASSWORD psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE -c "
+    PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "
     DROP TABLE IF EXISTS $backup_table;
     "
 
-    if [ $? -ne 0 ]; then
+    if ! PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1;" > /dev/null 2>&1; then
         echo "Failed to drop table $backup_table"
         exit 1
     fi

diff --git a/src/poller/tests/utils.rs b/src/poller/tests/utils.rs
@@ -268,7 +268,8 @@ async fn test_get_edit_data_and_note_start_id(pool: PgPool) -> Result<(), Error>
     "../../../tests/fixtures/Editor.sql",
     "../../../tests/fixtures/editor_dump.sql",
     "../../../tests/fixtures/EditNote.sql",
-    "../../../tests/fixtures/edit_note_dump.sql"
+    "../../../tests/fixtures/edit_note_dump.sql",
+    "../../../tests/fixtures/InitSpammer.sql"
 ))]
 async fn test_extract_url_from_edit_note(pool: PgPool) -> Result<(), Error> {
     let note_with_no_url = sqlx::query_as::<_, EditNote>(
@@ -319,7 +320,8 @@ async fn test_extract_url_from_edit_note(pool: PgPool) -> Result<(), Error> {
     "../../../tests/fixtures/EditData.sql",
     "../../../tests/fixtures/edit_data_dump.sql",
     "../../../tests/fixtures/Edit.sql",
-    "../../../tests/fixtures/edit_dump.sql"
+    "../../../tests/fixtures/edit_dump.sql",
+    "../../../tests/fixtures/InitSpammer.sql"
 ))]
 async fn test_extract_url_from_edit_data(pool: PgPool) -> Result<(), Error> {
     let edit_with_no_url = sqlx::query_as::<_, EditData>(

diff --git a/tests/fixtures/Edit.sql b/tests/fixtures/Edit.sql
@@ -12,7 +12,4 @@ CREATE TABLE musicbrainz.edit
     expire_time          TIMESTAMP WITH TIME ZONE NOT NULL,
     language            INTEGER, -- references language.id
     quality             SMALLINT NOT NULL DEFAULT 1
-);
-
--- Entry for spammer editor
-INSERT INTO musicbrainz.edit (id, editor, type, status, autoedit, open_time, close_time, expire_time, language, quality) VALUES (21965, 1, 207, 3, 0, '2000-11-14 11:51:55+00', '2000-11-16 11:51:55+00', '2000-11-16 03:51:55+00', NULL, 1);
+);
diff --git a/tests/fixtures/EditData.sql b/tests/fixtures/EditData.sql
@@ -3,7 +3,4 @@ CREATE SCHEMA IF NOT EXISTS musicbrainz;
 CREATE TABLE musicbrainz.edit_data (
     edit                INTEGER NOT NULL, -- PK, references edit.id
     data                JSONB NOT NULL
-);
-
--- Spammer editor
-INSERT INTO musicbrainz.edit_data (edit, data) VALUES (21965, '{"new": {"sort_name": "Animals, The"}, "old": {"sort_name": "The Animals"}, "entity": {"id": "1433", "name": "The Animals"}}');
+);
diff --git a/tests/fixtures/EditNote.sql b/tests/fixtures/EditNote.sql
@@ -7,6 +7,3 @@ CREATE TABLE musicbrainz.edit_note (
     text                TEXT NOT NULL,
     post_time            TIMESTAMP WITH TIME ZONE DEFAULT NOW()
 );
-
--- Spammer editor
-INSERT INTO musicbrainz.edit_note (id, editor, edit, text, post_time) VALUES (771, 1, 85521, 'This edit moderation clashes with an existing item in the database.', NULL);
diff --git a/tests/fixtures/Editor.sql b/tests/fixtures/Editor.sql
@@ -19,5 +19,3 @@ CREATE TABLE musicbrainz.editor
     ha1                 CHAR(32) NOT NULL,
     deleted             BOOLEAN NOT NULL DEFAULT FALSE
 );
--- Spammer
-INSERT INTO musicbrainz.editor (id, name, privs, email, website, bio, member_since, email_confirm_date, last_login_date, last_updated, birth_date, gender, area, password, ha1, deleted) VALUES (1, 'Anonymous', 4096, '', NULL, NULL, NULL, '2009-10-18 18:20:17.333759+00', '2024-05-04 00:17:56.699735+00', '2018-03-15 08:15:36.728395+00', NULL, NULL, NULL, '{CLEARTEXT}mb', 'fad9cdfaf96a2ddb0eceb7b07d269bea', false);
diff --git a/tests/fixtures/InitSpammer.sql b/tests/fixtures/InitSpammer.sql
@@ -0,0 +1,26 @@
+DO $$
+BEGIN
+   -- Insert into musicbrainz.editor if the table exists
+   IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'musicbrainz' AND table_name = 'editor') THEN
+      INSERT INTO musicbrainz.editor (id, name, privs, email, website, bio, member_since, email_confirm_date, last_login_date, last_updated, birth_date, gender, area, password, ha1, deleted)
+      VALUES (1, 'Anonymous', 4096, '', NULL, NULL, NULL, '2009-10-18 18:20:17.333759+00', '2024-05-04 00:17:56.699735+00', '2018-03-15 08:15:36.728395+00', NULL, NULL, NULL, '{CLEARTEXT}mb', 'fad9cdfaf96a2ddb0eceb7b07d269bea', false);
+   END IF;
+
+   -- Insert into musicbrainz.edit if the table exists
+   IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'musicbrainz' AND table_name = 'edit') THEN
+      INSERT INTO musicbrainz.edit (id, editor, type, status, autoedit, open_time, close_time, expire_time, language, quality)
+      VALUES (21965, 1, 207, 3, 0, '2000-11-14 11:51:55+00', '2000-11-16 11:51:55+00', '2000-11-16 03:51:55+00', NULL, 1);
+   END IF;
+
+   -- Insert into musicbrainz.edit_data if the table exists
+   IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'musicbrainz' AND table_name = 'edit_data') THEN
+      INSERT INTO musicbrainz.edit_data (edit, data)
+      VALUES (21965, '{"new": {"sort_name": "Animals, The"}, "old": {"sort_name": "The Animals"}, "entity": {"id": "1433", "name": "The Animals"}}');
+   END IF;
+
+   -- Insert into musicbrainz.edit_note if the table exists
+   IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'musicbrainz' AND table_name = 'edit_note') THEN
+      INSERT INTO musicbrainz.edit_note (id, editor, edit, text, post_time)
+      VALUES (771, 1, 85521, 'This edit moderation clashes with an existing item in the database.', NULL);
+   END IF;
+END $$;