Task/59421 help resolving database encoding related exception when migration to v15

config/initializers/03-db_check.rb

@@ -30,7 +30,7 @@
 
 if (db_config = ActiveRecord::Base.configurations.configs_for(env_name: env)[0]) &&
    db_config.configuration_hash["adapter"]&.start_with?("mysql")
-  warn <<~ERROR
+  abort <<~ERROR
     ======= INCOMPATIBLE DATABASE DETECTED =======
     Your database is set up for use with a MySQL or MySQL-compatible variant.
     This installation of OpenProject no longer supports these variants.
@@ -45,8 +45,4 @@
 
     ==============================================
   ERROR
-
-  # rubocop:disable Rails/Exit
-  Kernel.exit 1
-  # rubocop:enable Rails/Exit
 end

config/initializers/05-check_db_encoding.rb

@@ -0,0 +1,53 @@
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+if ENV["OPENPROJECT_SKIP_DB_ENCODING_CHECK"].blank?
+  icu_incompatible_encodings = %w[
+    EUC_JIS_2004
+    LATIN10
+    MULE_INTERNAL
+    SQL_ASCII
+    WIN874
+  ]
+
+  database_encoding = ActiveRecord::Base.connection.select_value("SHOW SERVER_ENCODING")
+
+  if database_encoding.in?(icu_incompatible_encodings)
+    abort <<~ERROR
+      INCOMPATIBLE DATABASE ENCODING DETECTED
+
+      Your database encoding is #{database_encoding}, which is incompatible with ICU
+      collation used in OpenProject v15.
+
+      Please check the instructions on how to change database encoding:
+      https://www.openproject.org/docs/installation-and-operations/misc/changing-database-encoding/
+
+      This check can be skipped by setting environment variable OPENPROJECT_SKIP_DB_ENCODING_CHECK=true
+    ERROR
+  end
+end

db/migrate/20240920152544_set_versions_name_collation.rb

@@ -1,8 +1,23 @@
 class SetVersionsNameCollation < ActiveRecord::Migration[7.1]
   def up
-    execute <<-SQL.squish
-      CREATE COLLATION IF NOT EXISTS versions_name (provider = icu, locale = 'und-u-kn-true');
-    SQL
+    begin
+      execute <<-SQL.squish
+        CREATE COLLATION IF NOT EXISTS versions_name (provider = icu, locale = 'und-u-kn-true');
+      SQL
+    rescue StandardError => e
+      raise unless e.message.include?("encoding")
+
+      abort <<~MESSAGE
+        \e[31mERROR:\e[0m Failed to create an ICU collation with current database encoding.
+        You need to change the database encoding before proceeding.
+
+        Please check the instructions on how to do it:
+        https://www.openproject.org/docs/installation-and-operations/misc/changing-database-encoding/
+
+        Original error:
+          #{e.message}
+      MESSAGE
+    end
 
     change_column :versions, :name, :string, collation: "versions_name"
   end

docs/installation-and-operations/misc/changing-database-encoding/README.md

@@ -0,0 +1,69 @@
+# Changing database encoding
+
+This instructions are primarily intended to help with an error encountered when migrating to OpenProject 15.
+The error happens when migration tries to create an ICU collation and database encoding doesn't support it.
+We suggest to use unicode encoding for maximum compatibility.
+
+## Preconditions
+
+* Credentials with the permission to create a database in the database server the OpenProject installation is running against.
+* Shell access to the OpenProject server.
+
+## 1. Create a database dump
+
+This and following steps assume that you are using built in `openproject` command.
+
+```shell
+openproject run backup
+```
+
+Ensure it finished successfully and note down the path after `Generating database backup` that should normally be
+in form `/var/db/openproject/backup/postgresql-dump-<DATE_TIME_DIGITS>.pgdump`.
+
+See also [Backing up your OpenProject installation page](../../operation/backing-up).
+
+## 2. Create a new database with different encoding
+
+Note down the database connection URL that should be in form `postgres://<USERNAME>:<PASSWORD>@<HOST>:<PORT>/<DATABASE>`:
+
+```shell
+openproject config:get DATABASE_URL
+```
+
+Create new database using `psql` command, after deciding on the name, for example `openproject-unicode`:
+
+```shell
+psql '<DATABASE_URL>' -c 'CREATE DATABASE "<NEW_DATABASE_NAME>" ENCODING UNICODE'
+```
+
+Options for `CREATE DATABASE` can be found at [PostgreSQL documentation page](https://www.postgresql.org/docs/current/sql-createdatabase.html).
+
+Or alternatively using `createdb` command:
+
+```shell
+su postgres -c createdb -E UNICODE -O <dbusernamer> openproject_backup
+```
+
+Instructions for `createdb` command can be found at [PostgreSQL documentation page](https://www.postgresql.org/docs/17/app-createdb.html).
+
+## 3. Restore the dump to the new database
+
+To get the new database URL you need to replace the old database name with the new database name in the connection URL that you got in the previous step.
+For example if it was `postgres://openproject:hard-password@some-host:5432/openproject` and new database name was chosen to be `openproject-unicode`, then
+new database URL will be `postgres://openproject:hard-password@some-host:5432/openproject-unicode`.
+
+```shell
+pg_restore -d '<NEW_DATABASE_URL>' '<PATH_TO_THE_DATABASE_DUMP>'
+```
+
+See also [Restoring an OpenProject backup](../../operation/restoring/).
+
+## 4. Change configuration to use the new database
+
+Using the new database URL from previous step:
+
+```shell
+openproject config:set DATABASE_URL=<NEW_DATABASE_URL>
+```
+
+See also [Configuring a custom database server page](../../configuration/database/).

spec/features/admin/enterprise/enterprise_spec.rb

@@ -108,6 +108,7 @@
 
         # Remove token
         click_on "Delete"
+        wait_for_network_idle
 
         # Expect modal
         find_test_selector("confirmation-modal--confirmed").click

spec/features/members/error_messages_spec.rb

@@ -28,7 +28,7 @@
 
 require "spec_helper"
 
-RSpec.describe "Group memberships through groups page" do
+RSpec.describe "Group memberships through groups page", :js, :with_cuprite do
   shared_let(:admin) { create(:admin) }
   let!(:project) { create(:project, name: "Project 1", identifier: "project1") }
 

spec/features/members/membership_filter_spec.rb

@@ -28,7 +28,9 @@
 
 require "spec_helper"
 
-RSpec.describe "group memberships through groups page", :js do
+RSpec.describe "group memberships through groups page",
+               :js,
+               :with_cuprite do
   shared_let(:admin) { create(:admin) }
   let!(:project) { create(:project, name: "Project 1", identifier: "project1") }
 

spec/features/members/pagination_spec.rb

@@ -28,7 +28,7 @@
 
 require "spec_helper"
 
-RSpec.describe "members pagination", :js do
+RSpec.describe "members pagination", :js, :with_cuprite do
   shared_let(:admin) { create(:admin) }
   let(:project) do
     create(:project,
@@ -59,11 +59,9 @@
       members_page.set_items_per_page! 2
 
       members_page.visit!
-      SeleniumHubWaiter.wait
       expect(members_page).to have_user "Alice Alison" # members are sorted by last name desc
       members_page.add_user! "Peter Pan", as: "Manager"
 
-      SeleniumHubWaiter.wait
       members_page.go_to_page! 2
       expect(members_page).to have_user "Peter Pan"
     end
@@ -82,12 +80,10 @@
       members_page.set_items_per_page! 1
 
       members_page.visit!
-      SeleniumHubWaiter.wait
       members_page.remove_user! "Alice Alison"
       expect_and_dismiss_flash message: "Removed Alice Alison from project"
       expect(members_page).to have_user "Bob Bobbit"
 
-      SeleniumHubWaiter.wait
       members_page.go_to_page! 2
       expect(members_page).to have_user "Peter Pan"
     end
@@ -98,13 +94,11 @@
       members_page.set_items_per_page! 1
 
       members_page.visit!
-      SeleniumHubWaiter.wait
       members_page.go_to_page! 2
       members_page.edit_user! "Bob Bobbit", add_roles: ["Developer"]
       expect(page).to have_text "Successful update"
       expect(members_page).to have_user "Bob Bobbit", roles: ["Developer", "Manager"]
 
-      SeleniumHubWaiter.wait
       members_page.go_to_page! 1
       expect(members_page).to have_user "Alice Alison"
     end

spec/support/pages/groups.rb

@@ -69,7 +69,7 @@ def group(group_name)
     end
   end
 
-  class Group < Pages::Page
+  class Group < Page
     include ::Components::Autocompleter::NgSelectAutocompleteHelpers
     attr_reader :id
 

spec/support/pages/members.rb

@@ -40,12 +40,6 @@ def initialize(project_identifier)
       @project_identifier = project_identifier
     end
 
-    def visit!
-      super
-
-      self
-    end
-
     def path
       "/projects/#{project_identifier}/members"
     end
@@ -61,6 +55,7 @@ def open_filters!
     def search_for_name(name)
       fill_in "name", with: name
       find(".simple-filters--controls input[type=submit]").click
+      wait_for_reload
     end
 
     def expect_menu_item(text, selected: false)
@@ -188,6 +183,7 @@ def edit_user!(name, add_roles: [], remove_roles: [])
       Array(remove_roles).each { |role| uncheck role }
 
       click_on "Change"
+      wait_for_reload
     end
 
     def has_group_membership?(user_name)
@@ -270,6 +266,7 @@ def expect_role(role_name, present: true)
 
     def go_to_page!(number)
       find(".op-pagination--pages a", text: number.to_s).click
+      wait_for_reload
     end
   end
 end

spec/support/pages/page.rb

@@ -46,9 +46,9 @@ def current_page?
     def visit!
       raise "No path defined" unless path
 
-      visit path
+      visit(path)
 
-      self
+      wait_for_reload
     end
 
     def reload!