From 8a99944fb54fe7edee7f7de47c8a1d4807da9a45 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Thu, 22 Aug 2024 15:38:01 +0900 Subject: [PATCH 1/8] Adding PostgreSQL 13 and MySQL 8.3 to GitHub Actions. --- .github/workflows/check.yml | 277 +++++++++++++++++++++++++----------- 1 file changed, 194 insertions(+), 83 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cf569691..98b072ac 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,7 +1,7 @@ name: Check on: [ pull_request, push ] jobs: - mysql: + mysql5_7: runs-on: ubuntu-latest # push: always run. # pull_request: run only when the PR is submitted from a forked repository, not within this repository. @@ -13,34 +13,98 @@ jobs: image: mysql:5.7 options: --health-cmd "mysqladmin ping -h localhost" --health-interval 20s --health-timeout 10s --health-retries 10 ports: - - "3306:3306" + - "3306:3306" env: MYSQL_ROOT_PASSWORD: root MYSQL_USER: ci MYSQL_PASSWORD: password steps: - - uses: actions/checkout@v3 - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - cache: "gradle" - - name: Connect - run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "show databases;" - - name: Create database - run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "create database ci_test;" - - name: Build with testing - run: ./gradlew --stacktrace :embulk-output-mysql:check + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Connect + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "show databases;" + - name: Create database + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "create database ci_test;" + - name: Build with testing + run: ./gradlew --stacktrace :embulk-output-mysql:check + env: + _JAVA_OPTIONS: "-Xmx2048m -Xms512m" + EMBULK_OUTPUT_MYSQL_TEST_CONFIG: "${{ github.workspace }}/ci/mysql.yml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: mysql5_7 + path: embulk-output-mysql/build/reports/tests/test + mysql8_3: + runs-on: ubuntu-latest + # push: always run. + # pull_request: run only when the PR is submitted from a forked repository, not within this repository. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + services: + mysql: + # Due to MySQL 8.4 disabled mysql_native_password by default, + # Connector/J 5.x can't connect to database. + # So, Use MySQL 8.3. + image: mysql:8.3 + options: --health-cmd "mysqladmin ping -h localhost" --health-interval 20s --health-timeout 10s --health-retries 10 + ports: + - "3306:3306" env: - _JAVA_OPTIONS: "-Xmx2048m -Xms512m" - EMBULK_OUTPUT_MYSQL_TEST_CONFIG: "${{ github.workspace }}/ci/mysql.yml" - - uses: actions/upload-artifact@v3 - if: always() - with: - name: mysql - path: embulk-output-mysql/build/reports/tests/test - postgresql: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: ci + MYSQL_PASSWORD: password + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Connect + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "show databases;" + - name: show version + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "select version();" + - name: Create database + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "create database ci_test;" + # + # MySQL 8 uses caching_sha2_password mechanism by default. + # Connector/J 5.x doesn't support it. + # + # This part change password mechanism to mysql_native_password. + # Remove the following part after update Connector/J + # + - name: Show password plugins + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "SELECT user, host, plugin FROM mysql.user;" + - name: Change password mechanism1 (root@localhost) + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root';" + - name: Change password mechanism2 (root@%) + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'root';" + - name: FLUSH PRIVILEGES + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "FLUSH PRIVILEGES;" + - name: Show password plugins2 + run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "SELECT user, host, plugin FROM mysql.user;" + # + # End caching_sha2_password workaround. + # + - name: Build with testing + run: ./gradlew --stacktrace :embulk-output-mysql:check + env: + _JAVA_OPTIONS: "-Xmx2048m -Xms512m" + EMBULK_OUTPUT_MYSQL_TEST_CONFIG: "${{ github.workspace }}/ci/mysql.yml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: mysql8_3 + path: embulk-output-mysql/build/reports/tests/test + postgresql9_4: runs-on: ubuntu-latest # push: always run. # pull_request: run only when the PR is submitted from a forked repository, not within this repository. @@ -56,31 +120,78 @@ jobs: env: POSTGRES_PASSWORD: postgres steps: - - uses: actions/checkout@v3 - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - cache: "gradle" - - name: Connect - run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "\l" - env: - PGPASSWORD: postgres - - name: Create database - run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "create database ci_test;" - env: - PGPASSWORD: postgres - - name: Build with testing - run: ./gradlew --stacktrace :embulk-output-postgresql:check + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Connect + run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "\l" + env: + PGPASSWORD: postgres + - name: Create database + run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "create database ci_test;" + env: + PGPASSWORD: postgres + - name: Build with testing + run: ./gradlew --stacktrace :embulk-output-postgresql:check + env: + _JAVA_OPTIONS: "-Xmx2048m -Xms512m" + EMBULK_OUTPUT_POSTGRESQL_TEST_CONFIG: "${{ github.workspace }}/ci/postgresql.yml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: postgresql9_4 + path: embulk-output-postgresql/build/reports/tests/test + # PostgreSQL 14 and later, raise the exception "The authentication type 10 is not supported." + # Use PostgreSQL 13 at this time. + postgresql13: + runs-on: ubuntu-latest + # push: always run. + # pull_request: run only when the PR is submitted from a forked repository, not within this repository. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + services: + postgres: + image: postgres:13 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - "5432:5432" env: - _JAVA_OPTIONS: "-Xmx2048m -Xms512m" - EMBULK_OUTPUT_POSTGRESQL_TEST_CONFIG: "${{ github.workspace }}/ci/postgresql.yml" - - uses: actions/upload-artifact@v3 - if: always() - with: - name: postgresql - path: embulk-output-postgresql/build/reports/tests/test + POSTGRES_PASSWORD: postgres + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Connect + run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "\l" + env: + PGPASSWORD: postgres + - name: Show version + run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "select * from version();" + env: + PGPASSWORD: postgres + - name: Create database + run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "create database ci_test;" + env: + PGPASSWORD: postgres + - name: Build with testing + run: ./gradlew --stacktrace :embulk-output-postgresql:check + env: + _JAVA_OPTIONS: "-Xmx2048m -Xms512m" + EMBULK_OUTPUT_POSTGRESQL_TEST_CONFIG: "${{ github.workspace }}/ci/postgresql.yml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: postgresql13 + path: embulk-output-postgresql/build/reports/tests/test redshift: runs-on: ubuntu-latest # push: always run. @@ -98,33 +209,33 @@ jobs: env: POSTGRES_PASSWORD: postgres steps: - - uses: actions/checkout@v3 - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - cache: "gradle" - - name: Connect - run: psql -h 127.0.0.1 -p 5439 -U postgres -d postgres -c "\l" - env: - PGPASSWORD: postgres - - name: Create database - run: psql -h 127.0.0.1 -p 5439 -U postgres -d postgres -c "create database ci_test;" - env: - PGPASSWORD: postgres - - name: Build with testing - run: ./gradlew --stacktrace :embulk-output-redshift:check - env: - _JAVA_OPTIONS: "-Xmx2048m -Xms512m" - EMBULK_OUTPUT_REDSHIFT_TEST_CONFIG: "${{ github.workspace }}/ci/redshift.yml" - - uses: actions/upload-artifact@v3 - if: always() - with: - name: redshift - path: embulk-output-redshift/build/reports/tests/test - if-no-files-found: ignore - sqlserver: + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Connect + run: psql -h 127.0.0.1 -p 5439 -U postgres -d postgres -c "\l" + env: + PGPASSWORD: postgres + - name: Create database + run: psql -h 127.0.0.1 -p 5439 -U postgres -d postgres -c "create database ci_test;" + env: + PGPASSWORD: postgres + - name: Build with testing + run: ./gradlew --stacktrace :embulk-output-redshift:check + env: + _JAVA_OPTIONS: "-Xmx2048m -Xms512m" + EMBULK_OUTPUT_REDSHIFT_TEST_CONFIG: "${{ github.workspace }}/ci/redshift.yml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: redshift + path: embulk-output-redshift/build/reports/tests/test + if-no-files-found: ignore + sqlserver: # TODO: Use https://hub.docker.com/_/microsoft-mssql-server runs-on: ubuntu-latest # push: always run. # pull_request: run only when the PR is submitted from a forked repository, not within this repository. @@ -132,12 +243,12 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v3 - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - cache: "gradle" - - name: Build-only - run: ./gradlew --stacktrace :embulk-output-sqlserver:compileJava :embulk-output-sqlserver:compileTestJava + - uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'zulu' + cache: "gradle" + - name: Build-only + run: ./gradlew --stacktrace :embulk-output-sqlserver:compileJava :embulk-output-sqlserver:compileTestJava From 5f55dcbce88d70fcae7b5909942bb41a81e9f175 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 28 Aug 2024 16:39:41 +0900 Subject: [PATCH 2/8] Update .github/workflows/check.yml Co-authored-by: Dai MIKURUBE --- .github/workflows/check.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 98b072ac..c194d574 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,9 +49,10 @@ jobs: fail-fast: false services: mysql: - # Due to MySQL 8.4 disabled mysql_native_password by default, - # Connector/J 5.x can't connect to database. - # So, Use MySQL 8.3. + # Testing with MySQL 8.3 here, neither with 8.4 nor 9.0, because + # MySQL 8.4 has disabled mysql_native_password by default. + # Connector/J 5.x cannot connect to MySQL 8.4 due to this. + # TODO: Start testing with MySQL 8.4 and/or 9.0 with Connector/J 8.x. image: mysql:8.3 options: --health-cmd "mysqladmin ping -h localhost" --health-interval 20s --health-timeout 10s --health-retries 10 ports: From 300d8a578e6ceb23f4b8d2899ae41c54099f3ec1 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 28 Aug 2024 16:39:52 +0900 Subject: [PATCH 3/8] Update .github/workflows/check.yml Co-authored-by: Dai MIKURUBE --- .github/workflows/check.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c194d574..706f4561 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -75,13 +75,10 @@ jobs: run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "select version();" - name: Create database run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "create database ci_test;" - # - # MySQL 8 uses caching_sha2_password mechanism by default. - # Connector/J 5.x doesn't support it. - # - # This part change password mechanism to mysql_native_password. - # Remove the following part after update Connector/J - # + # Workaround to change MySQL's password mechanism to `mysql_native_password` + # from `caching_sha2_password` that is the default in MySQL 8 because + # Connector/J 5.x does not support `caching_sha2_password`. + # TODO: Start testing with `caching_sha2_password` with Connector/J 8.x. - name: Show password plugins run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "SELECT user, host, plugin FROM mysql.user;" - name: Change password mechanism1 (root@localhost) From 2c5b8658ed8b186b0e0baca141c3828596698380 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 28 Aug 2024 17:20:44 +0900 Subject: [PATCH 4/8] Use double quotes instead of single quotes for consistency. --- .github/workflows/check.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 706f4561..63edcd83 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -24,7 +24,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Connect run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "show databases;" @@ -67,7 +67,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Connect run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "show databases;" @@ -123,7 +123,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Connect run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "\l" @@ -166,7 +166,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Connect run: psql -h 127.0.0.1 -p 5432 -U postgres -d postgres -c "\l" @@ -212,7 +212,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Connect run: psql -h 127.0.0.1 -p 5439 -U postgres -d postgres -c "\l" @@ -246,7 +246,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 8 - distribution: 'zulu' + distribution: "zulu" cache: "gradle" - name: Build-only run: ./gradlew --stacktrace :embulk-output-sqlserver:compileJava :embulk-output-sqlserver:compileTestJava From a179240f2dd4374480a94b0fd3c0dc1164238347 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 28 Aug 2024 17:22:44 +0900 Subject: [PATCH 5/8] Simplify changing password mechanism steps. --- .github/workflows/check.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 63edcd83..91677627 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -79,19 +79,14 @@ jobs: # from `caching_sha2_password` that is the default in MySQL 8 because # Connector/J 5.x does not support `caching_sha2_password`. # TODO: Start testing with `caching_sha2_password` with Connector/J 8.x. - - name: Show password plugins - run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "SELECT user, host, plugin FROM mysql.user;" - - name: Change password mechanism1 (root@localhost) + # + # See also: https://dev.mysql.com/doc/refman/8.4/en/account-names.html + - name: Change password (root@localhost) run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root';" - - name: Change password mechanism2 (root@%) + - name: Change password (root@%) run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'root';" - name: FLUSH PRIVILEGES run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "FLUSH PRIVILEGES;" - - name: Show password plugins2 - run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "SELECT user, host, plugin FROM mysql.user;" - # - # End caching_sha2_password workaround. - # - name: Build with testing run: ./gradlew --stacktrace :embulk-output-mysql:check env: From 1112f6c9b5ee049d666e46471833bbf6cae84dbd Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 28 Aug 2024 17:26:38 +0900 Subject: [PATCH 6/8] Update .github/workflows/check.yml Co-authored-by: Dai MIKURUBE --- .github/workflows/check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 91677627..cd337877 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -138,8 +138,9 @@ jobs: with: name: postgresql9_4 path: embulk-output-postgresql/build/reports/tests/test - # PostgreSQL 14 and later, raise the exception "The authentication type 10 is not supported." - # Use PostgreSQL 13 at this time. + # Testing with PostgreSQL 13 here, not with PostgreSQL 14 or later, because + # it raises an exception: "The authentication type 10 is not supported." + # TODO: Start testing with PostgreSQL 14 or later. postgresql13: runs-on: ubuntu-latest # push: always run. From cc58a28bd21498ac9f1bdabd1b98b42f910b42a6 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Thu, 29 Aug 2024 13:48:59 +0900 Subject: [PATCH 7/8] Update .github/workflows/check.yml Co-authored-by: Dai MIKURUBE --- .github/workflows/check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cd337877..f39c27f4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -78,9 +78,8 @@ jobs: # Workaround to change MySQL's password mechanism to `mysql_native_password` # from `caching_sha2_password` that is the default in MySQL 8 because # Connector/J 5.x does not support `caching_sha2_password`. + # See: https://dev.mysql.com/doc/refman/8.4/en/account-names.html # TODO: Start testing with `caching_sha2_password` with Connector/J 8.x. - # - # See also: https://dev.mysql.com/doc/refman/8.4/en/account-names.html - name: Change password (root@localhost) run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root';" - name: Change password (root@%) From 2172ffe9318fc080f90249d810ec26becf494e8d Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Thu, 29 Aug 2024 13:54:03 +0900 Subject: [PATCH 8/8] Remove FLUSH PRIVILEGES from the GitHub Actions --- .github/workflows/check.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f39c27f4..f7b0ec1d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -84,8 +84,6 @@ jobs: run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root';" - name: Change password (root@%) run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'root';" - - name: FLUSH PRIVILEGES - run: mysql -h 127.0.0.1 --port 3306 -uroot -proot -e "FLUSH PRIVILEGES;" - name: Build with testing run: ./gradlew --stacktrace :embulk-output-mysql:check env: