Merge pull request #1678 from ral-facilities/dependabot/npm_and_yarn/… #4150
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Build | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| # By default, the pull_request event type is not triggered when a PR is merged into main or develop | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| jobs: | |
| lint-and-unit-test: | |
| name: Lint & Unit Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
| with: | |
| node-version: 20 | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| sudo apt-get install libgconf-2-4 | |
| yarn --immutable | |
| # Linting and unit testing | |
| - name: Run linting | |
| run: yarn lint | |
| - name: Run unit tests | |
| run: yarn test:unit | |
| # Test coverage upload | |
| - name: Upload unit test coverage for the Common package | |
| if: success() | |
| uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 | |
| with: | |
| directory: ./packages/datagateway-common/ | |
| flags: common | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload unit test coverage for the DataView package | |
| if: success() | |
| uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 | |
| with: | |
| directory: ./packages/datagateway-dataview/ | |
| flags: dataview | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload unit test coverage for the Search package | |
| if: success() | |
| uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 | |
| with: | |
| directory: ./packages/datagateway-search/ | |
| flags: search | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload unit test coverage for the Download package | |
| if: success() | |
| uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 | |
| with: | |
| directory: ./packages/datagateway-download/ | |
| flags: download | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| dataview-e2e-tests: | |
| name: DataGateway DataView End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| # Fixes on ICAT components needed for e2e tests | |
| - name: Add anon user to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v9.0.3 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
| with: | |
| node-version: 20 | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| sudo apt-get install libgconf-2-4 | |
| yarn --immutable | |
| - name: Run datagateway-dataview e2e tests | |
| run: yarn workspace datagateway-dataview run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 | |
| with: | |
| name: DataView-Screenshots | |
| path: packages/datagateway-dataview/cypress/screenshots | |
| download-e2e-tests: | |
| name: DataGateway Download End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| # Fixes on ICAT components needed for e2e tests | |
| - name: Removing authenticator prefix for simple auth | |
| run: | | |
| sed -i 's/mechanism = simple/!mechanism = simple/' /home/runner/install/authn.simple/run.properties | |
| - name: Adding Chris481 user | |
| run: | | |
| sed -i '/user\.list/ s/$/ Chris481/' /home/runner/install/authn.simple/run.properties | |
| - name: Adding Chris481 user password | |
| run: | | |
| echo "user.Chris481.password = pw" >> /home/runner/install/authn.simple/run.properties | |
| - name: Reinstall authn.simple | |
| run: | | |
| cd /home/runner/install/authn.simple/ && ./setup -vv install | |
| - name: Add anon, root (simple without prefix) and Chris481 users to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= root Chris481 anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Disable Two Level Storage | |
| run: | | |
| sed -i '/plugin.archive.class/,/tidyBlockSize/ s/^/#/' /home/runner/install/ids.server/run.properties | |
| - name: Reinstall IDS Server | |
| run: | | |
| cd /home/runner/install/ids.server/ && python2 ./setup -vv install | |
| - name: Add root (simple without prefix) to datagateway-download-api adminUserNames | |
| run: | | |
| awk -F" =" '/adminUserNames/{$2="= root";print;next}1' /home/runner/install/datagateway-download-api/run.properties > /home/runner/install/datagateway-download-api/run.properties.tmp | |
| - name: Apply adminUserNames change | |
| run: | | |
| mv -f /home/runner/install/datagateway-download-api/run.properties.tmp /home/runner/install/datagateway-download-api/run.properties | |
| - name: Reinstall datagateway-download-api | |
| run: | | |
| cd /home/runner/install/datagateway-download-api/ && python2 ./setup -vv install | |
| # Disable Globus for Download e2e tests | |
| - name: Login to ICAT | |
| run: | | |
| curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output | |
| - name: Disable Globus for testing | |
| run: | | |
| curl -k --request PUT 'https://localhost:8181/topcat/admin/downloadType/globus/status' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` --data-urlencode 'facilityName=LILS' --data-urlencode 'disabled=True' --data-urlencode 'message="GLOBUS has been disabled for testing"' | |
| - name: Remove session ID data | |
| run: rm -f login_output | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v9.0.3 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # DOI minter setup | |
| - name: Adding 'User-defined' DataPublicationType (needed for DOI minting api) | |
| run: cd datagateway-api/; poetry run python ../.github/add_doi_datapublicationtype.py | |
| - name: 'Add password to env file' | |
| run: echo DATACITE_PASSWORD=${{ secrets.DATACITE_PASSWORD }} >> ./.github/config.env | |
| - name: Run minting api | |
| run: docker run --env-file ./.github/config.env -p 8000:8000 --add-host host.docker.internal:host-gateway -d harbor.stfc.ac.uk/icat/doi-mint-api | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
| with: | |
| node-version: 20 | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| sudo apt-get install libgconf-2-4 | |
| yarn --immutable | |
| - name: Run datagateway-download e2e tests | |
| run: yarn workspace datagateway-download run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 | |
| with: | |
| name: Download-Screenshots | |
| path: packages/datagateway-download/cypress/screenshots | |
| search-e2e-tests: | |
| name: DataGateway Search End to End Tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| - name: Add apt repo | |
| run: sudo add-apt-repository universe | |
| - name: Setup Java | |
| uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: 8 | |
| java-package: jdk | |
| - name: Setup Python | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 | |
| with: | |
| python-version: 3.6 | |
| architecture: x64 | |
| # ICAT Ansible clone and install dependencies | |
| - name: Checkout icat-ansible | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: icatproject-contrib/icat-ansible | |
| ref: master | |
| path: icat-ansible | |
| - name: Install Ansible | |
| run: pip install -r icat-ansible/requirements.txt | |
| # Prep for running the playbook | |
| - name: Create hosts file | |
| run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts | |
| - name: Prepare vault pass | |
| run: echo -e "icattravispw" > icat-ansible/vault_pass.txt | |
| - name: Move vault to directory it'll get detected by Ansible | |
| run: mv icat-ansible/vault.yml icat-ansible/group_vars/all | |
| - name: Replace default payara user with Actions user | |
| run: | | |
| sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml | |
| - name: Change icat.server version | |
| run: | | |
| echo "icat_server_version: '6.1.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml | |
| - name: Change icat.lucene version | |
| run: | | |
| echo "icat_lucene_version: '3.0.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml | |
| - name: Amending roles | |
| run: | | |
| sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml | |
| # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions | |
| - name: Change hostname to localhost | |
| run: sudo hostname -b localhost | |
| # Remove existing MySQL installation so it doesn't interfere with GitHub Actions | |
| - name: Remove existing mysql | |
| run: | | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | |
| sudo apt-get remove --purge "mysql*" | |
| sudo rm -rf /var/lib/mysql* /etc/mysql | |
| # Create local instance of ICAT | |
| - name: Run ICAT Ansible Playbook | |
| run: | | |
| ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv | |
| - name: Add anon user to rootUserNames | |
| run: | | |
| awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp | |
| - name: Apply rootUserNames change | |
| run: | | |
| mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties | |
| - name: Reinstall ICAT Server | |
| run: | | |
| cd /home/runner/install/icat.server/ && ./setup -vv install | |
| - name: Checkout datagateway-api | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: ral-facilities/datagateway-api | |
| path: datagateway-api | |
| ref: v9.0.3 | |
| # DataGateway API file setup | |
| - name: Create search_api_mapping.json | |
| run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json | |
| - name: Create log file | |
| run: touch datagateway-api/datagateway_api/logs.log | |
| - name: Create config.yaml | |
| run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml | |
| # DataGateway API dependencies | |
| - name: Install Poetry | |
| run: pip install poetry | |
| - name: Install dependencies | |
| run: cd datagateway-api/; poetry install | |
| - name: Add dummy data to icatdb | |
| run: | | |
| cd datagateway-api/; poetry run python -m util.icat_db_generator | |
| - name: Add ICAT rules & public steps | |
| run: cd datagateway-api/; poetry run python ../.github/add_icat_rules.py | |
| # Recreate Lucene indexes | |
| - name: Use icatadmin to reindex all indexes | |
| run: ~/bin/icatadmin https://localhost:8181 simple username root password pw -- populate --delete | |
| - name: Let reindexes complete | |
| run: sleep 30 | |
| - name: Start API | |
| run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt & | |
| # E2E tests | |
| - name: Setup Node.js | |
| uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
| with: | |
| node-version: 20 | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress) | |
| run: | | |
| sudo apt-get install libgconf-2-4 | |
| yarn --immutable | |
| - name: Run datagateway-search e2e tests | |
| run: yarn workspace datagateway-search run e2e | |
| - name: Upload Cypress screenshots | |
| if: failure() | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 | |
| with: | |
| name: Search-Screenshots | |
| path: packages/datagateway-search/cypress/screenshots | |
| docker: | |
| # This job triggers only if all the other jobs succeed. It builds the Docker image and if successful, | |
| # it pushes it to Harbor. | |
| needs: [lint-and-unit-test, dataview-e2e-tests, download-e2e-tests, search-e2e-tests] | |
| name: Docker | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| - name: Login to Harbor | |
| uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 | |
| with: | |
| registry: ${{ secrets.HARBOR_URL }} | |
| username: ${{ secrets.HARBOR_USERNAME }} | |
| password: ${{ secrets.HARBOR_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 | |
| with: | |
| images: ${{ secrets.HARBOR_URL }}/plugins | |
| - name: Build and push Docker image to Harbor | |
| uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4.2.1 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} |