From 4f163975b3a108c0d4c6bb5853bb668282ae65d1 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 22 Dec 2023 14:09:35 +0800 Subject: [PATCH] Problem: integration tests CI spend a lot of time (backport #1271) (#1277) * Problem: integration tests CI spend a lot of time (#1271) * Problem: integration tests CI spend a lot of time Solution: - run in parallel using markers and job matrix * fix matrix * flock is not nesserary * fix marker * break up ibc test cases * separate gov test cases * split out some slow test cases * split more ibc test * Update .github/workflows/test.yml Signed-off-by: yihuang * fix pytest lint * more balance --------- Signed-off-by: yihuang * Update integration_tests/conftest.py Co-authored-by: yihuang Signed-off-by: mmsqe --------- Signed-off-by: yihuang Signed-off-by: mmsqe Co-authored-by: yihuang --- .github/workflows/test.yml | 5 +++++ Makefile | 6 ++++++ integration_tests/conftest.py | 11 +++++++++++ integration_tests/test_exported_genesis.py | 2 ++ integration_tests/test_ibc.py | 2 ++ integration_tests/test_ibc_timeout.py | 2 ++ integration_tests/test_ibc_update_client.py | 2 ++ integration_tests/test_mempool.py | 2 ++ integration_tests/test_pruned_node.py | 2 ++ integration_tests/test_replay_block.py | 4 +++- integration_tests/test_upgrade.py | 2 ++ scripts/run-integration-tests | 12 ++++++++++-- 12 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2065b90206..d7eb3a15f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,11 @@ jobs: integration_tests: runs-on: ubuntu-latest timeout-minutes: 240 + strategy: + matrix: + tests: [unmarked, ibc, ibc_timeout, ibc_update_client, upgrade, slow] + env: + TESTS_TO_RUN: ${{ matrix.tests }} steps: - uses: actions/checkout@v3 with: diff --git a/Makefile b/Makefile index 35b3824ba7..b2467e668c 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,12 @@ test-sim-profile: ### Integration Test ### ############################################################################### +# possible values: +# - all: run all integration tests +# - unmarked: run integration tests that are not marked +# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers +TESTS_TO_RUN ?= all + run-integration-tests: @nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 980d4a9627..9b3f2ed053 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -11,8 +11,19 @@ def pytest_configure(config): + config.addinivalue_line("markers", "unmarked: fallback mark for unmarked tests") config.addinivalue_line("markers", "slow: marks tests as slow") config.addinivalue_line("markers", "gravity: gravity bridge test cases") + config.addinivalue_line("markers", "upgrade: marks upgrade tests") + config.addinivalue_line("markers", "ibc: marks default ibc tests") + config.addinivalue_line("markers", "ibc_timeout: marks ibc timeout tests") + config.addinivalue_line("markers", "ibc_update_client: marks ibc updateclient test") + + +def pytest_collection_modifyitems(items, config): + for item in items: + if not any(item.iter_markers()): + item.add_marker("unmarked") @pytest.fixture(scope="session") diff --git a/integration_tests/test_exported_genesis.py b/integration_tests/test_exported_genesis.py index 64ab52f626..6be07e59d2 100644 --- a/integration_tests/test_exported_genesis.py +++ b/integration_tests/test_exported_genesis.py @@ -6,6 +6,8 @@ from .network import setup_custom_cronos from .utils import ADDRS, CONTRACTS +pytestmark = pytest.mark.slow + @pytest.fixture(scope="module") def custom_cronos(tmp_path_factory): diff --git a/integration_tests/test_ibc.py b/integration_tests/test_ibc.py index 1affeb98af..7319faab07 100644 --- a/integration_tests/test_ibc.py +++ b/integration_tests/test_ibc.py @@ -19,6 +19,8 @@ wait_for_new_blocks, ) +pytestmark = pytest.mark.ibc + @pytest.fixture(scope="module", params=[True, False]) def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ibc_timeout.py b/integration_tests/test_ibc_timeout.py index 938a9e443e..ff33f7cb23 100644 --- a/integration_tests/test_ibc_timeout.py +++ b/integration_tests/test_ibc_timeout.py @@ -9,6 +9,8 @@ ) from .utils import ADDRS, eth_to_bech32, wait_for_fn +pytestmark = pytest.mark.ibc_timeout + @pytest.fixture(scope="module") def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ibc_update_client.py b/integration_tests/test_ibc_update_client.py index 25288f8d3a..e9133a6821 100644 --- a/integration_tests/test_ibc_update_client.py +++ b/integration_tests/test_ibc_update_client.py @@ -5,6 +5,8 @@ from .ibc_utils import prepare_network +pytestmark = pytest.mark.ibc_update_client + @pytest.fixture(scope="module") def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_mempool.py b/integration_tests/test_mempool.py index 65e25ee55b..a50a513081 100644 --- a/integration_tests/test_mempool.py +++ b/integration_tests/test_mempool.py @@ -14,6 +14,8 @@ wait_for_new_blocks, ) +pytestmark = pytest.mark.slow + @pytest.fixture(scope="module") def cronos_mempool(tmp_path_factory): diff --git a/integration_tests/test_pruned_node.py b/integration_tests/test_pruned_node.py index b5b682d21c..74092fe9fa 100644 --- a/integration_tests/test_pruned_node.py +++ b/integration_tests/test_pruned_node.py @@ -16,6 +16,8 @@ wait_for_new_blocks, ) +pytestmark = pytest.mark.slow + @pytest.fixture(scope="module") def cronos(request, tmp_path_factory): diff --git a/integration_tests/test_replay_block.py b/integration_tests/test_replay_block.py index 7a275ff695..722038fa7f 100644 --- a/integration_tests/test_replay_block.py +++ b/integration_tests/test_replay_block.py @@ -8,6 +8,8 @@ from .network import setup_custom_cronos from .utils import ADDRS, CONTRACTS, KEYS, deploy_contract, sign_transaction +pytestmark = pytest.mark.slow + @pytest.fixture(scope="module") def custom_cronos(tmp_path_factory): @@ -17,7 +19,7 @@ def custom_cronos(tmp_path_factory): ) -def test_replay_block(custom_cronos): +def test_block_overflow(custom_cronos): w3: web3.Web3 = custom_cronos.w3 contract = deploy_contract( w3, diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py index 84fec8a5fb..4f6f987dd4 100644 --- a/integration_tests/test_upgrade.py +++ b/integration_tests/test_upgrade.py @@ -22,6 +22,8 @@ wait_for_port, ) +pytestmark = pytest.mark.upgrade + def init_cosmovisor(home): """ diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests index 1609dd454b..c2278eb22b 100755 --- a/scripts/run-integration-tests +++ b/scripts/run-integration-tests @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e cd "$(dirname "$0")" @@ -10,4 +10,12 @@ cd ../integration_tests/contracts HUSKY_SKIP_INSTALL=1 npm install npm run typechain cd .. -nix-shell --run "pytest -vv -s" +TESTS_TO_RUN="${TESTS_TO_RUN:-all}" +if [[ "$TESTS_TO_RUN" == "all" ]]; then + echo "run all tests" + cmd="pytest -vv -s" +else + echo "run tests matching $TESTS_TO_RUN" + cmd="pytest -vv -s -m '$TESTS_TO_RUN'" +fi +nix-shell --run "$cmd"