Problem: can't get back tx result of false-failed tx (#438)

* Problem: can't get back tx result of false-failed tx

Closes: #437
Solution:
- Support fetch tx receipts by replaying block
- add integration test

* fix PR suggestions

app/app.go

@@ -453,6 +453,7 @@ func New(
 		app.TransferKeeper,
 		gravityKeeper,
 		app.EvmKeeper,
+		app.AccountKeeper,
 	)
 	cronosModule := cronos.NewAppModule(appCodec, app.CronosKeeper)
 

integration_tests/configs/low_block_gas_limit.yaml

@@ -0,0 +1,60 @@
+dotenv: ../../scripts/.env
+cronos_777-1:
+  cmd: cronosd
+  start-flags: "--trace"
+  app-config:
+    minimum-gas-prices: 0basetcro
+    json-rpc:
+      address: "0.0.0.0:{EVMRPC_PORT}"
+      ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
+      api: "eth,net,web3,debug,cronos"
+    evm:
+      max-tx-gas-wanted: 0
+  validators:
+    - coins: 1000000000000000000stake,10000000000000000000000basetcro
+      staked: 1000000000000000000stake
+      mnemonic: ${VALIDATOR1_MNEMONIC}
+    - coins: 1000000000000000000stake,10000000000000000000000basetcro
+      staked: 1000000000000000000stake
+      mnemonic: ${VALIDATOR2_MNEMONIC}
+  accounts:
+    - name: community
+      coins: 10000000000000000000000basetcro
+      mnemonic: ${COMMUNITY_MNEMONIC}
+    - name: signer1
+      coins: 20000000000000000000000basetcro
+      mnemonic: ${SIGNER1_MNEMONIC}
+    - name: signer2
+      coins: 30000000000000000000000basetcro
+      mnemonic: ${SIGNER2_MNEMONIC}
+
+  genesis:
+    consensus_params:
+      block:
+        max_bytes: "1048576"
+        max_gas: "815000"
+    app_state:
+      evm:
+        params:
+          evm_denom: basetcro
+      cronos:
+        params:
+          cronos_admin: ${CRONOS_ADMIN}
+          enable_auto_deployment: true
+          ibc_cro_denom: ${IBC_CRO_DENOM}
+      gov:
+        voting_params:
+          voting_period: "10s"
+        deposit_params:
+          max_deposit_period: "10s"
+          min_deposit:
+            - denom: "basetcro"
+              amount: "1"
+      transfer:
+        params:
+          receive_enabled: true
+          send_enabled: true
+      feemarket:
+        params:
+          no_base_fee: false
+          initial_base_fee: 100000000000

integration_tests/test_replay_block.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+
+import pytest
+import web3
+from web3._utils.method_formatters import receipt_formatter
+from web3.datastructures import AttributeDict
+
+from .network import setup_custom_cronos
+from .utils import ADDRS, CONTRACTS, KEYS, deploy_contract, sign_transaction
+
+
+@pytest.fixture(scope="module")
+def custom_cronos(tmp_path_factory):
+    path = tmp_path_factory.mktemp("cronos")
+    yield from setup_custom_cronos(
+        path, 26000, Path(__file__).parent / "configs/low_block_gas_limit.yaml"
+    )
+
+
+def test_replay_block(custom_cronos):
+    w3: web3.Web3 = custom_cronos.w3
+    contract = deploy_contract(
+        w3,
+        CONTRACTS["TestMessageCall"],
+        key=KEYS["community"],
+    )
+    iterations = 400
+    gas_limit = 800000
+    for i in range(10):
+        nonce = w3.eth.get_transaction_count(ADDRS["validator"])
+        txs = [
+            contract.functions.test(iterations).buildTransaction(
+                {
+                    "nonce": nonce,
+                    "gas": gas_limit,
+                }
+            ),
+            contract.functions.test(iterations).buildTransaction(
+                {
+                    "nonce": nonce + 1,
+                    "gas": gas_limit,
+                }
+            ),
+        ]
+        txhashes = [
+            w3.eth.send_raw_transaction(sign_transaction(w3, tx).rawTransaction)
+            for tx in txs
+        ]
+        receipt1 = w3.eth.wait_for_transaction_receipt(txhashes[0])
+        try:
+            receipt2 = w3.eth.wait_for_transaction_receipt(txhashes[1], timeout=10)
+        except web3.exceptions.TimeExhausted:
+            # expected exception, tx2 is included but failed.
+            receipt2 = None
+            break
+        if receipt1.blockNumber == receipt2.blockNumber:
+            break
+        print(
+            "tx1 and tx2 are included in two different blocks, retry now.",
+            receipt1.blockNumber,
+            receipt2.blockNumber,
+        )
+    else:
+        assert False, "timeout"
+    assert not receipt2
+    # check sender's nonce is increased twice, which means both txs are executed.
+    assert nonce + 2 == w3.eth.get_transaction_count(ADDRS["validator"])
+    rsp = w3.provider.make_request(
+        "cronos_replayBlock", [hex(receipt1.blockNumber), False]
+    )
+    assert "error" not in rsp, rsp["error"]
+    assert 2 == len(rsp["result"])
+
+    # check the replay receipts are the same
+    replay_receipts = [AttributeDict(receipt_formatter(item)) for item in rsp["result"]]
+    assert replay_receipts[0].gasUsed == replay_receipts[1].gasUsed == receipt1.gasUsed
+    assert replay_receipts[0].status == replay_receipts[1].status == receipt1.status
+    assert (
+        replay_receipts[0].logsBloom
+        == replay_receipts[1].logsBloom
+        == receipt1.logsBloom
+    )
+    assert replay_receipts[0].cumulativeGasUsed == receipt1.cumulativeGasUsed
+    assert replay_receipts[1].cumulativeGasUsed == receipt1.cumulativeGasUsed * 2
+
+    # check the postUpgrade mode
+    rsp = w3.provider.make_request(
+        "cronos_replayBlock", [hex(receipt1.blockNumber), True]
+    )
+    assert "error" not in rsp, rsp["error"]
+    assert 2 == len(rsp["result"])
+    replay_receipts = [AttributeDict(receipt_formatter(item)) for item in rsp["result"]]
+    assert replay_receipts[1].status == 0
+    assert replay_receipts[1].gasUsed == gas_limit

proto/cronos/query.proto

@@ -1,8 +1,11 @@
 syntax = "proto3";
 package cronos;
 
+import "gogoproto/gogo.proto";
 import "google/api/annotations.proto";
+import "google/protobuf/timestamp.proto";
 import "cosmos/base/query/v1beta1/pagination.proto";
+import "ethermint/evm/v1/tx.proto";
 // this line is used by starport scaffolding # 1
 
 option go_package = "github.com/crypto-org-chain/cronos/x/cronos/types";
@@ -19,11 +22,17 @@ service Query {
         option (google.api.http).get = "/cronos/v1/denom_by_contract/{contract}";
     }
 
+    // ReplayBlock replay the eth messages in the block to recover the results of false-failed txs.
+    rpc ReplayBlock(ReplayBlockRequest) returns (ReplayBlockResponse) { }
+
     // this line is used by starport scaffolding # 2
 }
 
 // ContractByDenomRequest is the request type of ContractByDenom call
 message ContractByDenomRequest {
+    option (gogoproto.equal) = false;
+    option (gogoproto.goproto_getters) = false;
+
     string denom = 1;
 }
 
@@ -35,6 +44,9 @@ message ContractByDenomResponse {
 
 // DenomByContractRequest is the request type of DenomByContract call
 message DenomByContractRequest {
+    option (gogoproto.equal) = false;
+    option (gogoproto.goproto_getters) = false;
+
     string contract = 1;
 }
 
@@ -43,4 +55,21 @@ message DenomByContractResponse {
     string denom = 1;
 }
 
+// ReplayBlockRequest
+message ReplayBlockRequest {
+    option (gogoproto.equal) = false;
+    option (gogoproto.goproto_getters) = false;
+
+    // the eth messages in the block
+    repeated ethermint.evm.v1.MsgEthereumTx msgs = 1;
+    int64 block_number = 2;
+    string block_hash = 3;
+    google.protobuf.Timestamp block_time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true];
+}
+
+// ReplayBlockResponse 
+message ReplayBlockResponse {
+  repeated ethermint.evm.v1.MsgEthereumTxResponse responses = 1;
+}
+
 // this line is used by starport scaffolding # 3

scripts/protoc-swagger-gen.sh

@@ -32,4 +32,4 @@ install_statik() {
 install_statik
 
 # generate binary for static server
-statik -src=./client/docs/swagger-ui -dest=./client/docs -f
+statik -src=./client/docs/swagger-ui -dest=./client/docs -f -ns cronos

x/cronos/keeper/evm_hooks_test.go

@@ -194,6 +194,7 @@ func (suite *KeeperTestSuite) TestEvmHooks() {
 					keepertest.IbcKeeperMock{},
 					suite.app.GravityKeeper,
 					suite.app.EvmKeeper,
+					suite.app.AccountKeeper,
 				)
 				suite.app.CronosKeeper = cronosKeeper
 

x/cronos/keeper/evm_log_handlers_test.go

@@ -289,6 +289,7 @@ func (suite *KeeperTestSuite) TestSendToIbcHandler() {
 				keepertest.IbcKeeperMock{},
 				suite.app.GravityKeeper,
 				suite.app.EvmKeeper,
+				suite.app.AccountKeeper,
 			)
 			handler := keeper.NewSendToIbcHandler(suite.app.BankKeeper, cronosKeeper)
 			tc.malleate()
@@ -378,6 +379,7 @@ func (suite *KeeperTestSuite) TestSendCroToIbcHandler() {
 				keepertest.IbcKeeperMock{},
 				suite.app.GravityKeeper,
 				suite.app.EvmKeeper,
+				suite.app.AccountKeeper,
 			)
 			handler := keeper.NewSendCroToIbcHandler(suite.app.BankKeeper, cronosKeeper)
 			tc.malleate()

x/cronos/keeper/grpc_query.go

@@ -3,10 +3,12 @@ package keeper
 import (
 	"context"
 	"fmt"
+	"math/big"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/crypto-org-chain/cronos/x/cronos/types"
 	"github.com/ethereum/go-ethereum/common"
+	evmtypes "github.com/tharsis/ethermint/x/evm/types"
 )
 
 var _ types.QueryServer = Keeper{}
@@ -40,3 +42,71 @@ func (k Keeper) DenomByContract(goCtx context.Context, req *types.DenomByContrac
 		Denom: denom,
 	}, nil
 }
+
+// ReplayBlock replay the eth messages in the block to recover the results of false-failed txs.
+func (k Keeper) ReplayBlock(goCtx context.Context, req *types.ReplayBlockRequest) (*types.ReplayBlockResponse, error) {
+	rsps := make([]*evmtypes.MsgEthereumTxResponse, 0, len(req.Msgs))
+
+	// prepare the block context, the multistore version should be setup already in grpc query context.
+	ctx := sdk.UnwrapSDKContext(goCtx).
+		WithBlockHeight(req.BlockNumber).
+		WithBlockTime(req.BlockTime).
+		WithHeaderHash(common.Hex2Bytes(req.BlockHash))
+
+	// load parameters
+	params := k.evmKeeper.GetParams(ctx)
+	chainID := k.evmKeeper.ChainID()
+	// the chain_id is irrelevant here
+	ethCfg := params.ChainConfig.EthereumConfig(chainID)
+
+	blockHeight := big.NewInt(req.BlockNumber)
+	homestead := ethCfg.IsHomestead(blockHeight)
+	istanbul := ethCfg.IsIstanbul(blockHeight)
+	london := ethCfg.IsLondon(blockHeight)
+	evmDenom := params.EvmDenom
+
+	// we assume the message executions are successful, they are filtered in json-rpc api
+	for _, msg := range req.Msgs {
+		// deduct fee
+		txData, err := evmtypes.UnpackTxData(msg.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		// populate the `From` field
+		if _, err := msg.GetSender(chainID); err != nil {
+			return nil, err
+		}
+
+		if _, err := k.evmKeeper.DeductTxCostsFromUserBalance(
+			ctx,
+			*msg,
+			txData,
+			evmDenom,
+			homestead,
+			istanbul,
+			london,
+		); err != nil {
+			return nil, err
+		}
+
+		// increase nonce
+		acc := k.accountKeeper.GetAccount(ctx, msg.GetFrom())
+		if acc == nil {
+			return nil, fmt.Errorf("account not found %s", msg.From)
+		}
+		if err := acc.SetSequence(acc.GetSequence() + 1); err != nil {
+			return nil, err
+		}
+		k.accountKeeper.SetAccount(ctx, acc)
+
+		rsp, err := k.evmKeeper.EthereumTx(sdk.WrapSDKContext(ctx), msg)
+		if err != nil {
+			return nil, err
+		}
+		rsps = append(rsps, rsp)
+	}
+	return &types.ReplayBlockResponse{
+		Responses: rsps,
+	}, nil
+}

x/cronos/keeper/ibc_hooks_test.go

@@ -77,6 +77,7 @@ func (suite *KeeperTestSuite) TestOnRecvVouchers() {
 				keepertest.IbcKeeperMock{},
 				suite.app.GravityKeeper,
 				suite.app.EvmKeeper,
+				suite.app.AccountKeeper,
 			)
 			suite.app.CronosKeeper = cronosKeeper
 

x/cronos/keeper/ibc_test.go

@@ -264,6 +264,7 @@ func (suite *KeeperTestSuite) TestIbcTransferCoins() {
 				keepertest.IbcKeeperMock{},
 				suite.app.GravityKeeper,
 				suite.app.EvmKeeper,
+				suite.app.AccountKeeper,
 			)
 			suite.app.CronosKeeper = cronosKeeper
 

x/cronos/keeper/keeper.go

@@ -31,6 +31,8 @@ type (
 		gravityKeeper types.GravityKeeper
 		// ethermint evm keeper
 		evmKeeper types.EvmKeeper
+		// account keeper
+		accountKeeper types.AccountKeeper
 
 		// this line is used by starport scaffolding # ibc/keeper/attribute
 	}
@@ -45,6 +47,7 @@ func NewKeeper(
 	transferKeeper types.TransferKeeper,
 	gravityKeeper types.GravityKeeper,
 	evmKeeper types.EvmKeeper,
+	accountKeeper types.AccountKeeper,
 	// this line is used by starport scaffolding # ibc/keeper/parameter
 ) *Keeper {
 
@@ -62,6 +65,7 @@ func NewKeeper(
 		transferKeeper: transferKeeper,
 		gravityKeeper:  gravityKeeper,
 		evmKeeper:      evmKeeper,
+		accountKeeper:  accountKeeper,
 		// this line is used by starport scaffolding # ibc/keeper/return
 	}
 }

x/cronos/keeper/params_test.go

@@ -46,6 +46,7 @@ func (suite *KeeperTestSuite) TestGetSourceChannelID() {
 				keepertest.IbcKeeperMock{},
 				suite.app.GravityKeeper,
 				suite.app.EvmKeeper,
+				suite.app.AccountKeeper,
 			)
 			suite.app.CronosKeeper = cronosKeeper