Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unable to run node services, unable to sync #176

Open
YuXiaoCoder opened this issue Oct 31, 2024 · 1 comment
Open

Unable to run node services, unable to sync #176

YuXiaoCoder opened this issue Oct 31, 2024 · 1 comment

Comments

@YuXiaoCoder
Copy link

YuXiaoCoder commented Oct 31, 2024

command:

/opt/auroramain/core/refiner -c /mnt/auroramain/conf/refiner/refiner.json run --height=131480846

refiner version: aurora-refiner 0.28.2-2.2.1

config.json

{
    "genesis_file": "/mnt/auroramain/node/near/genesis.json",
    "genesis_records_file": null,
    "node_key_file": "/mnt/auroramain/node/near/node_key.json",
    "rpc": {
        "addr": "0.0.0.0:3030",
        "prometheus_addr": null,
        "cors_allowed_origins": [
            "*"
        ],
        "polling_config": {
            "polling_interval": {
                "secs": 0,
                "nanos": 500000000
            },
            "polling_timeout": {
                "secs": 10,
                "nanos": 0
            }
        },
        "limits_config": {
            "json_payload_max_size": 10485760
        },
        "enable_debug_rpc": false,
        "experimental_debug_pages_src_path": null
    },
    "network": {
        "addr": "0.0.0.0:24567",
        "boot_nodes": "ed25519:[email protected]:24567,ed25519:[email protected]:24567,ed25519:[email protected]:24567,ed25519:[email protected]:24567,ed25519:[email protected]:24567",
        "whitelist_nodes": "",
        "max_num_peers": 40,
        "minimum_outbound_peers": 5,
        "ideal_connections_lo": 30,
        "ideal_connections_hi": 35,
        "so_recv_buffer_size": 1000000,
        "so_send_buffer_size": 1000000,
        "peer_recent_time_window": {
            "secs": 600,
            "nanos": 0
        },
        "safe_set_size": 20,
        "archival_peer_connections_lower_bound": 10,
        "handshake_timeout": {
            "secs": 20,
            "nanos": 0
        },
        "skip_sync_wait": false,
        "ban_window": {
            "secs": 10800,
            "nanos": 0
        },
        "blacklist": [],
        "ttl_account_id_router": {
            "secs": 3600,
            "nanos": 0
        },
        "peer_stats_period": {
            "secs": 5,
            "nanos": 0
        },
        "monitor_peers_max_period": {
            "secs": 60,
            "nanos": 0
        },
        "peer_states_cache_size": 1000,
        "snapshot_hosts_cache_size": 1000,
        "peer_expiration_duration": {
            "secs": 604800,
            "nanos": 0
        },
        "public_addrs": [],
        "allow_private_ip_in_public_addrs": false,
        "trusted_stun_servers": [
            "stun.l.google.com:19302",
            "stun1.l.google.com:19302",
            "stun2.l.google.com:19302",
            "stun3.l.google.com:19302",
            "stun4.l.google.com:19302"
        ],
        "experimental": {
            "inbound_disabled": false,
            "connect_only_to_boot_nodes": false,
            "skip_sending_tombstones_seconds": 0,
            "tier1_enable_inbound": true,
            "tier1_enable_outbound": true,
            "tier1_connect_interval": {
                "secs": 60,
                "nanos": 0
            },
            "tier1_new_connections_per_attempt": 50,
            "network_config_overrides": {
              "connect_to_reliable_peers_on_startup": null,
              "max_send_peers": null,
              "routed_message_ttl": null,
              "max_routes_to_store": null,
              "highest_peer_horizon": null,
              "push_info_period_millis": null,
              "outbound_disabled": null,
              "accounts_data_broadcast_rate_limit_burst": null,
              "accounts_data_broadcast_rate_limit_qps": null,
              "routing_table_update_rate_limit_burst": null,
              "routing_table_update_rate_limit_qps": null,
              "received_messages_rate_limits": null
            }
        }
    },
    "consensus": {
        "min_num_peers": 3,
        "block_production_tracking_delay": {
            "secs": 0,
            "nanos": 100000000
        },
        "min_block_production_delay": {
            "secs": 1,
            "nanos": 300000000
        },
        "max_block_production_delay": {
            "secs": 3,
            "nanos": 0
        },
        "max_block_wait_delay": {
            "secs": 6,
            "nanos": 0
        },
        "produce_empty_blocks": true,
        "block_fetch_horizon": 50,
        "block_header_fetch_horizon": 50,
        "catchup_step_period": {
            "secs": 0,
            "nanos": 100000000
        },
        "chunk_request_retry_period": {
            "secs": 0,
            "nanos": 400000000
        },
        "header_sync_initial_timeout": {
            "secs": 10,
            "nanos": 0
        },
        "header_sync_progress_timeout": {
            "secs": 2,
            "nanos": 0
        },
        "header_sync_stall_ban_timeout": {
            "secs": 120,
            "nanos": 0
        },
        "state_sync_timeout": {
            "secs": 60,
            "nanos": 0
        },
        "header_sync_expected_height_per_second": 10,
        "sync_check_period": {
            "secs": 10,
            "nanos": 0
        },
        "sync_step_period": {
            "secs": 0,
            "nanos": 10000000
        },
        "doomslug_step_period": {
            "secs": 0,
            "nanos": 100000000
        },
        "sync_height_threshold": 1,
        "sync_max_block_requests": 10
    },
    "tracked_accounts": [],
    "tracked_shadow_validator": null,
    "tracked_shards": [
        0
      ],
    "archive": false,
    "log_summary_style": "colored",
    "log_summary_period": {
        "secs": 10,
        "nanos": 0
    },
    "enable_multiline_logging": false,
    "gc_blocks_limit": 2,
    "gc_fork_clean_step": 100,
    "gc_num_epochs_to_keep": 5,
    "gc_step_period": {
        "secs": 1,
        "nanos": 0
      },
    "view_client_threads": 4,
    "epoch_sync_enabled": true,
    "view_client_throttle_period": {
        "secs": 30,
        "nanos": 0
    },
    "trie_viewer_state_size_limit": 50000,
    "store": {
        "path": null,
        "enable_statistics": false,
        "enable_statistics_export": true,
        "max_open_files": 10000,
        "col_state_cache_size": 3221225472,
        "col_flat_state_cache_size": 134217728,
        "block_size": 16384,
        "trie_cache": {
            "default_max_bytes": 500000000,
            "per_shard_max_bytes": {
              "s3.v3": 1500000000,
              "s5.v3": 3000000000,
              "s1.v2": 50000000,
              "s1.v1": 50000000,
              "s3.v1": 3000000000,
              "s4.v2": 3000000000,
              "s1.v3": 50000000,
              "s2.v2": 3000000000,
              "s2.v3": 1500000000
            },
            "shard_cache_deletions_queue_capacity": 100000
          },
        "view_trie_cache": {
            "default_max_bytes": 50000000,
            "per_shard_max_bytes": {},
            "shard_cache_deletions_queue_capacity": 100000
        },
        "enable_receipt_prefetching": true,
        "sweat_prefetch_receivers": [
            "token.sweat",
            "vfinal.token.sweat.testnet"
        ],
        "sweat_prefetch_senders": [
            "oracle.sweat",
            "sweat_the_oracle.testnet"
        ],
        "claim_sweat_prefetch_config": [
            {
              "receiver": "claim.sweat",
              "sender": "token.sweat",
              "method_name": "record_batch_for_hold"
            },
            {
              "receiver": "claim.sweat",
              "sender": "",
              "method_name": "claim"
            }
        ],
        "kaiching_prefetch_config": [
          {
            "receiver": "earn.kaiching",
            "sender": "wallet.kaiching",
            "method_name": "ft_on_transfer"
          }
        ],
        "load_mem_tries_for_shards": [],
        "load_mem_tries_for_tracked_shards": false,
        "state_snapshot_config": {
          "state_snapshot_type": "ForReshardingOnly"
        },
        "state_snapshot_enabled": false
      },
      "state_sync_enabled": true,
      "state_sync": {
        "sync": {
          "ExternalStorage": {
            "location": {
              "GCS": {
                "bucket": "state-parts"
              }
            },
            "num_concurrent_requests": 25,
            "num_concurrent_requests_during_catchup": 5,
            "external_storage_fallback_threshold": 5
          }
        }
      },
      "epoch_sync": {
        "enabled": false,
        "epoch_sync_horizon": 216000,
        "epoch_sync_accept_proof_max_horizon": 86400,
        "timeout_for_epoch_sync": {
          "secs": 60,
          "nanos": 0
        }
      },
      "transaction_pool_size_limit": 100000000,
      "resharding_config": {
        "batch_size": 500000,
        "batch_delay": {
          "secs": 0,
          "nanos": 100000000
        },
        "retry_delay": {
          "secs": 10,
          "nanos": 0
        },
        "initial_delay": {
          "secs": 0,
          "nanos": 0
        },
        "max_poll_time": {
          "secs": 7200,
          "nanos": 0
        }
      },
      "tx_routing_height_horizon": 4,
      "produce_chunk_add_transactions_time_limit": null,
      "orphan_state_witness_pool_size": 25,
      "orphan_state_witness_max_size": 40000000,
      "max_loaded_contracts": 256,
      "save_latest_witnesses": false
}

The error log is as follows:

2024-10-31T01:43:24.830032Z  INFO indexer: Load config from /mnt/auroramain/node/near...
2024-10-31T01:43:24.843590Z  WARN neard: /mnt/auroramain/node/near/config.json: encountered unrecognised field: state_sync.?.sync.?.external_storage_fallback_threshold
2024-10-31T01:43:24.843607Z  INFO config: Validating Config, extracted from config.json...
2024-10-31T01:43:24.849762Z  WARN genesis: Skipped genesis validation
2024-10-31T01:43:24.849812Z  INFO config: Validating Genesis config and records. This could take a few minutes...
2024-10-31T01:43:24.850006Z  INFO config: All validations have passed!
2024-10-31T01:43:24.858760Z  INFO db_opener: Opening NodeStorage path="/mnt/auroramain/node/near/data" cold_path="none"
2024-10-31T01:43:24.859324Z  INFO db: Opened a new RocksDB instance. num_instances=1
2024-10-31T01:43:24.915339Z  INFO db: Closed a RocksDB instance. num_instances=0
2024-10-31T01:43:24.915360Z  INFO db_opener: The database exists. path=/mnt/auroramain/node/near/data
2024-10-31T01:43:24.915369Z  INFO db: Opened a new RocksDB instance. num_instances=1
2024-10-31T01:43:25.376571Z  INFO db: Closed a RocksDB instance. num_instances=0
2024-10-31T01:43:25.376614Z  INFO db: Opened a new RocksDB instance. num_instances=1
2024-10-31T01:43:25.407669Z  INFO db: Closed a RocksDB instance. num_instances=0
2024-10-31T01:43:25.407689Z  INFO db: Opened a new RocksDB instance. num_instances=1
2024-10-31T01:43:25.649448Z  INFO runtime: Error when getting the gc stop height. This error may naturally occur after the gc_num_epochs_to_keep config is increased. It should disappear as soon as the node builds up all epochs it wants. Error: DB Not Found Error: epoch block: HRKbCC2Dt4tX7fHApoNnDkAChFYE5kqzWULkejRZZCyg
2024-10-31T01:43:25.664069Z  INFO memtrie: Loading tries to memory for shards []...
2024-10-31T01:43:25.665699Z  INFO memtrie: Memtries loading complete for shards []
2024-10-31T01:43:25.665711Z  INFO chain: Init: header head @ #131590669 7TEa7MedJ7o5vEirfPAzcU5bzCVhpnhwea2gob4fgtyq; block head @ #131432894 HRKbCC2Dt4tX7fHApoNnDkAChFYE5kqzWULkejRZZCyg
thread 'main' panicked at /usr/local/cargo/git/checkouts/nearcore-5bf7818cf2261fd0/971e76d/chain/client/src/client_actor.rs:168:6:
called `Result::unwrap()` on an `Err` value: Chain(DBNotFoundErr("epoch block: HRKbCC2Dt4tX7fHApoNnDkAChFYE5kqzWULkejRZZCyg"))
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
thread 'thread 'actix-rt|system:0|arbiter:1actix-rt|system:0|arbiter:0' panicked at ' panicked at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/time/entry.rs/usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/time/entry.rs::568568::99:
:
A Tokio 1.x context was found, but it is being shutdown.A Tokio 1.x context was found, but it is being shutdown.
@YuXiaoCoder
Copy link
Author

Using the same snapshot, the node version was upgraded to 0.28.2-2.3.0-rc.4, and it still failed to synchronise the blocks, with the following logs

2024-10-31T03:22:22.623075Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=176962720 total_msg_received_count=4232
2024-10-31T03:22:24.025494Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:23:22.623602Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=180610320 total_msg_received_count=4217
2024-10-31T03:23:38.850228Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:24:06.567739Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:24:22.624763Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=197019756 total_msg_received_count=4456
2024-10-31T03:24:24.026959Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:24:56.312199Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:25:22.625841Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=213825630 total_msg_received_count=4500
2024-10-31T03:25:22.931576Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:25:32.931503Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:25:42.920868Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:25:52.916033Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:26:02.950219Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:26:12.926831Z  INFO network: Failed to connect to ed25519:[email protected]:24567 err="PeerActor::spawn(): outbound not allowed: already connected to this peer"
2024-10-31T03:26:22.626399Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=249099237 total_msg_received_count=5924
2024-10-31T03:27:22.628350Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=227431702 total_msg_received_count=5631
2024-10-31T03:27:32.473537Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:27:37.743297Z  INFO network: Closing connection to Some(ed25519:[email protected]:24567) err=Recv(IO(Kind(UnexpectedEof)))
2024-10-31T03:27:42.467066Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:27:52.467338Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:02.467050Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:12.467973Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:22.468042Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:22.636529Z  INFO bandwidth: Bandwidth stats total_bandwidth_used_by_all_peers=222080917 total_msg_received_count=5531
2024-10-31T03:28:32.467711Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:42.467210Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:28:52.467504Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.
2024-10-31T03:29:02.467299Z ERROR metrics: Error when exporting postponed receipts count DB Not Found Error: BLOCK: H8wEeuPiUNektcpJ4NDhB38Aarq6oVzcPBtb4Ab2h6ZS.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant