diff --git a/CHANGELOG.md b/CHANGELOG.md index 838b59f0..5f13993a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## Version 0.57.0 + +- Shadow SMFT is prepared for first deployment + ## Version 0.56.8 - `./configs/ton-global.config.json` renamed to `./configs/ton-global-config-sample.json` because diff --git a/Cargo.toml b/Cargo.toml index a8716526..f1ffcff8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ build = 'common/build/build.rs' edition = '2021' name = 'ton_node' -version = '0.56.8' +version = '0.57.0' [workspace] members = [ 'storage' ] @@ -54,6 +54,7 @@ colored = '1.9.3' crossbeam-channel = '0.4.2' ctrlc = { features = [ 'termination' ], version = '3.4.0' } dashmap = '5.4.0' +deflate = '1.0.0' dirs = '2.0.2' enum-as-inner = '=0.5.1' env_logger = '0.7.1' @@ -61,6 +62,7 @@ failure = '0.1' futures = '0.3.1' futures-timer = '3.0.1' hex = '0.4' +inflate = '0.4.5' lazy_static = '1.4.0' log = '0.4' log4rs = '1.2' @@ -88,7 +90,7 @@ adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/to catchain = { path = 'catchain' } lockfree = { git = 'https://github.com/tonlabs/lockfree.git' } storage = { path = 'storage' } -ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.4.25' } +ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', tag = '2.4.25' } ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.72' } ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.141' } ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.231' } @@ -106,7 +108,7 @@ pretty_assertions = '1.3' tokio = { features = [ 'macros' ], version = '1.5' } [features] -default = [ 'telemetry' ] +default = [ 'telemetry', 'ton_types/export_key' ] export_key = [ 'catchain/export_key', 'ton_types/export_key' ] external_db = [ 'rdkafka' ] fast_finality_extra = [ ] @@ -114,11 +116,12 @@ gosh = [ 'ton_block/gosh', 'ton_vm/gosh' ] log_metrics = [ ] prometheus = [ 'metrics-exporter-prometheus', 'log_metrics' ] signature_with_id = [ 'ton_block/signature_with_id', 'ton_vm/signature_with_id', 'ton_executor/signature_with_id' ] -slashing = [ 'ton_abi', 'validator_session/slashing' ] +slashing = [ 'validator_session/slashing' ] statsd = [ 'metrics-exporter-statsd', 'log_metrics', 'dep:statsd' ] telemetry = [ 'adnl/telemetry', 'storage/telemetry' ] trace_alloc = [ ] trace_alloc_detail = [ 'trace_alloc' ] +verification = [ 'ton_types/export_key' ] [profile] diff --git a/bin/Elector.abi.json b/bin/Elector.abi.json new file mode 100644 index 00000000..811aa7e4 --- /dev/null +++ b/bin/Elector.abi.json @@ -0,0 +1,224 @@ +{ + "ABI version": 2, + "version": "2.3", + "header": ["time"], + "functions": [ + { + "name": "constructor", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "process_new_stake", + "id": "0x4E73744B", + "inputs": [ + {"name":"query_id","type":"uint64"}, + {"name":"validator_pubkey","type":"uint256"}, + {"name":"stake_at","type":"uint32"}, + {"name":"max_factor","type":"uint32"}, + {"name":"adnl_addr","type":"uint256"}, + {"name":"bls_key1","type":"uint256"}, + {"name":"bls_key2","type":"uint128"}, + {"name":"signature","type":"bytes"} + ], + "outputs": [ + ] + }, + { + "name": "config_set_confirmed_ok", + "id": "0xEE764F4B", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "config_set_confirmed_err", + "id": "0xEE764F6F", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "config_slash_confirmed_ok", + "id": "0xEE764F4C", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "config_slash_confirmed_err", + "id": "0xEE764F70", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "grant", + "id": "0x4772616E", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "take_change", + "inputs": [ + ], + "outputs": [ + ] + }, + { + "name": "recover_stake", + "id": "0x47657424", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "recover_stake_gracefully", + "id": "0x47657425", + "inputs": [ + {"name":"query_id","type":"uint64"}, + {"name":"elect_id","type":"uint32"} + ], + "outputs": [ + ] + }, + { + "name": "get_elect_at", + "id": "0x47657426", + "inputs": [ + {"name":"query_id","type":"uint64"} + ], + "outputs": [ + ] + }, + { + "name": "upgrade_code", + "id": "0x4E436F64", + "inputs": [ + {"name":"query_id","type":"uint64"}, + {"name":"code","type":"cell"}, + {"name":"data","type":"cell"} + ], + "outputs": [ + ] + }, + { + "name": "proceed_register_vote", + "id": "0x56744370", + "inputs": [ + {"name":"query_id","type":"uint64"}, + {"name":"signature_hi","type":"uint256"}, + {"name":"signature_lo","type":"uint256"}, + {"name":"sign_tag","type":"uint32"}, + {"name":"idx","type":"uint16"}, + {"name":"elect_id","type":"uint32"}, + {"name":"chash","type":"uint256"} + ], + "outputs": [ + ] + }, + { + "name": "report", + "inputs": [ + {"name":"signature_hi","type":"uint256"}, + {"name":"signature_lo","type":"uint256"}, + {"name":"reporter_pubkey","type":"uint256"}, + {"name":"victim_pubkey","type":"uint256"}, + {"name":"metric_id","type":"uint8"} + ], + "outputs": [ + ] + }, + { + "name": "get", + "inputs": [ + ], + "outputs": [ + {"name":"election_open","type":"bool"}, + {"components":[{"name":"elect_at","type":"uint32"},{"name":"elect_close","type":"uint32"},{"name":"min_stake","type":"varuint16"},{"name":"total_stake","type":"varuint16"},{"components":[{"name":"stake","type":"varuint16"},{"name":"time","type":"uint32"},{"name":"max_factor","type":"uint32"},{"name":"addr","type":"uint256"},{"name":"adnl_addr","type":"uint256"},{"name":"bls_key1","type":"uint256"},{"name":"bls_key2","type":"uint128"}],"name":"members","type":"map(uint256,tuple)"},{"name":"failed","type":"bool"},{"name":"finished","type":"bool"}],"name":"cur_elect","type":"tuple"}, + {"name":"credits","type":"map(uint256,varuint16)"}, + {"components":[{"name":"unfreeze_at","type":"uint32"},{"name":"stake_held","type":"uint32"},{"name":"vset_hash","type":"uint256"},{"components":[{"name":"addr","type":"uint256"},{"name":"weight","type":"uint64"},{"name":"stake","type":"varuint16"},{"name":"banned","type":"bool"}],"name":"frozen_dict","type":"map(uint256,tuple)"},{"name":"total_stake","type":"varuint16"},{"name":"bonuses","type":"varuint16"},{"components":[{"name":"tag","type":"uint8"},{"name":"complaint","type":"cell"},{"name":"voters","type":"map(uint16,uint32)"},{"name":"vset_id","type":"uint256"},{"name":"weight_remaining","type":"int64"}],"name":"complaints","type":"map(uint256,tuple)"}],"name":"past_elections","type":"map(uint32,tuple)"}, + {"name":"grams","type":"uint128"}, + {"name":"active_id","type":"uint32"}, + {"name":"active_hash","type":"uint256"} + ] + }, + { + "name": "get_banned", + "inputs": [ + ], + "outputs": [ + {"name":"value0","type":"map(uint256,bool)"} + ] + }, + { + "name": "get_buckets", + "inputs": [ + {"name":"victim_pubkey","type":"uint256"} + ], + "outputs": [ + {"components":[{"name":"weight","type":"uint64"},{"name":"reports","type":"map(uint256,uint64)"}],"name":"value0","type":"map(uint8,tuple)"} + ] + }, + { + "name": "get_buckets_workchain", + "inputs": [ + {"name":"victim_pubkey","type":"uint256"} + ], + "outputs": [ + {"components":[{"name":"weight","type":"uint64"},{"name":"reports","type":"map(uint256,uint64)"}],"name":"value0","type":"map(uint8,tuple)"} + ] + }, + { + "name": "active_election_id", + "inputs": [ + ], + "outputs": [ + {"name":"value0","type":"uint32"} + ] + }, + { + "name": "compute_returned_stake", + "inputs": [ + {"name":"wallet_addr","type":"uint256"} + ], + "outputs": [ + {"name":"value0","type":"uint256"} + ] + } + ], + "data": [ + ], + "events": [ + ], + "fields": [ + {"name":"_pubkey","type":"uint256"}, + {"name":"_constructorFlag","type":"bool"}, + {"name":"m_election_open","type":"bool"}, + {"components":[{"name":"elect_at","type":"uint32"},{"name":"elect_close","type":"uint32"},{"name":"min_stake","type":"varuint16"},{"name":"total_stake","type":"varuint16"},{"components":[{"name":"stake","type":"varuint16"},{"name":"time","type":"uint32"},{"name":"max_factor","type":"uint32"},{"name":"addr","type":"uint256"},{"name":"adnl_addr","type":"uint256"},{"name":"bls_key1","type":"uint256"},{"name":"bls_key2","type":"uint128"}],"name":"members","type":"map(uint256,tuple)"},{"name":"failed","type":"bool"},{"name":"finished","type":"bool"}],"name":"m_cur_elect","type":"tuple"}, + {"name":"m_credits","type":"map(uint256,varuint16)"}, + {"components":[{"name":"unfreeze_at","type":"uint32"},{"name":"stake_held","type":"uint32"},{"name":"vset_hash","type":"uint256"},{"components":[{"name":"addr","type":"uint256"},{"name":"weight","type":"uint64"},{"name":"stake","type":"varuint16"},{"name":"banned","type":"bool"}],"name":"frozen_dict","type":"map(uint256,tuple)"},{"name":"total_stake","type":"varuint16"},{"name":"bonuses","type":"varuint16"},{"components":[{"name":"tag","type":"uint8"},{"name":"complaint","type":"cell"},{"name":"voters","type":"map(uint16,uint32)"},{"name":"vset_id","type":"uint256"},{"name":"weight_remaining","type":"int64"}],"name":"complaints","type":"map(uint256,tuple)"}],"name":"m_past_elections","type":"map(uint32,tuple)"}, + {"name":"m_grams","type":"varuint16"}, + {"name":"m_active_id","type":"uint32"}, + {"name":"m_active_hash","type":"uint256"}, + {"name":"m_masterchain_vtors_weight","type":"uint64"}, + {"name":"m_workchain_vtors_weight","type":"uint64"}, + {"name":"m_banned","type":"map(uint256,bool)"}, + {"components":[{"name":"weight","type":"uint64"},{"name":"reports","type":"map(uint256,uint64)"}],"name":"m_reports","type":"map(uint256,map(uint8,tuple))"}, + {"components":[{"name":"weight","type":"uint64"},{"name":"reports","type":"map(uint256,uint64)"}],"name":"m_reports_workchain","type":"map(uint256,map(uint8,tuple))"} + ] +} diff --git a/bin/console.rs b/bin/console.rs index 211a2fb4..756de54b 100644 --- a/bin/console.rs +++ b/bin/console.rs @@ -14,7 +14,8 @@ use adnl::{ common::TaggedTlObject, client::{AdnlClient, AdnlClientConfig, AdnlClientConfigJson} }; -use std::{convert::TryInto, env, str::FromStr, time::Duration}; +use std::{collections::HashMap, convert::TryInto, env, str::FromStr, time::Duration}; +use ton_abi::{Contract, Token, TokenValue, Uint}; use ton_api::{ serialize_boxed, ton::{ @@ -35,6 +36,10 @@ use ton_types::{ include!("../common/src/test.rs"); +const ELECTOR_ABI: &[u8] = include_bytes!("Elector.abi.json"); //elector's ABI +const ELECTOR_PROCESS_NEW_STAKE_FUNC_NAME: &str = "process_new_stake"; //elector process_new_stake function name +const USE_FIFTH_ELECTOR: bool = true; + trait SendReceive { fn send(params: impl Iterator) -> Result; fn receive( @@ -90,6 +95,7 @@ commands! { AddValidatorAdnlAddr, "addvalidatoraddr", "addvalidatoraddr \tadd validator ADNL addr" AddValidatorPermKey, "addpermkey", "addpermkey \tadd validator permanent key" AddValidatorTempKey, "addtempkey", "addtempkey \tadd validator temp key" + AddValidatorBlsKey, "addblskey", "addblskey \t add validator bls key" Bundle, "bundle", "bundle \tprepare bundle" ExportPub, "exportpub", "exportpub \texports public key by key hash" FutureBundle, "future_bundle", "future_bundle \tprepare future bundle" @@ -225,8 +231,18 @@ impl SendReceive for GetSessionStats { } impl SendReceive for NewKeypair { - fn send(_params: impl Iterator) -> Result { - Ok(TLObject::new(ton::rpc::engine::validator::GenerateKeyPair)) + fn send(mut params: impl Iterator) -> Result { + match params.next() { + None => Ok(TLObject::new(ton::rpc::engine::validator::GenerateKeyPair)), + Some(param) => { + let mut param = param.to_string(); + param.make_ascii_lowercase(); + match param.as_ref() { + "bls" => Ok(TLObject::new(ton::rpc::engine::validator::GenerateBlsKeyPair)), + _ => fail!("invalid parameters!") + } + }, + } } fn receive( answer: TLObject, @@ -254,11 +270,14 @@ impl SendReceive for ExportPub { mut _params: impl Iterator ) -> Result<(String, Vec)> { let answer = downcast::(answer)?; - let pub_key = answer - .key() - .ok_or_else(|| error!("Public key not found in answer!"))? - .as_slice() - .to_vec(); + let pub_key = match answer.key() { + Some(key) => key.clone().into_vec(), + None => { + answer.bls_key() + .ok_or_else(|| error!("Public key not found in answer!"))? + .clone() + } + }; let msg = format!("imported key: {} {}", hex::encode(&pub_key), base64_encode(&pub_key)); Ok((msg, pub_key)) } @@ -300,6 +319,19 @@ impl SendReceive for AddValidatorPermKey { } } +impl SendReceive for AddValidatorBlsKey { + fn send(mut params: impl Iterator) -> Result { + let permanent_key_hash = parse_int256(params.next(), "permanent_key_hash")?; + let key_hash = parse_int256(params.next(), "key_hash")?; + let ttl = parse_int(params.next(), "expire_at")? - now(); + Ok(TLObject::new(ton::rpc::engine::validator::AddValidatorBlsKey { + permanent_key_hash, + key_hash, + ttl + })) + } +} + impl SendReceive for AddValidatorTempKey { fn send(mut params: impl Iterator) -> Result { let permanent_key_hash = parse_int256(params.next(), "permanent_key_hash")?; @@ -602,6 +634,14 @@ impl ControlClient { Ok((format!("Message body is {} saved to path {}", base64_encode(&data), path), data)) } + fn convert_to_uint(value: &[u8], bytes_count: usize) -> TokenValue { + assert!(value.len() == bytes_count); + TokenValue::Uint(Uint { + number: num_bigint::BigUint::from_bytes_be(value), + size: value.len() * 8, + }) + } + // @input elect_time expire_time // @output validator-query.boc async fn process_election_bid( @@ -677,19 +717,68 @@ impl ControlClient { Ed25519KeyOption::from_public_key(&pub_key[..].try_into()?) .verify(&data, &signature)?; - let query_id = now() as u64; - // validator-elect-signed.fif - let mut data = 0x4E73744Bu32.to_be_bytes().to_vec(); - data.extend_from_slice(&query_id.to_be_bytes()); - data.extend_from_slice(&pub_key); - data.extend_from_slice(&elect_time.to_be_bytes()); - data.extend_from_slice(&max_factor.to_be_bytes()); - data.extend_from_slice(&adnl); - let len = data.len() * 8; - let mut body = BuilderData::with_raw(data, len)?; - let len = signature.len() * 8; - body.checked_append_reference(BuilderData::with_raw(signature, len)?.into_cell()?)?; - let body = body.into_cell()?; + let body = if USE_FIFTH_ELECTOR { + let query_id = now() as u64; + // validator-elect-signed.fif + let mut data = 0x4E73744Bu32.to_be_bytes().to_vec(); + data.extend_from_slice(&query_id.to_be_bytes()); + data.extend_from_slice(&pub_key); + data.extend_from_slice(&elect_time.to_be_bytes()); + data.extend_from_slice(&max_factor.to_be_bytes()); + data.extend_from_slice(&adnl); + let len = data.len() * 8; + let mut body = BuilderData::with_raw(data, len)?; + let len = signature.len() * 8; + body.checked_append_reference(BuilderData::with_raw(signature, len)?.into_cell()?)?; + let body = body.into_cell()?; + body + } else { + let (s, bls) = self.process_command("newkey", vec!["bls"].iter()).await?; + log::trace!("{}", s); + let bls_str = &hex::encode_upper(&bls)[..]; + + let (s, bls_pub_key) = self.process_command("exportpub", vec![bls_str].iter()).await?; + log::trace!("{}", s); + + let (s, _) = self.process_command("addblskey", vec![perm_str.clone(), bls_str.to_string(), elect_time_str].iter()).await?; + log::trace!("{}", s); + + let contract = Contract::load(ELECTOR_ABI).expect("Elector's ABI must be valid"); + let process_new_stake_fn = contract + .function(ELECTOR_PROCESS_NEW_STAKE_FUNC_NAME) + .expect("Elector contract must have 'process_new_stake' function for elections") + .clone(); + log::trace!("Use process new stake function '{}' with id={:08X}", + ELECTOR_PROCESS_NEW_STAKE_FUNC_NAME, process_new_stake_fn.get_function_id()); + + let time_now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + let header: HashMap<_, _> = vec![("time".to_owned(), TokenValue::Time(time_now_ms))] + .into_iter() + .collect(); + + let query_id = now() as u64; + + let parameters = [ + Token::new("query_id", Self::convert_to_uint(&query_id.to_be_bytes(), 8)), + Token::new("validator_pubkey", Self::convert_to_uint(&pub_key, 32)), + Token::new("stake_at", Self::convert_to_uint(&elect_time.to_be_bytes(), 4)), + Token::new("max_factor", Self::convert_to_uint(&max_factor.to_be_bytes(), 4)), + Token::new("adnl_addr", Self::convert_to_uint(&adnl, 32)), + Token::new("bls_key1", Self::convert_to_uint(&bls_pub_key[0..32], 32)), //256 bits + Token::new("bls_key2", Self::convert_to_uint(&bls_pub_key[32..], 16)), //128 bits + Token::new("signature", TokenValue::Bytes(signature.to_vec())), + ]; + + const INTERNAL_CALL: bool = true; //internal message + + let body = process_new_stake_fn + .encode_input(&header, ¶meters, INTERNAL_CALL, None, None) + .and_then(|builder| builder.into_cell())?; + body + }; log::trace!("message body {}", body); let data = ton_types::write_boc(&body)?; let path = params.next().map( diff --git a/catchain/src/lib.rs b/catchain/src/lib.rs index 04806131..07d46192 100644 --- a/catchain/src/lib.rs +++ b/catchain/src/lib.rs @@ -25,8 +25,7 @@ mod receiver_source; pub mod utils; use crate::{profiling::InstanceCounter, utils::MetricsHandle}; -use adnl::node::AdnlNode; -use adnl::PrivateOverlayShortId; +use adnl::{PrivateOverlayShortId, node::AdnlNode}; use std::{ any::Any, cell::RefCell, fmt, path::Path, rc::{Rc, Weak}, sync::Arc, time::SystemTime, diff --git a/catchain/src/profiling.rs b/catchain/src/profiling.rs index c6f57e1e..36014352 100644 --- a/catchain/src/profiling.rs +++ b/catchain/src/profiling.rs @@ -525,16 +525,16 @@ macro_rules! function { macro_rules! instrument { () => { lazy_static::lazy_static! { - static ref INSTRUMENT_CODE_LINE: crate::profiling::ProfileCodeLine = - crate::profiling::ProfileCodeLine { + static ref INSTRUMENT_CODE_LINE: $crate::profiling::ProfileCodeLine = + $crate::profiling::ProfileCodeLine { file_name: file!(), - function_name: crate::function!(85), + function_name: $crate::function!(85), line: line!(), }; } let _instrument_guard = - crate::profiling::ProfileGuard::new(&INSTRUMENT_CODE_LINE); + $crate::profiling::ProfileGuard::new(&INSTRUMENT_CODE_LINE); }; } @@ -543,17 +543,17 @@ macro_rules! instrument { macro_rules! check_execution_time { ($max_duration_microseconds:expr) => { lazy_static::lazy_static! { - static ref CHECK_EXECUTION_TIME_CODE_LINE: crate::profiling::ProfileCodeLine = - crate::profiling::ProfileCodeLine { + static ref CHECK_EXECUTION_TIME_CODE_LINE: $crate::profiling::ProfileCodeLine = + $crate::profiling::ProfileCodeLine { file_name: file!(), - function_name: crate::function!(95), + function_name: $crate::function!(95), line: line!(), }; static ref CHECK_EXECUTION_TIME_MAX_DURATION: std::time::Duration = std::time::Duration::from_micros($max_duration_microseconds); } - let _check_execution_time_guard = crate::profiling::ExecutionTimeGuard::new( + let _check_execution_time_guard = $crate::profiling::ExecutionTimeGuard::new( &CHECK_EXECUTION_TIME_CODE_LINE, &CHECK_EXECUTION_TIME_MAX_DURATION, ); diff --git a/catchain/src/utils.rs b/catchain/src/utils.rs index 7f1c0e63..ebc65ab8 100644 --- a/catchain/src/utils.rs +++ b/catchain/src/utils.rs @@ -13,7 +13,7 @@ /// Imports use crate::{ - serialize_tl_boxed_object, + serialize_tl_boxed_object, BlockHash, BlockPayloadPtr, CatchainFactory, PrivateKey, PublicKey, PublicKeyHash, RawBuffer, Receiver, SessionId, ton, utils }; @@ -506,6 +506,25 @@ impl MetricsDumper { self.prev_metrics = metrics; } + pub fn enumerate_as_f64(&self, handler: F) + where + F: Fn(String, f64), + { + for (key, metric) in &self.prev_metrics { + use MetricUsage::*; + + let value = match metric.usage { + Counter => metric.value as f64, + Derivative => metric.value as f64 / Self::METRIC_DERIVATIVE_MULTIPLIER, + Percents => (metric.value as f64) / Self::METRIC_FLOAT_MULTIPLIER * 100.0, + Float => (metric.value as f64) / Self::METRIC_FLOAT_MULTIPLIER, + Latency => (metric.value as f64) / Self::METRIC_FLOAT_MULTIPLIER, + }; + + handler(key.clone(), value); + } + } + pub fn dump(&self, handler: F) where F: Fn(String), diff --git a/config.md b/config.md index 82c5de59..5b9ed53b 100644 --- a/config.md +++ b/config.md @@ -65,3 +65,6 @@ Enables participation in validator REMP protocols. Default value is `true`. for longer periods will be easily identified as duplicates (the validator will already have the same message received through Catchain from another validtor). The parameter specifies maximal delay. + +* `smft_disabled`: manually disables participation of the node in SMFT protocol even if corresponding network config is set; false by default + diff --git a/src/config.rs b/src/config.rs index 4cae53c2..d8ceaff7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,8 @@ */ use crate::network::node_network::NodeNetwork; +#[cfg(feature="workchains")] +use crate::validator::validator_utils::mine_key_for_workchain; use adnl::{ client::AdnlClientConfigJson, @@ -21,9 +23,9 @@ use adnl::{ use storage::shardstate_db_async::CellsDbConfig; use std::{ collections::{HashMap, HashSet}, convert::TryInto, fs::File, fmt::{Display, Formatter}, - io::BufReader, path::Path, sync::{Arc, atomic::{self, AtomicI32}}, time::{Duration} + io::BufReader, path::{Path, PathBuf}, sync::{Arc, atomic::{self, AtomicI32}}, + time::{Duration} }; -use std::path::PathBuf; use ton_api::{ IntoBoxed, ton::{ @@ -35,8 +37,8 @@ use ton_block::{BlockIdExt, ShardIdent}; #[cfg(feature="external_db")] use ton_block::{BASE_WORKCHAIN_ID, MASTERCHAIN_ID}; use ton_types::{ - error, fail, base64_decode, base64_encode, Ed25519KeyOption, KeyId, KeyOption, KeyOptionJson, - Result, UInt256 + error, fail, base64_decode, base64_encode, BlsKeyOption, Ed25519KeyOption, KeyId, KeyOption, + KeyOptionJson, Result, UInt256 }; #[macro_export] @@ -54,7 +56,7 @@ macro_rules! key_option_public_key { #[async_trait::async_trait] pub trait KeyRing : Sync + Send { - async fn generate(&self) -> Result<[u8; 32]>; + async fn generate(&self, key_type: i32) -> Result<[u8; 32]>; // find private key in KeyRing by public key hash fn find(&self, key_hash: &[u8; 32]) -> Result>; fn sign_data(&self, key_hash: &[u8; 32], data: &[u8]) -> Result>; @@ -178,6 +180,8 @@ pub struct TonNodeConfig { states_cache_mode: ShardStatesCacheMode, #[serde(default)] sync_by_archives: bool, + #[serde(default)] + smft_disabled: bool, } pub struct TonNodeGlobalConfig(TonNodeGlobalConfigJson); @@ -210,7 +214,8 @@ impl NodeExtensions { struct ValidatorKeysJson { election_id: i32, validator_key_id: String, - validator_adnl_key_id: Option + validator_adnl_key_id: Option, + validator_bls_key: Option, } #[derive(serde::Deserialize, serde::Serialize, Default, Debug, Clone)] @@ -422,6 +427,10 @@ impl TonNodeConfig { self.boot_from_zerostate.unwrap_or(false) } + pub fn is_smft_disabled(&self) -> bool { + self.smft_disabled + } + pub fn from_file( configs_dir: &str, json_file_name: &str, @@ -716,6 +725,7 @@ impl TonNodeConfig { if keys_info.election_id == updated_info.election_id { keys_info.validator_key_id = updated_info.validator_key_id; keys_info.validator_adnl_key_id = updated_info.validator_adnl_key_id; + keys_info.validator_bls_key = updated_info.validator_bls_key; return Ok(keys_info.clone()); } } @@ -738,9 +748,17 @@ impl TonNodeConfig { Ok(()) } - fn generate_and_save_keys(&mut self) -> Result<([u8; 32], Arc)> { - let (private, public) = crate::validator::validator_utils::mine_key_for_workchain(self.workchain); + fn generate_and_save_keys(&mut self, _key_type: i32) -> Result<([u8; 32], Arc)> { + #[cfg(feature="workchains")] + let (private, public) = match _key_type { + Ed25519KeyOption::KEY_TYPE => mine_key_for_workchain(self.workchain), + BlsKeyOption::KEY_TYPE => BlsKeyOption::generate_with_json()?, + _ => fail!("Unknown generate key type!"), + }; + #[cfg(not(feature="workchains"))] + let (private, public) = Ed25519KeyOption::generate_with_json()?; let key_id = public.id().data(); + log::info!("generate_and_save_keys: generate new key (id: {:?})", key_id); let key_ring = self.validator_key_ring.get_or_insert_with(|| HashMap::new()); key_ring.insert(base64_encode(key_id), private); Ok((key_id.clone(), public)) @@ -765,7 +783,8 @@ impl TonNodeConfig { let key_info = ValidatorKeysJson { election_id, validator_key_id: base64_encode(key_id), - validator_adnl_key_id: None + validator_adnl_key_id: None, + validator_bls_key: None }; if !self.is_correct_election_id(election_id) { @@ -792,6 +811,19 @@ impl TonNodeConfig { Ok(key_info) } + fn add_validator_bls_key( + &mut self, + validator_key_id: &[u8; 32], + bls_key: &[u8; 32] + ) -> Result { + if let Some(mut key_info) = self.get_validator_key_info(&base64_encode(validator_key_id))? { + key_info.validator_bls_key = Some(base64_encode(bls_key)); + self.update_validator_key_info(key_info) + } else { + fail!("Validator key have not been added!") + } + } + fn add_validator_adnl_key( &mut self, validator_key_id: &[u8; 32], @@ -805,7 +837,7 @@ impl TonNodeConfig { key_info.validator_adnl_key_id = Some(base64_encode(adnl_key_id)); self.update_validator_key_info(key_info) } else { - fail!("Validator key was not added!") + fail!("Validator key have not been added!") } } @@ -840,10 +872,12 @@ pub trait NodeConfigSubscriber: Send + Sync { #[derive(Debug)] enum Task { - Generate, + Generate(i32), AddValidatorKey([u8; 32], i32), AddValidatorAdnlKey([u8; 32], [u8; 32]), + AddValidatorBlsKey([u8; 32], [u8; 32]), GetKey([u8; 32]), + GetBlsKey([u8; 32]), StoreStatesGcInterval(u32), } @@ -907,6 +941,29 @@ impl NodeConfigHandler { } } + pub async fn add_validator_bls_key( + &self, + validator_key_hash: &[u8; 32], + validator_bls_key_hash: &[u8; 32] + ) -> Result<()> { + let (wait, mut queue_reader) = Wait::new(); + let pushed_task = Arc::new(( + wait.clone(), + Task::AddValidatorBlsKey(validator_key_hash.clone(), validator_bls_key_hash.clone()) + )); + + wait.request(); + if let Err(e) = self.sender.send(pushed_task) { + fail!("Error add_validator_bls_key: {}", e); + } + match wait.wait(&mut queue_reader, true).await { + Some(None) => fail!("Answer was not set!"), + Some(Some(Answer::Result(result))) => result, + Some(Some(_)) => fail!("Bad answer (AddValidatorBlsKey)!"), + None => fail!("Waiting returned an internal error!") + } + } + pub async fn add_validator_adnl_key( &self, validator_key_hash: &[u8; 32], @@ -1008,6 +1065,13 @@ impl NodeConfigHandler { } } + pub async fn get_validator_bls_key(&self, key_id: &Arc) -> Option> { + match self.validator_keys.get(&base64_encode(key_id.data())) { + Some(_key) => self.get_bls_key_raw(*key_id.data()).await, + None => None, + } + } + async fn get_key_raw(&self, key_hash: [u8; 32]) -> Option> { let (wait, mut queue_reader) = Wait::new(); let pushed_task = Arc::new((wait.clone(), Task::GetKey(key_hash))); @@ -1022,21 +1086,38 @@ impl NodeConfigHandler { } } + async fn get_bls_key_raw(&self, key_hash: [u8; 32]) -> Option> { + let (wait, mut queue_reader) = Wait::new(); + let pushed_task = Arc::new((wait.clone(), Task::GetBlsKey(key_hash))); + wait.request(); + if let Err(e) = self.sender.send(pushed_task) { + log::warn!("Error get_bls_key_raw {}", e); + return None; + } + match wait.wait(&mut queue_reader, true).await { + Some(Some(Answer::GetKey(key))) => key, + _ => return None + } + } + fn generate_and_save( key_ring: &Arc>>, + key_type: i32, config: &mut TonNodeConfig, config_name: &str ) -> Result<[u8; 32]> { - let (key_id, public_key) = config.generate_and_save_keys()?; + log::info!("start generate key (type: {})", key_type); + let (key_id, public_key) = config.generate_and_save_keys(key_type)?; config.save_to_file(config_name)?; let id = base64_encode(&key_id); key_ring.insert(id, public_key.clone()); + log::info!("finish generate key (type: {}), key_id: {:?}", key_type, key_id); Ok(key_id) } fn revision_validator_keys( - validator_keys: Arc, + validator_keys: &Arc, config: &mut TonNodeConfig )-> Result<()> { loop { @@ -1054,6 +1135,9 @@ impl NodeConfigHandler { if let Some(adnl_key_id) = oldest_key.validator_adnl_key_id { config.remove_key_from_key_ring(&adnl_key_id); } + if let Some(bls_key_id) = oldest_key.validator_bls_key { + config.remove_key_from_key_ring(&bls_key_id); + } } } else { break; @@ -1065,29 +1149,52 @@ impl NodeConfigHandler { Ok(()) } + fn add_validator_bls_key_and_save( + self: Arc, + validator_keys: &Arc, + config: &mut TonNodeConfig, + validator_key_hash: &[u8; 32], + validator_bls_key_hash: &[u8; 32] + )-> Result<()> { + let key = config.add_validator_bls_key(&validator_key_hash, validator_bls_key_hash)?; + if key.validator_adnl_key_id.is_some() && key.validator_bls_key.is_some() { + validator_keys.add(key)?; + } + + // check validator keys + Self::revision_validator_keys(validator_keys, config)?; + config.save_to_file(&config.file_name)?; + Ok(()) + } + fn add_validator_adnl_key_and_save( self: Arc, - validator_keys: Arc, + validator_keys: &Arc, config: &mut TonNodeConfig, validator_key_hash: &[u8; 32], validator_adnl_key_hash: &[u8; 32], - subscribers: Vec> + subscribers: &Vec> )-> Result<()> { let key = config.add_validator_adnl_key(&validator_key_hash, validator_adnl_key_hash)?; let election_id = key.election_id.clone(); - validator_keys.add(key)?; + //if key.validator_adnl_key_id.is_some() && key.validator_bls_key.is_some() { + validator_keys.add(key)?; + //} let adnl_key_id = KeyId::from_data(*validator_adnl_key_hash); - self.clone().runtime_handle.spawn(async move { - for subscriber in subscribers.iter() { + for subscriber in subscribers.iter() { + let subscriber = subscriber.clone(); + let adnl_key_id = adnl_key_id.clone(); + self.clone().runtime_handle.spawn(async move { if let Err(e) = subscriber.event( - ConfigEvent::AddValidatorAdnlKey(adnl_key_id.clone(), election_id) + ConfigEvent::AddValidatorAdnlKey(adnl_key_id, election_id) ).await { log::warn!("subscriber error: {:?}", e); } - } - }); + }); + } + // check validator keys Self::revision_validator_keys(validator_keys, config)?; config.save_to_file(&config.file_name)?; @@ -1126,7 +1233,7 @@ impl NodeConfigHandler { if let Some(validator_key_ring) = &config.validator_key_ring { if let Some(key_data) = validator_key_ring.get(&base64_encode(&key_id)) { match Ed25519KeyOption::from_private_key_json(&key_data) { - Ok(key) => { return Some(key)}, + Ok(key) => { return Some(key) }, _ => return None } } @@ -1134,6 +1241,22 @@ impl NodeConfigHandler { None } + fn get_bls_key(config: &TonNodeConfig, key_id: [u8; 32]) -> Option> { + if let Ok(Some(key_info)) = config.get_validator_key_info(&base64_encode(key_id)) { + if let Some(validator_key_ring) = &config.validator_key_ring { + if let Some(bls_key) = &key_info.validator_bls_key { + if let Some(key_data) = validator_key_ring.get(bls_key) { + match BlsKeyOption::from_private_key_json(&key_data) { + Ok(key) => { return Some(key) }, + _ => return None + } + } + } + } + } + None + } + fn load_config(&self, config: &TonNodeConfig, subscribers: &Vec>) -> Result<()> { // load key ring if let Some(key_ring) = &config.validator_key_ring { @@ -1174,7 +1297,12 @@ impl NodeConfigHandler { } fn add_key_to_dynamic_key_ring(&self, key_id: String, key_json: &KeyOptionJson) -> Result<()> { - if let Some(key) = self.key_ring.insert(key_id, Ed25519KeyOption::from_private_key_json(key_json)?) { + let key = match *key_json.type_id() { + Ed25519KeyOption::KEY_TYPE => Ed25519KeyOption::from_private_key_json(key_json)?, + BlsKeyOption::KEY_TYPE => BlsKeyOption::from_private_key_json(key_json)?, + _ => fail!("Unknown key type (key_id: {})", key_id), + }; + if let Some(key) = self.key_ring.insert(key.id().to_string(), key) { log::warn!("Added key was already in key ring collection (id: {})", key.key()); } @@ -1196,31 +1324,52 @@ impl NodeConfigHandler { self.clone().runtime_handle.spawn(async move { while let Some(task) = reader.recv().await { let answer = match task.1 { - Task::Generate => { - let result = NodeConfigHandler::generate_and_save(&key_ring, &mut actual_config, &name); + Task::Generate(key_type) => { + let result = NodeConfigHandler::generate_and_save(&key_ring, key_type, &mut actual_config, &name); Answer::Generate(result) - } + }, Task::AddValidatorAdnlKey(key, adnl_key) => { let result = NodeConfigHandler::add_validator_adnl_key_and_save( self.clone(), - validator_keys.clone(), + &validator_keys, &mut actual_config, &key, &adnl_key, - subscribers.clone() + &subscribers ); Answer::Result(result) - } + }, + Task::AddValidatorBlsKey(key, bls_key_id) => { + let result = NodeConfigHandler::add_validator_bls_key_and_save( + self.clone(), + &validator_keys, + &mut actual_config, + &key, + &bls_key_id + ); + Answer::Result(result) + }, Task::AddValidatorKey(key, election_id) => { let result = NodeConfigHandler::add_validator_key_and_save( validator_keys.clone(), &mut actual_config, &key, election_id ); Answer::Result(result) - } + }, Task::GetKey(key_data) => { let result = NodeConfigHandler::get_key(&actual_config, key_data); Answer::GetKey(result) - } + + }, + Task::GetBlsKey(key_data) => { + let result = NodeConfigHandler::get_bls_key(&actual_config, key_data); + Answer::GetKey(result) + }, + #[cfg(feature="workchains")] + Task::StoreWorkchainId(workchain_id) => { + actual_config.workchain = Some(workchain_id); + let result = actual_config.save_to_file(&name); + Answer::Result(result) + }, Task::StoreStatesGcInterval(interval) => { if let Some(c) = &mut actual_config.gc { c.cells_gc_config.gc_interval_sec = interval; @@ -1247,9 +1396,10 @@ impl NodeConfigHandler { #[async_trait::async_trait] impl KeyRing for NodeConfigHandler { - async fn generate(&self) -> Result<[u8; 32]> { + async fn generate(&self, key_type: i32) -> Result<[u8; 32]> { + log::info!("request generate key (key_type: {})", key_type); let (wait, mut queue_reader) = Wait::new(); - let pushed_task = Arc::new((wait.clone(), Task::Generate)); + let pushed_task = Arc::new((wait.clone(), Task::Generate(key_type))); wait.request(); if let Err(e) = self.sender.send(pushed_task) { fail!("Error generate: {}", e); @@ -1591,7 +1741,8 @@ pub struct ValidatorManagerConfig { pub unsafe_resync_catchains: HashSet, /// Maps catchain_seqno to block_seqno and unsafe rotation id pub unsafe_catchain_rotates: HashMap, - pub no_countdown_for_zerostate: bool + pub no_countdown_for_zerostate: bool, + pub smft_disabled: bool, } #[derive(serde::Deserialize, serde::Serialize)] @@ -1621,7 +1772,7 @@ impl Display for ValidatorManagerConfig { } impl ValidatorManagerConfig { - pub fn read_configs(config_files: Vec, validation_countdown_mode: Option) -> ValidatorManagerConfig { + pub fn read_configs(config_files: Vec, validation_countdown_mode: Option, smft_disabled: bool) -> ValidatorManagerConfig { log::debug!(target: "validator", "Reading validator manager config files: {}", config_files.iter().map(|x| format!("{}; ",x)).collect::()); @@ -1636,6 +1787,8 @@ impl ValidatorManagerConfig { None => () } + validator_config.smft_disabled = smft_disabled; + 'iterate_configs: for one_config in config_files.into_iter() { if let Ok(config_file) = std::fs::File::open(one_config.clone()) { let reader = std::io::BufReader::new(config_file); @@ -1686,7 +1839,8 @@ impl Default for ValidatorManagerConfig { update_interval: Duration::from_secs(3), unsafe_resync_catchains: HashSet::new(), unsafe_catchain_rotates: HashMap::new(), - no_countdown_for_zerostate: false + no_countdown_for_zerostate: false, + smft_disabled: false, } } } diff --git a/src/engine.rs b/src/engine.rs index adf0c5e5..132fca80 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -138,6 +138,8 @@ pub struct Engine { sync_status: AtomicU32, remp_capability: AtomicBool, + smft_capability: AtomicBool, + test_bundles_config: CollatorTestBundlesGeneralConfig, collator_config: CollatorConfig, @@ -808,6 +810,7 @@ impl Engine { will_validate: AtomicBool::new(false), sync_status: AtomicU32::new(0), remp_capability: AtomicBool::new(false), + smft_capability: AtomicBool::new(false), test_bundles_config, collator_config, shard_states_keeper: shard_states_keeper.clone(), @@ -958,6 +961,10 @@ impl Engine { self.overlay_operations.clone().add_overlay(id, !foreign).await } + pub fn calc_overlay_id(&self, workchain: i32, shard: u64) -> Result<(Arc, adnl::OverlayId)> { + self.overlay_operations.calc_overlay_id(workchain, shard) + } + pub fn shard_states_awaiters(&self) -> &AwaitersPool> { &self.shard_states_awaiters } @@ -1114,6 +1121,14 @@ impl Engine { self.candidate_db.destroy_db(session_id) } + pub fn smft_capability(&self) -> bool { + self.smft_capability.load(Ordering::Relaxed) + } + + pub fn set_smft_capability(&self, value: bool) { + self.smft_capability.store(value, Ordering::Relaxed); + } + pub fn split_queues_cache(&self) -> &lockfree::map::Map)>> { @@ -1428,6 +1443,10 @@ impl Engine { block.get_config_params()?.has_capability(GlobalCapabilities::CapRemp), Ordering::Relaxed ); + self.smft_capability.store( + block.get_config_params()?.has_capability(GlobalCapabilities::CapSmft), + Ordering::Relaxed + ); // While the node boots start key block is not processed by this function. // So see process_full_state_in_ext_db for the same code } @@ -1553,7 +1572,9 @@ impl Engine { Broadcast::TonNode_ConnectivityCheckBroadcast(broadcast) => { self.network.clone().process_connectivity_broadcast(broadcast); } - Broadcast::TonNode_BlockCandidateBroadcast(_) => { } + Broadcast::TonNode_BlockCandidateBroadcast(broadcast) => { + log::warn!("TonNode_BlockCandidateBroadcast from {}: {:?}", src, broadcast); + } } } } @@ -2398,6 +2419,10 @@ async fn boot( state.config_params()?.has_capability(GlobalCapabilities::CapRemp), Ordering::Relaxed ); + engine.smft_capability.store( + state.config_params()?.has_capability(GlobalCapabilities::CapSmft), + Ordering::Relaxed + ); (block_id.clone(), false) } Err(err) => { @@ -2492,7 +2517,8 @@ pub async fn run( let remp_config = node_config.remp_config().clone(); let vm_config = ValidatorManagerConfig::read_configs( node_config.unsafe_catchain_patches_files(), - node_config.validation_countdown_mode() + node_config.validation_countdown_mode(), + node_config.is_smft_disabled(), ); let wc_from_config = node_config.workchain(); let remp_client_pool = node_config.remp_config().remp_client_pool(); diff --git a/src/engine_operations.rs b/src/engine_operations.rs index b13f2a29..72d887b6 100644 --- a/src/engine_operations.rs +++ b/src/engine_operations.rs @@ -110,6 +110,10 @@ impl EngineOperations for Engine { self.network().activate_validator_list(validator_list_id) } + async fn get_validator_bls_key(&self, key_id: &Arc) -> Option> { + self.network().get_validator_bls_key(key_id).await + } + fn set_sync_status(&self, status: u32) { self.set_sync_status(status); } @@ -118,6 +122,10 @@ impl EngineOperations for Engine { self.get_sync_status() } + fn calc_overlay_id(&self, workchain: i32, shard: u64) -> Result<(Arc, adnl::OverlayId)> { + self.calc_overlay_id(workchain, shard) + } + fn validation_status(&self) -> ValidationStatus { self.validation_status() } @@ -713,10 +721,14 @@ impl EngineOperations for Engine { } // Initialisation of remp_capability after cold boot by first processed master state + // same for SMFT if state.block_id().shard().is_masterchain() { self.set_remp_capability( state.config_params()?.has_capability(GlobalCapabilities::CapRemp), ); + self.set_smft_capability( + state.config_params()?.has_capability(GlobalCapabilities::CapSmft), + ); } Ok(()) } @@ -1017,6 +1029,10 @@ impl EngineOperations for Engine { Engine::remp_capability(self) } + fn smft_capability(&self) -> bool { + Engine::smft_capability(self) + } + // Get current list of new shard blocks with respect to last mc block. // If given mc_seq_no is not equal to last mc seq_no - function fails. async fn get_shard_blocks( diff --git a/src/engine_traits.rs b/src/engine_traits.rs index 1181718e..cb143730 100644 --- a/src/engine_traits.rs +++ b/src/engine_traits.rs @@ -114,6 +114,8 @@ pub trait PrivateOverlayOperations: Sync + Send { async fn remove_validator_list(&self, validator_list_id: UInt256) -> Result; + async fn get_validator_bls_key(&self, key_id: &Arc) -> Option>; + fn create_catchain_client( &self, validator_list_id: UInt256, @@ -141,6 +143,10 @@ pub trait EngineOperations : Sync + Send { unimplemented!() } + fn calc_overlay_id(&self, workchain: i32, shard: u64) -> Result<(Arc, OverlayId)> { + unimplemented!() + } + fn validation_status(&self) -> ValidationStatus { unimplemented!() } @@ -194,6 +200,10 @@ pub trait EngineOperations : Sync + Send { unimplemented!() } + async fn get_validator_bls_key(&self, key_id: &Arc) -> Option> { + unimplemented!() + } + fn set_sync_status(&self, status: u32) { unimplemented!() } @@ -769,6 +779,10 @@ pub trait EngineOperations : Sync + Send { false } + fn smft_capability(&self) -> bool { + false + } + async fn update_validators( &self, to_resolve: Vec, diff --git a/src/network/catchain_client.rs b/src/network/catchain_client.rs index f811810d..52abd1a0 100644 --- a/src/network/catchain_client.rs +++ b/src/network/catchain_client.rs @@ -257,6 +257,7 @@ impl CatchainClient { let overlay_id = private_overlay_id.clone(); let self1 = self.clone(); let self2 = self.clone(); + let self3 = self.clone(); let overlay = overlay_node.clone(); let keys = validator_keys.clone(); let listener = catchain_listener.clone(); @@ -274,6 +275,14 @@ impl CatchainClient { log::warn!(target: Self::TARGET, "ERROR: {}", e) } }); + let overlay_id = private_overlay_id.clone(); + let overlay = overlay_node.clone(); + let listener = catchain_listener.clone(); + runtime_handle.spawn(async move { + if let Err(e) = CatchainClient::wait_block_candidate_status_broadcast(self3, &overlay_id, overlay, &listener).await { + log::warn!(target: Self::TARGET, "ERROR: {}", e) + } + }); } async fn wait_broadcasts( @@ -354,6 +363,44 @@ impl CatchainClient { let _result = result.ok_or_else(|| error!("Failed to receive a private overlay broadcast!"))?; Ok(()) } + + async fn wait_block_candidate_status_broadcast( + self: Arc, + overlay_id: &Arc, + overlay: Arc, + catchain_listener: &CatchainOverlayListenerPtr) -> Result<()> { + let receiver = overlay.clone(); + let result: Option> = None; + let catchain_listener = catchain_listener.clone(); + + while let None = result { + if self.is_stop.load(atomic::Ordering::Relaxed) { + break; + }; + let message = receiver.wait_for_block_candidate_status(overlay_id).await; + match message { + Ok(Some((block_status, source_id))) => { + log::trace!(target: Self::TARGET, "private overlay broadcast BlockCandidateStatus (successed)"); + if let Some(listener) = catchain_listener.upgrade() { + let mut data: catchain::RawBuffer = catchain::RawBuffer::default(); + let mut serializer = ton_api::Serializer::new(&mut data); + serializer.write_boxed(&block_status)?; + let data = catchain::CatchainFactory::create_block_payload(data); + listener + .on_message( + source_id, + &data); + } + }, + Ok(None) => { return Ok(())}, + Err(e) => { + log::error!(target: Self::TARGET, "private overlay broadcast err: {}", e); + }, + }; + } + let _result = result.ok_or_else(|| error!("Failed to receive a private overlay broadcast!"))?; + Ok(()) + } } impl CatchainOverlay for CatchainClient { diff --git a/src/network/control.rs b/src/network/control.rs index 5c55ed5a..fefff6fd 100644 --- a/src/network/control.rs +++ b/src/network/control.rs @@ -42,6 +42,7 @@ use ton_api::{ }, rpc::engine::validator::{ AddAdnlId, AddValidatorAdnlAddress, AddValidatorPermanentKey, AddValidatorTempKey, + AddValidatorBlsKey, GenerateBlsKeyPair, ControlQuery, ExportPublicKey, GenerateKeyPair, Sign, GetBundle, GetFutureBundle } } @@ -483,9 +484,9 @@ impl ControlQuerySubscriber { } - async fn process_generate_keypair(&self) -> Result { + async fn process_generate_keypair(&self, key_type: i32) -> Result { let ret = KeyHash { - key_hash: UInt256::with_array(self.key_ring.generate().await?) + key_hash: UInt256::with_array(self.key_ring.generate(key_type).await?) }; Ok(ret) } @@ -529,6 +530,11 @@ impl ControlQuerySubscriber { Ok(Success::Engine_Validator_Success) } + async fn add_validator_bls_key(&self, perm_key_hash: &[u8; 32], key_hash: &[u8; 32], _ttl: ton::int) -> Result { + self.config.add_validator_bls_key(perm_key_hash, key_hash).await?; + Ok(Success::Engine_Validator_Success) + } + fn add_adnl_address(&self, _key_hash: &[u8; 32], _category: ton::int) -> Result { Ok(Success::Engine_Validator_Success) } @@ -631,8 +637,16 @@ impl ControlQuerySubscriber { Err(query) => query }; let query = match query.downcast::() { - Ok(_) => return QueryResult::consume( - self.process_generate_keypair().await?, + Ok(_params) => return QueryResult::consume( + self.process_generate_keypair(ton_types::Ed25519KeyOption::KEY_TYPE).await?, + #[cfg(feature = "telemetry")] + None + ), + Err(query) => query + }; + let query = match query.downcast::() { + Ok(_params) => return QueryResult::consume( + self.process_generate_keypair(ton_types::BlsKeyOption::KEY_TYPE).await?, #[cfg(feature = "telemetry")] None ), @@ -684,6 +698,15 @@ impl ControlQuerySubscriber { ), Err(query) => query }; + let query = match query.downcast::() { + Ok(query) => return QueryResult::consume_boxed( + self.add_validator_bls_key( + query.permanent_key_hash.as_slice(), query.key_hash.as_slice(), query.ttl + ).await?, + None + ), + Err(query) => query + }; let query = match query.downcast::() { Ok(query) => return QueryResult::consume_boxed( self.add_adnl_address(query.key_hash.as_slice(), query.category)?, diff --git a/src/network/neighbours.rs b/src/network/neighbours.rs index 7bb54ba4..c9455463 100644 --- a/src/network/neighbours.rs +++ b/src/network/neighbours.rs @@ -451,7 +451,7 @@ impl Neighbours { self.cancellation_token.cancel(); } - fn add_new_peers(self: Arc, peers: Vec>) { + pub fn add_new_peers(self: Arc, peers: Vec>) { let this = self.clone(); tokio::spawn(async move { for peer in peers.iter() { diff --git a/src/network/node_network.rs b/src/network/node_network.rs index 9bd99418..7d953873 100644 --- a/src/network/node_network.rs +++ b/src/network/node_network.rs @@ -982,7 +982,26 @@ impl NodeNetwork { } match self.config_handler.get_validator_key(&validator_adnl_key_id).await { Some((adnl_key, _)) => { - let id = self.network_context.adnl.add_key(adnl_key, election_id as usize)?; + let id = match self.network_context.adnl.add_key(adnl_key.clone(), election_id as usize) { + Ok(id) => id, + Err(e) => { + if let Ok(old_key) = self.network_context.adnl.key_by_tag(election_id as usize) { + if *old_key.id() != validator_adnl_key_id { + log::warn!("load_and_store_adnl_key: remove old key {}, election_id: {}", + old_key.id(), election_id); + } else { + log::warn!("load_and_store_adnl_key: remove key {}, election_id: {}", + old_key.id(), election_id); + } + + let _ = self.network_context.adnl.delete_key(old_key.id(), election_id as usize); + self.network_context.adnl.add_key(adnl_key, election_id as usize)? + } else { + fail!("failed initialization new validator key (id: {}, election_id: {}): {:?}", + validator_adnl_key_id, election_id, e); + } + } + }; NodeNetwork::periodic_store_ip_addr( self.network_context.dht.clone(), self.network_context.adnl.key_by_id(&id)?, @@ -1088,6 +1107,10 @@ impl OverlayOperations for NodeNetwork { #[async_trait::async_trait] impl PrivateOverlayOperations for NodeNetwork { + async fn get_validator_bls_key(&self, key_id: &Arc) -> Option> { + self.config_handler.get_validator_bls_key(key_id).await + } + async fn set_validator_list( &self, validator_list_id: UInt256, @@ -1352,7 +1375,7 @@ impl NodeConfigSubscriber for NodeNetwork { // Unused // ConfigEvent::RemoveValidatorAdnlKey(validator_adnl_key_id, election_id) => { // log::info!("config event (RemoveValidatorAdnlKey) id: {}.", &validator_adnl_key_id); -// self.adnl.delete_key(&validator_adnl_key_id, election_id as usize)?; +// self.network_context.adnl.delete_key(&validator_adnl_key_id, election_id as usize)?; // let status = self.validator_context.actual_local_adnl_keys.remove(&validator_adnl_key_id).is_some(); // log::info!("config event (RemoveValidatorAdnlKey) id: {} finished({}).", &validator_adnl_key_id, &status); // return Ok(status); diff --git a/src/shard_state.rs b/src/shard_state.rs index b2c1a04e..7f2ee01c 100644 --- a/src/shard_state.rs +++ b/src/shard_state.rs @@ -20,8 +20,8 @@ use adnl::{declare_counted, common::{CountedObject, Counter}}; use std::sync::atomic::Ordering; use std::{io::Write, sync::Arc}; use ton_block::{ - BlockIdExt, ShardAccount, ShardIdent, ShardStateUnsplit, ShardStateSplit, ValidatorSet, - CatchainConfig, Serializable, Deserializable, ConfigParams, McShardRecord, + BlockIdExt, ShardAccount, ShardIdent, ShardStateUnsplit, ShardStateSplit, + Serializable, Deserializable, ConfigParams, McShardRecord, McStateExtra, ShardDescr, ShardHashes, HashmapAugType, InRefValue, BinTree, BinTreeType, WorkchainDescr, OutMsgQueue, ProcessedInfo, MerkleProof, }; @@ -416,10 +416,6 @@ impl ShardStateStuff { Ok(vec) } - pub fn read_cur_validator_set_and_cc_conf(&self) -> Result<(ValidatorSet, CatchainConfig)> { - self.config_params()?.read_cur_validator_set_and_cc_conf() - } - // Unused // pub fn find_account(&self, account_id: &UInt256) -> Result> { // self diff --git a/src/tests/test_helper.rs b/src/tests/test_helper.rs index 63df3217..27515f91 100644 --- a/src/tests/test_helper.rs +++ b/src/tests/test_helper.rs @@ -791,6 +791,7 @@ impl TestEngine { self.clone(), true, true, + None, ); validator_query.try_validate().await?; diff --git a/src/validating_utils.rs b/src/validating_utils.rs index c9447f6a..74cfa299 100644 --- a/src/validating_utils.rs +++ b/src/validating_utils.rs @@ -31,6 +31,7 @@ pub fn supported_capabilities() -> u64 { GlobalCapabilities::CapReportVersion as u64 | GlobalCapabilities::CapShortDequeue as u64 | GlobalCapabilities::CapRemp as u64 | + GlobalCapabilities::CapSmft as u64 | GlobalCapabilities::CapInitCodeHash as u64 | GlobalCapabilities::CapOffHypercube as u64 | GlobalCapabilities::CapFixTupleIndexBug as u64 | @@ -57,7 +58,7 @@ pub fn supported_capabilities() -> u64 { } pub fn supported_version() -> u32 { - 49 + 50 } pub fn check_this_shard_mc_info( diff --git a/src/validator/fabric.rs b/src/validator/fabric.rs index ed4ee164..08712c81 100644 --- a/src/validator/fabric.rs +++ b/src/validator/fabric.rs @@ -18,6 +18,7 @@ use std::{ use super::validator_utils::{validator_query_candidate_to_validator_block_candidate, pairvec_to_cryptopair_vec, get_first_block_seqno_after_prevs}; use crate::{ collator_test_bundle::CollatorTestBundle, engine_traits::EngineOperations, + validator::verification::VerificationManagerPtr, validator::{CollatorSettings, validate_query::ValidateQuery, collator}, validating_utils::{fmt_next_block_descr_from_next_seqno, fmt_next_block_descr} }; use ton_block::{BlockIdExt, ShardIdent, ValidatorSet, Deserializable}; @@ -35,17 +36,37 @@ pub async fn run_validate_query_any_candidate( let prev = info.read_prev_ids()?; let mc_state = engine.load_last_applied_mc_state().await?; let min_masterchain_block_id = mc_state.find_block_id(info.min_ref_mc_seqno())?; - let (set, _) = mc_state.read_cur_validator_set_and_cc_conf()?; - run_validate_query( - shard, - SystemTime::now(), - min_masterchain_block_id, - prev, - block, - set, - engine, - SystemTime::now() - ).await + let mut cc_seqno_with_delta = 0; + if let Some(mc_state_extra) = mc_state.state()?.read_custom()? { + let cc_seqno_from_state = if shard.is_masterchain() { + mc_state_extra.validator_info.catchain_seqno + } else { + mc_state_extra.shards.calc_shard_cc_seqno(&shard)? + }; + let nodes = crate::validator::validator_utils::compute_validator_set_cc( + &*engine.load_last_applied_mc_state().await?, + &shard, + engine.now(), + cc_seqno_from_state, + &mut cc_seqno_with_delta + )?; + let validator_set = ValidatorSet::with_cc_seqno(0, 0, 0, cc_seqno_with_delta, nodes)?; + + log::debug!(target: "verificator", "ValidatorSetForVerification cc_seqno: {:?}", validator_set.cc_seqno()); + run_validate_query( + shard, + SystemTime::now(), + min_masterchain_block_id, + prev, + block, + validator_set, + engine, + SystemTime::now(), + None, //no verification manager for validations within verification + ).await + } else { + Err(failure::format_err!("MC state is None")) + } } pub async fn run_validate_query( @@ -57,6 +78,7 @@ pub async fn run_validate_query( set: ValidatorSet, engine: Arc, _timeout: SystemTime, + verification_manager: Option, ) -> Result { let next_block_descr = fmt_next_block_descr(&block.block_id); @@ -86,6 +108,7 @@ pub async fn run_validate_query( engine.clone(), false, true, + verification_manager, ).try_validate().await } else { let query = ValidateQuery::new( @@ -97,6 +120,7 @@ pub async fn run_validate_query( engine.clone(), false, true, + verification_manager, ); let validator_result = query.try_validate().await; if let Err(err) = &validator_result { diff --git a/src/validator/mod.rs b/src/validator/mod.rs index 247aeb99..7287f2f3 100644 --- a/src/validator/mod.rs +++ b/src/validator/mod.rs @@ -36,6 +36,7 @@ pub mod remp_service; pub mod telemetry; #[cfg(feature = "slashing")] pub mod slashing; +mod verification; use std::sync::Arc; use ton_types::{Result, UInt256, error}; diff --git a/src/validator/tests/test_collator.rs b/src/validator/tests/test_collator.rs index f0ac0400..c6b4fb2c 100644 --- a/src/validator/tests/test_collator.rs +++ b/src/validator/tests/test_collator.rs @@ -102,6 +102,7 @@ async fn try_collate_by_engine( engine.clone(), true, true, + None, ); validator_query.try_validate().await?; Ok((new_block, new_state)) diff --git a/src/validator/tests/test_validator_utils.rs b/src/validator/tests/test_validator_utils.rs index d0e36962..929052b7 100644 --- a/src/validator/tests/test_validator_utils.rs +++ b/src/validator/tests/test_validator_utils.rs @@ -34,6 +34,16 @@ fn block_config(block_stuff: &BlockStuff) -> Result { ) } +pub fn mine_key_for_workchain(id_opt: Option) -> (ton_types::KeyOptionJson, Arc) { + loop { + if let Ok((private, public)) = Ed25519KeyOption::generate_with_json() { + if id_opt.is_none() || Some(calc_workchain_id_by_adnl_id(public.id().data())) == id_opt { + return (private, public) + } + } + } +} + #[test] fn test_mine_key() { // let now = std::time::Instant::now(); diff --git a/src/validator/validate_query.rs b/src/validator/validate_query.rs index 6aad1339..2309db0f 100644 --- a/src/validator/validate_query.rs +++ b/src/validator/validate_query.rs @@ -27,6 +27,7 @@ use crate::{ UNREGISTERED_CHAIN_MAX_LEN, fmt_next_block_descr, }, validator::{out_msg_queue::MsgQueueManager, validator_utils::calc_subset_for_masterchain}, + validator::verification::VerificationManagerPtr, CHECK, }; @@ -222,6 +223,7 @@ pub struct ValidateQuery { block_create_count: HashMap, engine: Arc, + verification_manager: Option, next_block_descr: Arc, } @@ -239,6 +241,7 @@ impl ValidateQuery { engine: Arc, is_fake: bool, multithread: bool, + verification_manager: Option, ) -> Self { let next_block_descr = Arc::new(fmt_next_block_descr(&block_candidate.block_id)); Self { @@ -260,6 +263,7 @@ impl ValidateQuery { block_create_count: Default::default(), next_block_descr, + verification_manager, } } @@ -850,6 +854,14 @@ impl ValidateQuery { reject_query!("new shard configuration for shard {} contains different next_validator_shard {}", shard, info.descr.next_validator_shard) } + if let Some(ref verification_manager) = &self.verification_manager { + const MAX_VERIFICATION_TIMEOUT : std::time::Duration = std::time::Duration::from_millis(20); + //TODO: rewrite check_one_shard for async + if !verification_manager.wait_for_block_verification(&info.block_id, &MAX_VERIFICATION_TIMEOUT) { + //reject_query!("can't verify block {}", info.block_id) + log::warn!(target:"verificator", "can't verify block {} in MC", info.block_id); + } + } if info.descr.collators.is_some() { reject_query!("Configuration for shard {} could not contain collators", shard); } diff --git a/src/validator/validator_group.rs b/src/validator/validator_group.rs index 3e3c0d70..a379e13f 100644 --- a/src/validator/validator_group.rs +++ b/src/validator/validator_group.rs @@ -61,6 +61,8 @@ use crate::validator::validator_utils::get_first_block_seqno_after_prevs; // #[cfg(feature = "fast_finality")] // use crate::validator::workchains_fast_finality::compute_actual_finish; +use crate::validator::verification::{VerificationManagerPtr}; + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] pub enum ValidatorGroupStatus { Created, Countdown { start_at: tokio::time::Instant }, @@ -352,6 +354,7 @@ pub struct ValidatorGroup { #[cfg(feature = "slashing")] slashing_manager: SlashingManagerPtr, + verification_manager: Option, last_validation_time: AtomicU64, last_collation_time: AtomicU64, } @@ -369,6 +372,7 @@ impl ValidatorGroup { allow_unsafe_self_blocks_resync: bool, #[cfg(feature = "slashing")] slashing_manager: SlashingManagerPtr, + verification_manager: Option, ) -> Self { let group_impl = ValidatorGroupImpl::new( general_session_info.shard.clone(), @@ -393,6 +397,7 @@ impl ValidatorGroup { receiver: Arc::new(receiver), #[cfg(feature = "slashing")] slashing_manager, + verification_manager, last_validation_time: AtomicU64::new(0), last_collation_time: AtomicU64::new(0) } @@ -630,9 +635,8 @@ impl ValidatorGroup { } } - pub async fn on_generate_slot(&self, round: u32, callback: ValidatorBlockCandidateCallback) { + pub async fn on_generate_slot(&self, round: u32, callback: ValidatorBlockCandidateCallback, rt: tokio::runtime::Handle) { let next_block_descr = self.get_next_block_descr().await; - log::info!( target: "validator", "({}): ValidatorGroup::on_generate_slot: collator request, {}", @@ -687,6 +691,13 @@ impl ValidatorGroup { None => Err(error!("Min masterchain block id missing")), }; + let candidate = match self.verification_manager.clone() { + Some(_) => match &result { + Ok(candidate) => Some(candidate.clone()), + _ => None + }, + None => None, + }; let result_message = match &result { Ok(_) => { let now = std::time::SystemTime::now() @@ -739,7 +750,36 @@ impl ValidatorGroup { self.group_impl.execute_sync(|group_impl| group_impl.on_generate_slot_invoked = true).await; - callback(result) + callback(result); + + if let Some(verification_manager) = self.verification_manager.clone() { + if let Some(candidate) = candidate { + log::debug!(target:"verificator", "Received new candidate for round {} for shard {:?}", round, self.shard()); + let verification_manager = verification_manager.clone(); + let next_block_id = match self.group_impl.execute_sync(|group_impl| + group_impl.create_next_block_id( + candidate.id.root_hash.clone(), + get_hash(&candidate.data.data()), + self.shard().clone() + ) + ).await { + Err(x) => { log::error!(target: "validator", "{}", x); return }, + Ok(x) => x + }; + + let candidate = super::BlockCandidate { + block_id: next_block_id, + data: candidate.data.data().to_vec(), + collated_file_hash: candidate.collated_file_hash.clone(), + collated_data: candidate.collated_data.data().to_vec(), + created_by: self.local_key.pub_key().expect("source must contain pub_key").into(), + }; + + rt.clone().spawn(async move { + verification_manager.send_new_block_candidate(&candidate).await; + }); + } + } } // Validate_query @@ -809,6 +849,7 @@ impl ValidatorGroup { self.validator_set.clone(), self.engine.clone(), SystemTime::now() + Duration::new(10, 0), + self.verification_manager.clone(), ).await } None => Err(failure::err_msg("Min masterchain block id missing")), diff --git a/src/validator/validator_manager.rs b/src/validator/validator_manager.rs index 982709a8..0f68f2d5 100644 --- a/src/validator/validator_manager.rs +++ b/src/validator/validator_manager.rs @@ -20,6 +20,7 @@ use crate::{ remp_block_parser::{RempMasterblockObserver, find_previous_sessions}, remp_manager::{RempManager, RempInterfaceQueues}, sessions_computing::{GeneralSessionInfo, SessionValidatorsInfo, SessionValidatorsList, SessionValidatorsCache}, + fabric::run_validate_query_any_candidate, validator_group::{ValidatorGroup, ValidatorGroupStatus}, validator_utils::{ get_first_block_seqno_after_prevs, @@ -27,7 +28,8 @@ use crate::{ compute_validator_list_id, get_group_members_by_validator_descrs, is_remp_enabled, try_calc_subset_for_workchain, validatordescr_to_catchain_node, validatorset_to_string, - ValidatorListHash, ValidatorSubsetInfo + ValidatorListHash, ValidatorSubsetInfo, + try_calc_vset_for_workchain, }, out_msg_queue::OutMsgQueueInfoStuff, }, @@ -45,6 +47,10 @@ use std::{ sync::Arc, time::{Duration, SystemTime} }; +use crate::validator::verification::{VerificationFactory, VerificationListener, VerificationManagerPtr}; +use crate::validator::verification::GENERATE_MISSING_BLS_KEY; +use crate::validator::BlockCandidate; +use spin::mutex::SpinMutex; use tokio::time::timeout; use ton_api::IntoBoxed; use ton_block::{ @@ -228,7 +234,9 @@ impl ValidationStatus { struct ValidatorListStatus { known_lists: HashMap, curr: Option, - next: Option + next: Option, + curr_utime_since: Option, + next_utime_since: Option, } impl ValidatorListStatus { @@ -273,6 +281,10 @@ impl ValidatorListStatus { fn known_hashes (&self) -> HashSet { return self.known_lists.keys().cloned().collect(); } + + fn get_curr_utime_since(&self) -> &Option { + &self.curr_utime_since + } } impl Default for ValidatorListStatus { @@ -280,7 +292,9 @@ impl Default for ValidatorListStatus { return ValidatorListStatus { known_lists: HashMap::default(), curr: None, - next: None + next: None, + curr_utime_since: None, + next_utime_since: None, } } } @@ -289,6 +303,25 @@ fn rotate_all_shards(mc_state_extra: &McStateExtra) -> bool { mc_state_extra.validator_info.nx_cc_updated } +struct VerificationListenerImpl { + engine: Arc, +} + +#[async_trait::async_trait] +impl VerificationListener for VerificationListenerImpl { + async fn verify(&self, block_candidate: &BlockCandidate) -> bool { + ValidatorManagerImpl::verify_block_candidate(block_candidate, self.engine.clone()).await + } +} + +impl VerificationListenerImpl { + fn new(engine: Arc) -> Self { + Self { + engine + } + } +} + struct ValidatorManagerImpl { engine: Arc, rt: tokio::runtime::Handle, @@ -303,6 +336,8 @@ struct ValidatorManagerImpl { #[cfg(feature = "slashing")] slashing_manager: SlashingManagerPtr, + verification_manager: SpinMutex>, + verification_listener: SpinMutex>>, } impl ValidatorManagerImpl { @@ -321,8 +356,8 @@ impl ValidatorManagerImpl { engine.set_validation_status(ValidationStatus::Disabled); (ValidatorManagerImpl { - engine, - rt, + engine: engine.clone(), + rt: rt.clone(), validator_sessions: HashMap::default(), validator_list_status: ValidatorListStatus::default(), session_info_cache: SessionValidatorsCache::new(), @@ -332,9 +367,51 @@ impl ValidatorManagerImpl { block_observer: None, #[cfg(feature = "slashing")] slashing_manager: SlashingManager::create(), + verification_manager: SpinMutex::new(None), + verification_listener: SpinMutex::new(None), }, remp_interface_queues) } + /// Create/destroy verification manager + fn update_verification_manager_usage(&self, mc_state_extra: &McStateExtra) { + let smft_enabled = !self.config.smft_disabled && crate::validator::validator_utils::is_smft_enabled(self.engine.clone(), mc_state_extra.config()); + let verification_manager = self.verification_manager.lock().clone(); + let has_verification_manager = verification_manager.is_some(); + + if smft_enabled && has_verification_manager { + return; //verification manager is enabled and created + } + if !smft_enabled && !has_verification_manager { + return; //verification manager is disabled and destroyed + } + + if !smft_enabled && has_verification_manager { + //SMFT is disabled, but verification manager is created + log::info!(target: "verificator", "Destroy verification manager for validator manager"); + + *self.verification_manager.lock() = None; + *self.verification_listener.lock() = None; + + return; + } + + if smft_enabled && !has_verification_manager { + //SMFT is enabled, but verification manager has not been created + + log::info!(target: "verificator", "Initialize verification manager for validator manager"); + + let verification_manager = VerificationFactory::create_manager(self.engine.clone(), self.rt.clone()); + let verification_listener = Arc::new(VerificationListenerImpl::new(self.engine.clone())); + + *self.verification_manager.lock() = Some(verification_manager); + *self.verification_listener.lock() = Some(verification_listener); + + log::info!(target: "verificator", "Verification manager for validator manager has been initialized"); + + return; + } + } + /// find own key in validator subset fn find_us(&self, validators: &[ValidatorDescr]) -> Option { if let Some(lk) = self.validator_list_status.get_local_key() { @@ -397,10 +474,12 @@ impl ValidatorManagerImpl { }; self.validator_list_status.curr = self.update_single_validator_list(validator_set.list(), "current").await?; + self.validator_list_status.curr_utime_since = Some(validator_set.utime_since()); if let Some(id) = self.validator_list_status.curr.as_ref() { self.engine.activate_validator_list(id.clone())?; } self.validator_list_status.next = self.update_single_validator_list(next_validator_set.list(), "next").await?; + self.validator_list_status.next_utime_since = Some(next_validator_set.utime_since()); metrics::gauge!("in_current_vset_p34", if self.validator_list_status.curr.is_some() { 1 } else { 0 } as f64); metrics::gauge!("in_next_vset_p36", if self.validator_list_status.next.is_some() { 1 } else { 0 } as f64); @@ -891,6 +970,7 @@ impl ValidatorManagerImpl { #[cfg(feature = "slashing")] let slashing_manager = self.slashing_manager.clone(); let remp_manager = self.remp_manager.clone(); + let verification_manager = self.verification_manager.lock().clone(); let allow_unsafe_self_blocks_resync = self.config.unsafe_resync_catchains.contains(&cc_seqno); let session = self.validator_sessions.entry(session_id.clone()).or_insert_with(|| Arc::new(ValidatorGroup::new( @@ -905,6 +985,7 @@ impl ValidatorManagerImpl { allow_unsafe_self_blocks_resync, #[cfg(feature = "slashing")] slashing_manager, + verification_manager, )) ); @@ -970,6 +1051,9 @@ impl ValidatorManagerImpl { ); } + // update verification manager (create / destroy depends on SMFT capabilities) + self.update_verification_manager_usage(&mc_state_extra); + let master_cc_seqno = get_masterchain_seqno(self.engine.clone(), &mc_state).await?; if let Some(remp) = &self.remp_manager { if last_masterchain_block.seq_no == 0 { @@ -1095,6 +1179,9 @@ impl ValidatorManagerImpl { let next_validator_set = mc_state_extra.config.next_validator_set()?; let full_validator_set = mc_state_extra.config.validator_set()?; let possible_validator_change = next_validator_set.total() > 0; + let mut mc_validators = Vec::new(); + + mc_validators.reserve(full_validator_set.total() as usize); for ident in future_shards.iter() { log::trace!(target: "validator_manager", "Future shard {}", ident); @@ -1190,6 +1277,10 @@ impl ValidatorManagerImpl { // Iterate over future shards and create all future sessions for (ident, (wc, next_cc_seqno, next_val_list_id)) in our_future_shards.iter() { + if ident.is_masterchain() { + mc_validators.append(&mut wc.validators.clone()); + } + if let Some(local_id) = self.find_us(&wc.validators) { let new_session_info = Arc::new(GeneralSessionInfo { shard: ident.clone(), @@ -1198,6 +1289,7 @@ impl ValidatorManagerImpl { key_seqno: keyblock_seqno, max_vertical_seqno: 0 }); + let session_id = get_session_id( new_session_info.clone(), &wc.validators, @@ -1205,6 +1297,7 @@ impl ValidatorManagerImpl { ); gc_validator_sessions.remove(&session_id); if !self.validator_sessions.contains_key(&session_id) { + let verification_manager = self.verification_manager.lock().clone(); let session = Arc::new(ValidatorGroup::new( new_session_info, local_id, @@ -1216,7 +1309,8 @@ impl ValidatorManagerImpl { self.engine.clone(), self.config.unsafe_resync_catchains.contains(next_cc_seqno), #[cfg(feature = "slashing")] - self.slashing_manager.clone() + self.slashing_manager.clone(), + verification_manager, )); self.validator_sessions.insert(session_id, session); } @@ -1227,7 +1321,9 @@ impl ValidatorManagerImpl { for (_, session) in &self.validator_sessions { for id in &blocks_before_split { if id.shard().is_parent_for(session.shard()) { - log::trace!(target: "validator_manager", "precalc_split_queues_for {}", id); + log::trace!( + target: "validator_manager", "precalc_split_queues_for {}", id + ); precalc_split_queues_for.insert(id.clone()); } } @@ -1237,15 +1333,112 @@ impl ValidatorManagerImpl { for id in precalc_split_queues_for { let engine = self.engine.clone(); tokio::spawn(async move { - log::trace!(target: "validator_manager", "Split queues precalculating for {}", id); + log::trace!( + target: "validator_manager", "Split queues precalculating for {}", id + ); match OutMsgQueueInfoStuff::precalc_split_queues(&engine, &id).await { - Ok(_) => log::trace!(target: "validator_manager", "Split queues precalculated for {}", id), - Err(e) => log::error!(target: "validator_manager", "Can't precalculate split queues for {}: {}", id, e) + Ok(_) => log::trace!( + target: "validator_manager", "Split queues precalculated for {}", id + ), + Err(e) => log::error!( + target: "validator_manager", + "Can't precalculate split queues for {}: {}", id, e + ) } }); } } + let verification_manager = self.verification_manager.lock().clone(); + if let Some(verification_manager) = verification_manager { + if let Some(verification_listener) = self.verification_listener.lock().clone() { + let config = &mc_state_extra.config; + let mut are_workchains_updated = false; + match try_calc_vset_for_workchain( + &full_validator_set, config, &catchain_config, workchain_id + ) { + Ok(workchain_validators) => { + let found_in_wc = self.find_us(&workchain_validators).is_some(); + let found_in_mc = self.find_us(&mc_validators).is_some(); + let verification_listener: Arc = + verification_listener.clone(); + let verification_listener = Arc::downgrade(&verification_listener); + if let Some(local_key) = self.validator_list_status.get_local_key() { + let local_key_id = local_key.id(); + + if let Some(utime_since) = + self.validator_list_status.get_curr_utime_since() + { + log::trace!(target: "verificator", "Request BLS key"); + let mut local_bls_key = + self.engine.get_validator_bls_key(local_key_id).await; + log::trace!(target: "verificator", "Request BLS key done"); + if local_bls_key.is_none() && GENERATE_MISSING_BLS_KEY { + match VerificationFactory::generate_test_bls_key(&local_key) { + Ok(bls_key) => local_bls_key = Some(bls_key), + Err(err) => log::error!( + target: "verificator", + "Can't generate test BLS key: {:?}", err + ), + } + } + + match local_bls_key { + Some(local_bls_key) => { + if found_in_wc || found_in_mc { + log::debug!( + target: "verificator", "Update workchains start" + ); + verification_manager.update_workchains( + local_key_id.clone(), + local_bls_key, + workchain_id, + *utime_since, + &workchain_validators, + &mc_validators, + &verification_listener + ).await; + are_workchains_updated = true; + log::debug!( + target: "verificator", "Update workchains finish" + ); + } else { + log::debug!( + target: "verificator", + "Skip workchains update because we are not present \ + in WC/MC sets for workchain_id={}", + workchain_id + ); + } + }, + None => log::error!( + target: "verificator", + "Can't create verification workchains: \ + no BLS private key attached" + ), + } + } else { + log::warn!( + target: "verificator", "Validator curr_utime_since is not set" + ) + } + } else { + log::warn!(target: "verificator", "Validator local key is not found"); + } + } + Err(err) => { + log::error!(target: "validator", "Can't create verification workchains: {:?}", err); + } + } + + if !are_workchains_updated { + log::debug!(target: "verificator", "Reset workchains start"); + verification_manager.reset_workchains().await; + log::debug!(target: "verificator", "Reset workchains finish"); + } + } + } + if rotate_all_shards(&mc_state_extra) { log::info!(target: "validator_manager", "New last rotation block: {}", last_masterchain_block); self.engine.save_last_rotation_block_id(last_masterchain_block)?; @@ -1478,6 +1671,20 @@ impl ValidatorManagerImpl { log::info!(target: "validator_manager", "Engine is stopped. Exiting from invocation loop (while applying state)"); Ok(()) } + + async fn verify_block_candidate(block_candidate: &BlockCandidate, engine: Arc) -> bool { + let candidate_id = block_candidate.block_id.clone(); + + log::debug!(target: "verificator", "Verifying block candidate {:?}", candidate_id); + + match run_validate_query_any_candidate(block_candidate.clone(), engine).await { + Ok(_time) => return true, + Err(err) => { + log::warn!(target: "verificator", "Block {:?} verification error: {:?}", candidate_id, err); + return false; + } + } + } } /// main entry point to validation process diff --git a/src/validator/validator_session_listener.rs b/src/validator/validator_session_listener.rs index 56d74505..fdeff1e3 100644 --- a/src/validator/validator_session_listener.rs +++ b/src/validator/validator_session_listener.rs @@ -218,12 +218,12 @@ impl CatchainReplayListener for ValidatorSessionListener { } } -async fn process_validation_action (action: ValidationAction, g: Arc) { +async fn process_validation_action (action: ValidationAction, g: Arc, rt: tokio::runtime::Handle) { let action_str = format!("{}", action); let next_block_descr = g.get_next_block_descr().await; log::info!(target: "validator", "({}): Processing action: {}, {}", next_block_descr, action_str, g.info().await); match action { - ValidationAction::OnGenerateSlot {round, callback} => g.on_generate_slot (round, callback).await, + ValidationAction::OnGenerateSlot {round, callback} => g.on_generate_slot (round, callback, rt).await, ValidationAction::OnCandidate {round, source, root_hash, data, collated_data, callback} => g.on_candidate (round, source, root_hash, data, collated_data, callback).await, @@ -316,8 +316,9 @@ pub async fn process_validation_queue( } let start_time = SystemTime::now(); + let rt_clone = rt.clone(); let mut join_handle = rt.spawn(async move { - process_validation_action (action, g_clone).await; + process_validation_action (action, g_clone, rt_clone).await; }); loop { diff --git a/src/validator/validator_utils.rs b/src/validator/validator_utils.rs index 4d0dfbec..4a02c914 100644 --- a/src/validator/validator_utils.rs +++ b/src/validator/validator_utils.rs @@ -20,12 +20,13 @@ use catchain::{BlockPayloadPtr, CatchainNode, PublicKey, PublicKeyHash}; use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; use ton_api::ton::engine::validator::validator::groupmember::GroupMember; use ton_block::{ - BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, ConfigParams, CryptoSignature, - CryptoSignaturePair, Deserializable, GlobalCapabilities, Message, Serializable, - ShardIdent, SigPubKey, UnixTime32, ValidatorBaseInfo, ValidatorDescr, ValidatorSet + BlockIdExt, BlockInfo, BlockSignatures, BlockSignaturesPure, CatchainConfig, ConfigParams, + CryptoSignature, CryptoSignaturePair, Deserializable, GlobalCapabilities, Message, + Serializable, ShardIdent, SigPubKey, UnixTime32, ValidatorBaseInfo, ValidatorDescr, + ValidatorSet, Workchains, WorkchainDescr }; use ton_types::{ - error, fail, BuilderData, HashmapType, Ed25519KeyOption, KeyId, KeyOption, KeyOptionJson, + error, fail, BuilderData, HashmapType, Ed25519KeyOption, KeyId, Result, Sha256, UInt256 }; use validator_session::SessionNode; @@ -324,6 +325,49 @@ pub fn try_calc_subset_for_workchain_standard( } } +lazy_static::lazy_static! { + static ref SINGLE_WORKCHAIN: Workchains = { + let mut workchains = Workchains::default(); + workchains.set(&0, &WorkchainDescr::default()).unwrap(); + workchains + }; +} + +pub fn try_calc_vset_for_workchain( + vset: &ValidatorSet, + config: &ConfigParams, + cc_config: &CatchainConfig, + workchain_id: i32, +) -> Result> { + let full_list = if cc_config.isolate_mc_validators { + if vset.total() <= vset.main() { + failure::bail!("Count of validators is too small to make sharde's subset while `isolate_mc_validators` flag is set"); + } + let list = vset.list()[vset.main() as usize .. ].to_vec(); + list + } else { + vset.list().to_vec().clone() + }; + // in case on old block proof it doesn't contain workchains in config so 1 by default + let workchains = config.workchains().unwrap_or_else(|_| SINGLE_WORKCHAIN.clone()); + match workchains.len()? as i32 { + 0 => failure::bail!("workchain description is empty"), + 1 => { Ok(full_list) }, + count => { + let mut list = Vec::new(); + + for descr in vset.list() { + let id = calc_workchain_id(descr); + if (id == workchain_id) || (id >= count) { + list.push(descr.clone()); + } + } + + Ok(list) + } + } +} + pub fn try_calc_subset_for_workchain( vset: &ValidatorSet, mc_state: &ShardStateStuff, @@ -373,16 +417,6 @@ pub fn calc_subset_for_workchain_standard( } } -pub fn mine_key_for_workchain(id_opt: Option) -> (KeyOptionJson, Arc) { - loop { - if let Ok((private, public)) = Ed25519KeyOption::generate_with_json() { - if id_opt.is_none() || Some(calc_workchain_id_by_adnl_id(public.id().data())) == id_opt { - return (private, public) - } - } - } -} - pub async fn get_shard_by_message(engine: Arc, message: Arc) -> Result { let dst_wc = message.dst_workchain_id() .ok_or_else(|| error!("Can't get workchain id from message"))?; @@ -414,6 +448,10 @@ pub fn is_remp_enabled(_engine: Arc, config_params: &Confi return config_params.has_capability(GlobalCapabilities::CapRemp); } +pub fn is_smft_enabled(_engine: Arc, config_params: &ConfigParams) -> bool { + return config_params.has_capability(GlobalCapabilities::CapSmft); +} + pub fn get_message_uid(msg: &Message) -> UInt256 { match msg.body() { Some(slice) => slice.into_cell().repr_hash(), diff --git a/src/validator/verification/README.md b/src/validator/verification/README.md new file mode 100644 index 00000000..e96982b4 --- /dev/null +++ b/src/validator/verification/README.md @@ -0,0 +1,201 @@ +# 1 Overview + +This document describes Accelerated SMFT protocol which is designed to decrease EverScale blockchain finality time due to shard-chains consensus process change. This protocol removes BFT consensus from shard-validator sessions and proposes to use rarely changing collators to speed up collation process and soft-majority voting for consensus in shards. Master-chain consensus remains unchanged with responsibilities for forks resolution and shard-sessions configuration selection (collator, validators). + +# 2 Requirements + +The list below represents the requirements which Acelerated SMFT protocol should satisfy: +- multiple block candidate validations by several validator nodes (verificators) +- deterministic verificators choosing for further slashing algorithms implementations +- unpredictability of choice of the verificator by whole workchain except of verificator itself during the block candidate validation stage +- single node collations of block candidates during validation session +- validation session collator switch in case of collator malfunction + +# 3 Specification + +## 3.1 Primitives + +### 3.1.1 BlockCandidateBroadcast + +This broadcasts is sent by collator to all validators in workchain with collated data of new block candidate. + +```rust +pub struct BlockCandidateBroadcast { + pub id: crate::ton::ton_node::blockidext::BlockIdExt, + pub data: crate::ton::bytes, + pub collated_data: crate::ton::bytes, + pub collated_data_file_hash: crate::ton::int256, + pub created_by: crate::ton::int256, + pub created_timestamp: crate::ton::long, +} +``` + +### 3.1.2 BlockCandidateStatus + +This message is used to interact between validators to synchronize status of block-candidate processing. Block is identified by ```candidate_id``` field. Signature fields (```deliveries_signature```, ```rejections_signature```, ```approvals_signature``` and ```timeouts_signature```) contain corresponding BLS signatures for different stages of block processing. + +```rust +pub struct BlockCandidateStatus { + pub candidate_id: crate::ton::int256, + pub deliveries_signature: crate::ton::bytes, + pub approvals_signature: crate::ton::bytes, + pub rejections_signature: crate::ton::bytes, + pub timeouts_signature: crate::ton::bytes, + pub merges_cnt: crate::ton::int, + pub created_timestamp: crate::ton::long, +} +``` + +### 3.1.3 BlockCandidateArbitrage + +This message is used to initiate arbitrage process in case of received rejections (NACK). Master-chain validator sends ```BlockCandiateArbitrage``` message to several workchain validators and waits for ```BlockCandidateStatus``` message back from them as a response. + +```rust +pub struct BlockCandidateArbitrage { + pub candidate_id: crate::ton::int256, + pub created_timestamp: crate::ton::long, +} +``` + +### 3.1.3 ShardCollatorBlame + +This message is sent by workchain validators to workchain to identify collator work malfunction. The message may be sent in case of collation timeout and in case of malicious block-candidate generation. This message is also used in consensus process between master-chain validators to vote for collator blame and rotation. + +```rust +pub struct ShardCollatorBlame { + pub validator_session: crate::ton::int256, + pub collator_pub_key: crate::ton::int256, + pub timeouts_blame_signature: crate::ton::bytes, + pub correctness_blame_signature: crate::ton::bytes, +} +``` + +## 3.2 Collator Choosing + +To speed up collation process and decrease latencies collation rotation period is increased up to validator session time. So during the validation session there is no collator change. Collator is choosen for shard-blocks interval which is specified in master-chain block. So it is always possible to find out collator which corresponds to each particular shard-block. + +Collator change is done by master-chain validators in one of following cases: +- master-chain approved shard-collator validity interval is expired (regular case) +- master-chain approved shard-collator timeout blame +- master-chain approved shard-collator validity blame + +**Regular Case** + +Usually shard-collator is selected for some shard-blocks interval. This interval is written in master-chain block. So each node in network may deterministically detect a collator for each block in each shard. + +Shard-collators change is a deterministic process which is initiated by master-chain collator and validated by master-chain validators. So master-chain collator proposes new collator for shard-chain using deterministic pseudo RNG scheme. This selection leads to new validator session creation with such collator as a lead in a shard. + +**Shard-collator timeout blame** + +Each workchain validator monitors shards health based on timeouts between shard blocks. If such timeout for particular shard is more than configurable timeout shreshold, workchain validator initiates timeout blame via ```ShardCollatorBlame``` message: +- workchain validator signs ```ShardCollatorBlame::timeouts_blame_signature``` with its BLS private key +- workchain validator sends ```ShardCollatorBlame``` message to all master-chain validators +- master-chain collator checks the blame and if ```ShardCollatorBlame::timeouts_blame_signature``` is more then 1/2 of all workchain voting weight, changes shard-collator for compromised shard in next master-chain block candidate; as a proof of change master-chain writes received ```ShardCollatorBlame``` to a new master block-candidate which allow further validation of shard collator rotation by other master-chain validators + +**Shard-collator validity blame** + +Validity blame may be initiated by master-chain validator in following cases: +- incoming shard-block header is malicious (for example, self-fork by workchain collator) +- shard-block was marked as malicios during the arbitrage process + +Validity blame process is following: +- master-chain validator signs ```ShardCollatorBlame::validity_blame_signature``` with its BLS private key +- master-chain validator broadcasts ```ShardCollatorBlame``` message via private master-chain overlay +- master-chain collator checks received aggregated shard validator-sessions blames from other master-chain validators and in case of blame threshold (1/2 of master-chain total weight) changes shard-collator for compromised shard in next master-chain block candidate +- each master-chain validator can check ```ShardCollatorBlame::validity_blame_signature``` BLS signature during master-chain block-candidate validation to validate shard-collator switch +- validity blame signature with corresponding block information is stored in master-chain for further slashing + +## 3.3 SMFT Protocol + +## 3.3.1 Protocol Stages + +SMFT protocol processes each new block-candidate from generation by shard-collator to inclusion to master-chain block. Basically protocol may be split to two big stages: +- Workchain Block Processing Stage +- Masterchain Block Processing Stage + +## 3.3.2 Workchain Block Processing Stage + +Block-candidate processing within workchain has following flow: +- **Block candidate collation**: + - block-candidate is generated by active shard collator + - shard collator broadcasts the block-candidate using ```BlockCandidateBroadcast``` message among neighbors +- **Block candidate body delivery**. When node receives ```BlockCandidateBroadcast``` it has to: + - update corresponding ```BlockCandidateStatus``` record by adding node's BLS signature ```BlockCandidateStatus::deliveries_signature``` field to indicate the block has been successfully delivered; then initiate the ```BlockCandidateStatus``` status delivery among neighbours + - check if the node has to verify this corresponding block-candidate based on private BLS node's key and block-candidate's hash + - if the node has to verify block-candidate, it starts validation process with one of following results + - validation passes: the node adds its signature to the corresponding ```BlockCandidateStatus::approvals_signature``` record in local database and sends updated status to all masterchain validators + - validation fails: the node adds its signature to the corresponding ```BlockCandidateStatus::rejections_signature``` record in local database and sends updated status to all masterchain validators + - validation timeout: the node adds its signature to the corresponding ```BlockCandidateStatus::timeouts_signature``` record in local database and sends updated status to all masterchain validators + - **Block candidate status delivery**: + - each workchain node aggregates block-candidate status information in internal database of ```BlockCandidateStatus``` records + - when node receives ```BlockCandidateStatus``` message it has to: + - check it correctness; cancel all further processing in case of malfunction message received + - merge received status with corresponding ```BlockCandidateStatus``` in local node's database + - in case of any BLS signatures change after merge save updated status and initiate its delivery among neighbours + - if accumulated weight of all validators which received block (according to ```BlockCandidateStatus::deliveries_signature``` field) is more than threshold, the node sends updated status to all masterchain validators + +## 3.3.3 Masterchain Block Processing Stage + +Block-candidate processing within workchain has following flow: +- **BlockCandidateStatus Processing**: + - when master-chain validator receives ```BlockCandidateStatus``` it checks it correctness + - merges received ```BlockCandidateStatus``` message with corresponding record from local database and stores it +- **Shard-block Header Validation**: + - each master-chain validator checks correctness of received shard-block header before adding it to master-chain + - if block-check fails, whole validation session is marked blamed by this master-chain validator and validator session rotation process is initiated by sending ```ShardCollatorBlame``` messsage to other master-chain validators + - if block-check passes: + - if corresponding ```BlockCandidateStatus``` record is absent in a local master-chain node's database, wait for it during block-chain configurable timeout; otherwise: + - if the accumulated weight of validators which received block-candidate according to ```BlockCandidateStatus::deliveries_signature``` is less than approval threshold, wait for ```BlockCandidateStatus``` update + - if corresponding ```BlockCandidateStatus::rejections_signature``` is not empty, + start validation arbitrage process + - if rejections wait timeout is expired but ```BlockCandidateStatus::deliveries_signature``` is more then threshold (1/2 of total workchain's weight) add shard-block header to master-chain +- **Validation Arbitrage Process**. The validation arbitrage process flow may be initiated by any master-chain validator to find consensus on particular shard-block correctness: +- master-chain validator sends to random workchain validators ```BlockCandidateArbitrage``` message; accumulated weight of such validators subset has to be not less than configurable threshold +- each workchain validator which received ```BlockCandidateArbitrage``` has to validate corresponding block-candidate and update ```BlockCandidateStatus::approvals_signature``` or ```BlockCandidateStatus::rejections_signature``` fields in corresponding status record in local database, then updated ```BlockCandidateStatus``` has to be sent back to master-chain validator +- master-chain validator merges received from workchain validators statuses until ```BlockCandidateStatus::approvals_signature``` or ```BlockCandidateStatus::rejections_signature``` becomes not less than configurable threshold + - if according to the workchain nodes consensus block is rejected, the validator session is marked as blamed (so all futher blocks from it won't be accepted) and validator session rotation process is initiated by sending ```ShardCollatorBlame``` messsage to other master-chain validators + - otherwise blame all validators from initial for arbitrage ```BlockCandidateStatus::rejections_signature``` field and continue interrupted flow process + +# 4 Runtime metrics + +All SMFT metrics are configured to be sent via metrics crate (metrics::gauge! macro) for further tracking. The main metrics are listed below. Note, some metrics are computer for specific workchain NNN. The placeholder NNN should be replaced with specific workchain number (for example, NNN=0). +- smft_block - number of blocks stored by SMFT internal database (.create - number of created, .drop - number of dropped, w/0 suffix - remaining count) +- smft_candidates - number of block candidates received by SMFT for further processing +- smft_mcNNN_overlay - number of nodes in MC overlay which are visible by this node +- smft_mcNNN_overlay_in_messages - number of messages which are received by this node in MC overlay +- smft_mcNNN_overlay_in_queries - number of queries which are received by this node in MC overlay +- smft_mcNNN_overlay_out_messages - number of messages which are sent by this node to MC overlay +- smft_mcNNN_overlay_out_queries - number of queries which are sent by this node to MC overlay (also change speed for this parameter is computed with suffix '.speed') +- smft_mc_overlays - number of overlays for communication with MC +- smft_wcNNN_block_candidate_processings - number of processed block candidates in workchain +- smft_wcNNN_block_candidate_verifications - numbfer of verifications initiated for block candidates in WC overlay (with linked metrics '.failure' - failed verifications, '.success' - approved verifications, '.total' - all verification requests) +- smft_wcNNN_block_status_merges - number of merges for block candidate statuses for this workchain +- smft_wcNNN_block_status_merges_count - histogram for merges count (with suffixes '.min', '.max', '.med') +- smft_wcNNN_block_status_processings - number of block status processings in workchain +- smft_wcNNN_block_status_received_in_mc - number of received block statuses in MC +- smft_wcNNN_block_status_sets - number of block status modifications +- smft_wcNNN_block_status_to_mc_sends - number of sendings from this node to MC overlay of block statuses +- smft_wcNNN_block_status_within_wc_sends - number of sendings of block statuses from this node to other workchain nodes +- smft_wcNNN_in_block_candidates - number of incoming block candidates (from other workchain nodes) +- smft_wcNNN_mc_approved - number of ACKs for blocks requested by MC (with derivative metric '.frequency') +- smft_wcNNN_mc_delivered - number of delivered and requested block statuses to MC +- smft_wcNNN_mc_rejected - number of NACKs for blocks requested by MC (with derivative metric '.frequency') +- smft_wcNNN_mc_requests - number of blocks requested by MC +- smft_wcNNN_mc_sends_per_block_candidate - average number of block statuses sends per one block +- smft_wcNNN_merges_per_block - average number of merges per block +- smft_wcNNN_new_block_candidates - number of new block candidates which are sent by this node +- smft_wcNNN_overlay_in_broadcasts - number of incoming broadcasts in workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_in_messages - number of incoming messages in workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_in_queries - number of incoming queries in workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_out_broadcasts - number of outgoing broadcasts to workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_out_messages - number of outgoing messages to workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_out_queries - number of outgoing queries to workchain overlay (with derivative metric '.speed') +- smft_wcNNN_overlay_send_message_to_neighbours_calls - number of block status sends to neighbour nodes +- smft_wcNNN_updates_per_mc_send - average number of block status updates per one block status send to MC +- smft_wc_overlays - number of workchain overlays +- smft_workchains - number of workchains +- smft_workchains_updates - number of workchain configuration update requests +- smft_wcNNN_block_mc_delay - histogram in milliseconds for delay between MC request and SMFT delivery response (with suffixes '.min', '.max', '.med') +- smft_wcNNN_stage1_block_candidate_delivered_in_wc - histogram in milliseconds for latency between block candidate collation and receive on workchain node (with suffixes '.min', '.max', '.med') +- smft_wcNNN_stage2_block_status_send_to_mc_latency - histogram in milliseconds for latency between block candidate collation and corresponding block status send from this node to MC validators (with suffixes '.min', '.max', '.med') +- smft_wcNNN_stage3_block_status_received_in_mc_latency - histogram in milliseconds for latency between block candidate collation and corresponding block status receive on MC node (with suffixes '.min', '.max', '.med') diff --git a/src/validator/verification/block.rs b/src/validator/verification/block.rs new file mode 100644 index 00000000..b586f8d9 --- /dev/null +++ b/src/validator/verification/block.rs @@ -0,0 +1,417 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +//pub use multi_signature_unsafe::MultiSignature; +pub use multi_signature_bls::MultiSignature; +use super::*; +use catchain::BlockPayloadPtr; +use log::*; +use catchain::profiling::InstanceCounter; +use spin::mutex::SpinMutex; +use ton_api::ton::ton_node::blockcandidatestatus::BlockCandidateStatus; +use ton_api::ton::ton_node::broadcast::BlockCandidateBroadcast; +use ton_api::IntoBoxed; +use ton_types::Result; +use validator_session::ValidatorWeight; +use catchain::serialize_tl_boxed_object; + +//TODO: merges count + +/* +=============================================================================== + Block +=============================================================================== +*/ + +#[derive(Debug)] +pub struct BlockCandidateBody { + candidate: BlockCandidateBroadcast, //candidate + serialized_candidate: BlockPayloadPtr, //serialized candidate + hash: UInt256, //hash of this candiate +} + +impl BlockCandidateBody { + /// Access to broadcast + pub fn candidate(&self) -> &BlockCandidateBroadcast { + &self.candidate + } + + /// Access to serialized broadcast + pub fn serialized_candidate(&self) -> &BlockPayloadPtr { + &self.serialized_candidate + } + + /// Constructor + pub fn new(candidate: BlockCandidateBroadcast) -> Self { + //todo: optimize clone + let serialized_candidate = catchain::CatchainFactory::create_block_payload( + serialize_tl_boxed_object!(&candidate.clone().into_boxed()), + ); + + Self { + serialized_candidate: serialized_candidate.clone(), + candidate, + hash: catchain::utils::get_hash_from_block_payload(&serialized_candidate), + } + } +} + +pub struct Block { + block_candidate: Option>, //block candidate + serialized_block_status: Option, //serialized status + candidate_id: UInt256, //block ID + deliveries_signature: MultiSignature, //signature for deliveries + approvals_signature: MultiSignature, //signature for approvals + rejections_signature: MultiSignature, //signature for rejections + signatures_hash: u32, //hash for signatures + created_timestamp: Option, //time of block creation + first_appearance_time: std::time::SystemTime, //time of first block appearance in a network + delivery_state_change_time: Option, //time when block is delivered + merges_count: u32, //merges count + initially_mc_processed: bool, //was this block process in MC + mc_originated: bool, //was this block appeared from MC + mc_delivered: bool, //was this block delivered to MC because of cutoff weight of delivery signatures + ready_for_send: bool, //is this block ready for sending + _instance_counter: InstanceCounter, //instance counter + first_external_request_time: Option, //time of first external request +} + +pub type BlockPtr = Arc>; + +impl std::fmt::Debug for Block { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Block[candidate_id={:?}, deliveries={:?}, approvals={:?}, rejections={:?}, signatures_hash={}]", self.candidate_id, self.deliveries_signature, self.approvals_signature, self.rejections_signature, self.signatures_hash) + } +} + +impl Block { + /// Candidate ID + pub fn get_id(&self) -> &UInt256 { + &self.candidate_id + } + + /// Is block delivered + pub fn is_delivered( + &self, + validators: &Vec, + cutoff_weight: ValidatorWeight, + ) -> bool { + let delivered_weight = self.deliveries_signature.get_total_weight(validators); + + delivered_weight >= cutoff_weight + } + + /// Is block rejected + pub fn is_rejected(&self) -> bool { + !self.rejections_signature.empty() + } + + /// Does block have approves + pub fn has_approves(&self) -> bool { + !self.approvals_signature.empty() + } + + /// Block first appearance time + pub fn get_first_appearance_time(&self) -> &std::time::SystemTime { + &self.first_appearance_time + } + + /// Block creation time + pub fn get_creation_time(&self) -> Option { + match self.created_timestamp { + Some(value) => Some(std::time::UNIX_EPOCH + std::time::Duration::from_millis(value as u64)), + None => None + } + } + + /// Get block delivery latency + pub fn get_delivery_latency(&self) -> Option { + match self.get_creation_time() { + Some(value) => match value.elapsed() { + Ok(elapsed) => Some(elapsed), + Err(_) => None, + }, + None => None, + } + } + + /// Set block creation time + pub fn set_creation_timestamp(&mut self, created_timestamp: i64) { + if created_timestamp == 0 { + return; + } + + match self.created_timestamp { + None => self.created_timestamp = Some(created_timestamp), + Some(value) => { + if value > created_timestamp { + self.created_timestamp = Some(created_timestamp); + } + } + } + } + + /// Get first external request time + pub fn get_first_external_request_time(&self) -> &Option { + &self.first_external_request_time + } + + /// Set first external request time + pub fn set_first_external_request_time(&mut self, request_time: &std::time::SystemTime) { + if let Some(first_external_request_time) = self.first_external_request_time { + if first_external_request_time < *request_time { + return; + } + } + + self.first_external_request_time = Some(*request_time); + } + + /// Get time when block becomes delivered + pub fn get_delivery_state_change_time(&self) -> &Option { + &self.delivery_state_change_time + } + + /// Set delivery state change time + pub fn set_delivery_state_change_time(&mut self, time: &std::time::SystemTime) { + if let Some(delivery_state_change_time) = self.delivery_state_change_time { + if delivery_state_change_time < *time { + return; + } + } + + self.delivery_state_change_time = Some(*time); + } + + /// Set MC processed status + pub fn mark_as_mc_processed(&mut self) { + self.initially_mc_processed = true; + } + + /// Get MC processed status + pub fn was_mc_processed(&self) -> bool { + self.initially_mc_processed + } + + /// Set MC delivery status + pub fn mark_as_mc_delivered(&mut self) { + self.mc_delivered = true; + } + + /// Get MC delivery status + pub fn was_mc_delivered(&self) -> bool { + self.mc_delivered + } + + /// Set origin + pub fn mark_as_mc_originated(&mut self) { + self.mc_originated = true; + } + + /// Get origin + pub fn is_mc_originated(&self) -> bool { + self.mc_originated + } + + /// Set ready for sending flag + pub fn toggle_send_ready(&mut self, new_state: bool) -> bool { + let prev_state = self.ready_for_send; + + self.ready_for_send = new_state; + + prev_state + } + + /// Get status + pub fn status(&self) -> BlockCandidateStatus { + BlockCandidateStatus { + candidate_id: self.candidate_id.clone(), + deliveries_signature: self.deliveries_signature.serialize().into(), + approvals_signature: self.approvals_signature.serialize().into(), + rejections_signature: self.rejections_signature.serialize().into(), + merges_cnt: (self.merges_count + 1) as i32, //increase number of merges before send + created_timestamp: match self.created_timestamp { Some (value) => value, None => 0 }, + } + } + + /// Serialized TON block + pub fn serialize(&mut self) -> BlockPayloadPtr { + if let Some(serialized_block_status) = &self.serialized_block_status { + return serialized_block_status.clone(); + } + + let ton_block = self.status().into_boxed(); + let serialized_block_status = serialize_tl_boxed_object!(&ton_block); + let serialized_block_status = + catchain::CatchainFactory::create_block_payload(serialized_block_status); + + self.serialized_block_status = Some(serialized_block_status.clone()); + + serialized_block_status + } + + /// Update status of the block (returns true, if the block should be updated in the network) + pub fn set_status(&mut self, local_key: &PrivateKey, local_idx: u16, nodes_count: u16, status: Option) -> Result { + let prev_hash = self.get_signatures_hash(); + + let mut new_deliveries_signature = self.deliveries_signature.clone(); + let mut new_approvals_signature = self.approvals_signature.clone(); + let mut new_rejections_signature = self.rejections_signature.clone(); + + if let Some(status) = status { + let signature = if status { + &mut new_approvals_signature + } else { + &mut new_rejections_signature + }; + + signature.sign(local_key, local_idx, nodes_count)?; + } + + new_deliveries_signature.sign(local_key, local_idx, nodes_count)?; + + let new_hash = Self::compute_hash( + &new_deliveries_signature, + &new_approvals_signature, + &new_rejections_signature); + + if new_hash != prev_hash { //prevent duplicate merges + self.deliveries_signature = new_deliveries_signature; + self.approvals_signature = new_approvals_signature; + self.rejections_signature = new_rejections_signature; + + self.signatures_hash = new_hash; + } + + Ok(new_hash != prev_hash) + } + + /// Merge status from another block + pub fn merge_status( + &mut self, + deliveries_signature: &MultiSignature, + approvals_signature: &MultiSignature, + rejections_signature: &MultiSignature, + merges_count: u32, + created_timestamp: i64, + ) -> Result { + let prev_hash = self.get_signatures_hash(); + let mut new_deliveries_signature = self.deliveries_signature.clone(); + let mut new_approvals_signature = self.approvals_signature.clone(); + let mut new_rejections_signature = self.rejections_signature.clone(); + + new_deliveries_signature.merge(deliveries_signature)?; + new_approvals_signature.merge(approvals_signature)?; + new_rejections_signature.merge(rejections_signature)?; + + let new_hash = Self::compute_hash( + &new_deliveries_signature, + &new_approvals_signature, + &new_rejections_signature); + + if new_hash != prev_hash { //prevent duplicate merges + self.deliveries_signature = new_deliveries_signature; + self.approvals_signature = new_approvals_signature; + self.rejections_signature = new_rejections_signature; + + if self.merges_count < merges_count { + self.merges_count = merges_count; //do not change hash + } + + self.signatures_hash = new_hash; + } + + self.set_creation_timestamp(created_timestamp); + + Ok(new_hash != prev_hash) + } + + /// Get signatures hash + pub(crate) fn get_signatures_hash(&self) -> u32 { //TODO: remove pub(crate) + self.signatures_hash + } + + /// Compute hash + fn compute_hash( + deliveries_signature: &MultiSignature, + approvals_signature: &MultiSignature, + rejections_signature: &MultiSignature, + ) -> u32 { + deliveries_signature.get_hash() ^ + approvals_signature.get_hash() ^ + rejections_signature.get_hash() + } + + /// Update hash + fn update_hash(&mut self) { + let prev_hash = self.signatures_hash; + + self.signatures_hash = Self::compute_hash( + &self.deliveries_signature, + &self.approvals_signature, + &self.rejections_signature); + + if self.signatures_hash != prev_hash { + self.serialized_block_status = None; + } + } + + /// Update candidate (for later candidate body receivements) + pub fn update_block_candidate(&mut self, new_block_candidate: Arc) -> bool { + if self.block_candidate.is_none() { + self.created_timestamp = Some(new_block_candidate.candidate.created_timestamp); + self.block_candidate = Some(new_block_candidate); + return true; + } + + let cur_block_candidate = self.block_candidate.as_ref().unwrap(); + if &new_block_candidate.hash != &cur_block_candidate.hash { + warn!(target: "verificator", "Attempt to update block candidate {:?} body with a new data: prev={:?}, new={:?}", self.candidate_id, cur_block_candidate.hash, new_block_candidate.hash); + return false; + } + + self.set_creation_timestamp(new_block_candidate.candidate.created_timestamp); + + false + } + + /// Create new block + pub fn create( + candidate_id: UInt256, + instance_counter: &InstanceCounter, + ) -> Arc> { + let mut body = Self { + candidate_id: candidate_id.clone(), + block_candidate: None, + deliveries_signature: MultiSignature::new(1, candidate_id.clone()), + approvals_signature: MultiSignature::new(2, candidate_id.clone()), + rejections_signature: MultiSignature::new(3, candidate_id.clone()), + signatures_hash: 0, + serialized_block_status: None, + created_timestamp: None, + merges_count: 0, + initially_mc_processed: false, + mc_originated: false, + mc_delivered: false, + ready_for_send: false, + first_appearance_time: std::time::SystemTime::now(), + _instance_counter: instance_counter.clone(), + first_external_request_time: None, + delivery_state_change_time: None, + }; + + body.update_hash(); + + Arc::new(SpinMutex::new(body)) + } +} diff --git a/src/validator/verification/mod.rs b/src/validator/verification/mod.rs new file mode 100644 index 00000000..88b0d27f --- /dev/null +++ b/src/validator/verification/mod.rs @@ -0,0 +1,105 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +extern crate catchain; + +use crate::engine_traits::EngineOperations; +use crate::validator::BlockCandidate; +use std::collections::HashMap; +/// API dependencies +use std::sync::Arc; +use std::sync::Weak; +use ton_block::BlockIdExt; +use ton_block::ValidatorDescr; +use ton_types::UInt256; +use ton_types::KeyOption; +use ton_types::Result; +use validator_session::PrivateKey; +use validator_session::PublicKeyHash; +use catchain::profiling::InstanceCounter; + +mod block; +mod multi_signature_bls; +mod multi_signature_unsafe; +mod verification_manager; +mod workchain; +mod workchain_overlay; +mod utils; + +pub const GENERATE_MISSING_BLS_KEY: bool = true; //generate missing BLS key from ED25519 public key (only for testing) + +/// Engine ptr +type EnginePtr = Arc; + +/// Verification manager pointer +pub type VerificationManagerPtr = Arc; + +/// Pointer to verification listener +pub type VerificationListenerPtr = Weak; + +/// Trait for verification events +#[async_trait::async_trait] +pub trait VerificationListener: Sync + Send { + /// Verify block candidate + async fn verify(&self, block_candidate: &BlockCandidate) -> bool; +} + +/// Verification manager +#[async_trait::async_trait] +pub trait VerificationManager: Sync + Send { + /// New block broadcast has been generated + async fn send_new_block_candidate(&self, candidate: &BlockCandidate); + + /// Get block status (delivered, rejected) + fn get_block_status( + &self, + block_id: &BlockIdExt, + ) -> (bool, bool); + + /// Wait for block verification + fn wait_for_block_verification( + &self, + block_id: &BlockIdExt, + timeout: &std::time::Duration, + ) -> bool; + + /// Update workchains + async fn update_workchains<'a>( + &'a self, + local_key_id: PublicKeyHash, + local_bls_key: PrivateKey, + workchain_id: i32, + utime_since: u32, + workchain_validators: &'a Vec, + mc_validators: &'a Vec, + listener: &'a VerificationListenerPtr, + ); + + /// Reset workchains + async fn reset_workchains<'a>(&'a self); +} + +/// Factory for verification objects +pub struct VerificationFactory {} + +impl VerificationFactory { + /// Create new verification manager + pub fn create_manager(engine: EnginePtr, runtime: tokio::runtime::Handle) -> VerificationManagerPtr { + verification_manager::VerificationManagerImpl::create(engine, runtime) + } + + /// Generate test BLS key based on public key + pub fn generate_test_bls_key(public_key: &Arc) -> Result> { + utils::generate_test_bls_key(public_key) + } +} diff --git a/src/validator/verification/multi_signature_bls.rs b/src/validator/verification/multi_signature_bls.rs new file mode 100644 index 00000000..a2896ee9 --- /dev/null +++ b/src/validator/verification/multi_signature_bls.rs @@ -0,0 +1,238 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +use super::*; +use ton_types::Result; +use std::time::SystemTime; +use std::time::Duration; +use validator_session::ValidatorWeight; +use catchain::check_execution_time; +use ton_types::bls::get_nodes_info_from_sig; +use ton_types::bls::sign_and_add_node_info; +use ton_types::bls::aggregate_two_bls_signatures; +use ton_types::bls::aggregate_public_keys_based_on_nodes_info; +use ton_types::bls::NodesInfo; +use ton_types::bls::BLS_PUBLIC_KEY_LEN; +use ton_types::bls::BLS_SECRET_KEY_LEN; +use ton_types::bls::truncate_nodes_info_and_verify; + +/* + Constants +*/ + +const BLS_DESERIALIZE_WARN_DELAY: Duration = Duration::from_millis(50); //delay for BLS signatures warnings dump + +/* +=============================================================================== + MultiSignature +=============================================================================== +*/ + +#[derive(Clone)] +pub struct MultiSignature { + hash: u32, //hash of signature + msg: Vec, //serialized block candidate for signing + nodes: Vec, //signers + signature: Vec, //signature +} + +impl std::fmt::Debug for MultiSignature { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Ok(nodes_info) = get_nodes_info_from_sig(&self.signature) { + if let Ok(nodes_info) = NodesInfo::deserialize(&nodes_info) { + write!(f, "[")?; + + let mut first = true; + + for (index, number_of_occurrence) in &nodes_info.map { + if !first { + write!(f, ", ")?; + } else { + first = false; + } + + write!(f, "{}x{}", index, number_of_occurrence)?; + } + + return write!(f, "]"); + } + } + + write!(f, "[???]") + } +} + +impl MultiSignature { + /// Add node to signature + pub fn sign(&mut self, local_key: &PrivateKey, idx: u16, nodes_count: u16) -> Result<()> { + let sk_bytes = local_key.export_key()?; + let sk_bytes: [u8; BLS_SECRET_KEY_LEN] = sk_bytes.to_vec().try_into().unwrap_or_else(|v: Vec| panic!("Expected a Vec of length {} but it was {}", BLS_SECRET_KEY_LEN, v.len())); + + self.merge_impl(&sign_and_add_node_info(&sk_bytes, &self.msg, idx, nodes_count)?) + } + + /// Merge signatures + pub fn merge(&mut self, other: &MultiSignature) -> Result<()> { + assert!(self.msg == other.msg); + self.merge_impl(&other.signature) + } + + /// Merge signatures (internal) + fn merge_impl(&mut self, other_signature: &Vec) -> Result<()> { + check_execution_time!(5_000); + if self.signature.len() == 0 { + self.signature = other_signature.clone(); + + self.update_hash(); + + return Ok(()); + } else if other_signature.len() == 0 { + return Ok(()); + } else { + let new_signature = aggregate_two_bls_signatures(&self.signature, other_signature)?; + let (new_nodes, new_hash) = Self::compute_nodes_and_hash(&new_signature); + + if new_hash != self.hash { + self.signature = new_signature; + self.nodes = new_nodes; + self.hash = new_hash; + } + + return Ok(()); + } + } + + /// Get hash + pub fn get_hash(&self) -> u32 { + self.hash + } + + /// Is the signature empty + pub fn empty(&self) -> bool { + self.signature.len() == 0 + } + + /// Total weight + pub fn get_total_weight(&self, validators: &Vec) -> ValidatorWeight { + let mut total_weight = 0; + + for validator_idx in &self.nodes { + let validator_idx = *validator_idx as usize; + if validator_idx >= validators.len() { + continue; + } + + total_weight += validators[validator_idx].weight; + } + + total_weight + } + + /// Compute nodes and hash + fn compute_nodes_and_hash(signature: &Vec) -> (Vec, u32) { + check_execution_time!(1_000); + let mut new_nodes = Vec::new(); + + if signature.len() > 0 { + if let Ok(nodes_info) = get_nodes_info_from_sig(&signature) { + if let Ok(nodes_info) = NodesInfo::deserialize(&nodes_info) { + for (validator_idx, _number_of_occurrence) in &nodes_info.map { + new_nodes.push(*validator_idx); + } + + new_nodes.sort(); + } + } + } + + let hash = ton_types::crc32_digest(Self::raw_byte_access(&new_nodes)); + + (new_nodes, hash) + } + + /// Update hash + fn update_hash(&mut self) { + (self.nodes, self.hash) = Self::compute_nodes_and_hash(&self.signature); + } + + /// Get bytes for hash computation + fn raw_byte_access(s16: &[u16]) -> &[u8] { + unsafe { std::slice::from_raw_parts(s16.as_ptr() as *const u8, s16.len() * 2) } + } + + /// Serialize signature + pub fn serialize(&self) -> Vec { + //deflate::deflate_bytes(&self.signature) + self.signature.clone() + } + + /// Deserialize signature + pub fn deserialize(type_id: u8, candidate_id: &UInt256, wc_pub_key_refs: &Vec<&[u8; BLS_PUBLIC_KEY_LEN]>, serialized_signature: &[u8]) -> Result { + check_execution_time!(20_000); + + //let signature = inflate::inflate_bytes(serialized_signature); + let signature: Result> = Ok(serialized_signature.to_vec()); + + if let Err(err) = signature { + failure::bail!("inflate error: {}", err); + } + + let signature = signature.unwrap(); + + let mut body = Self::new(type_id, candidate_id.clone()); + + body.signature = signature.clone(); + + body.update_hash(); + + if signature.len() > 0 { + let start_time = SystemTime::now(); + let nodes_info = get_nodes_info_from_sig(&signature)?; + let aggregated_pub_key = aggregate_public_keys_based_on_nodes_info(wc_pub_key_refs, &nodes_info)?; + + if !truncate_nodes_info_and_verify(&signature, &aggregated_pub_key, &body.msg)? { + failure::bail!("Can't verify block candidate {:?} signature {:?} (type {})", candidate_id, body, type_id); + } + + let processing_delay = match start_time.elapsed() { + Ok(elapsed) => elapsed, + Err(_err) => Duration::default(), + }; + + if processing_delay > BLS_DESERIALIZE_WARN_DELAY { + log::warn!(target: "verificator", "Long BLS deserialization latency={:.3}s for signature={:?}", processing_delay.as_secs_f64(), body); + } + } + + Ok(body) + } + + /// Create new signature + pub fn new(type_id: u8, candidate_id: UInt256) -> Self { + let msg: [u8; 32] = candidate_id.as_array().clone().into(); + let mut msg = msg.to_vec(); + + msg.push(type_id); + + let mut body = Self { + msg, + signature: Vec::new(), + nodes: Vec::new(), + hash: 0, + }; + + body.update_hash(); + + body + } +} diff --git a/src/validator/verification/multi_signature_unsafe.rs b/src/validator/verification/multi_signature_unsafe.rs new file mode 100644 index 00000000..1bd4a63a --- /dev/null +++ b/src/validator/verification/multi_signature_unsafe.rs @@ -0,0 +1,185 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +use super::*; +use ton_types::Result; +use validator_session::ValidatorWeight; +use ton_types::bls::BLS_PUBLIC_KEY_LEN; + +/* +=============================================================================== + MultiSignature +=============================================================================== +*/ + +#[derive(Clone)] +pub struct MultiSignature { + hash: u32, //hash of signature + nodes: Vec, //vector of node indexes which signed (fake, for testing only, should be replace with BLS) +} + +impl std::fmt::Debug for MultiSignature { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[{:?}]", self.nodes) + } +} + +//TODO: enable dead-code warnings after merging usafe with BLS implementations +#[allow(dead_code)] +impl MultiSignature { + /// Add node to signature + pub fn sign(&mut self, _local_key: &PrivateKey, idx: u16, _nodes_count: u16) -> Result<()> { + match self.nodes.binary_search(&idx) { + Ok(_pos) => {} // element already in vector + Err(pos) => { + self.nodes.insert(pos, idx); + self.update_hash(); + } + } + + Ok(()) + } + + /// Merge signatures + pub fn merge(&mut self, other: &MultiSignature) -> Result<()> { + let a = &self.nodes; + let b = &other.nodes; + let (mut i, mut j) = (0, 0); + let mut sorted = vec![]; + let remaining; + let remaining_idx; + loop { + if i == a.len() { + remaining = b; + remaining_idx = j; + break; + } + + if j == b.len() { + remaining = a; + remaining_idx = i; + break; + } + + if a[i] <= b[j] { + sorted.push(a[i]); + + if a[i] == b[j] { + j += 1; + } + + i += 1; + } else { + sorted.push(b[j]); + j += 1; + } + } + + for i in remaining_idx..remaining.len() { + sorted.push(remaining[i]); + } + + if self.nodes.len() != sorted.len() { + //new elements appeared + + if sorted.len() == other.nodes.len() { + self.nodes = other.nodes.clone(); + self.hash = other.hash; + } else { + self.nodes = sorted; + self.update_hash(); + } + } + + Ok(()) + } + + /// Get hash + pub fn get_hash(&self) -> u32 { + self.hash + } + + /// Is the signature empty + pub fn empty(&self) -> bool { + self.nodes.len() == 0 + } + + /// Total weight + pub fn get_total_weight(&self, validators: &Vec) -> ValidatorWeight { + let mut total_weight = 0; + + for validator_idx in &self.nodes { + let validator_idx = *validator_idx as usize; + if validator_idx >= validators.len() { + continue; + } + + total_weight += validators[validator_idx].weight; + } + + total_weight + } + + /// Get bytes for hash computation + fn raw_byte_access(s16: &[u16]) -> &[u8] { + unsafe { std::slice::from_raw_parts(s16.as_ptr() as *const u8, s16.len() * 2) } + } + + /// Get words from bytes + fn raw_words_access(s8: &[u8]) -> &[u16] { + unsafe { std::slice::from_raw_parts(s8.as_ptr() as *const u16, s8.len() / 2) } + } + + /// Update hash + fn update_hash(&mut self) { + self.hash = ton_types::crc32_digest(Self::raw_byte_access(&self.nodes)) + } + + /// Serialize signature + pub fn serialize(&self) -> Vec { + deflate::deflate_bytes(Self::raw_byte_access(&self.nodes)) + } + + /// Deserialize signature + pub fn deserialize(_type: u8, _candidate_id: &UInt256, _wc_pub_key_refs: &Vec<&[u8; BLS_PUBLIC_KEY_LEN]>, serialized_signature: &[u8]) -> Result { + let decoded = inflate::inflate_bytes(serialized_signature); + + if let Err(err) = decoded { + failure::bail!("inflate error: {}", err); + } + + let decoded = decoded.unwrap(); + + let decoded = Self::raw_words_access(&decoded); + let mut body = Self { + nodes: decoded.into(), + hash: 0, + }; + + body.update_hash(); + + Ok(body) + } + + /// Create new signature + pub fn new(_type: u8, _candidate_id: UInt256) -> Self { + let mut body = Self { + nodes: Vec::new(), + hash: 0, + }; + + body.update_hash(); + + body + } +} diff --git a/src/validator/verification/simulator/new_consensus_simulator.py b/src/validator/verification/simulator/new_consensus_simulator.py new file mode 100644 index 00000000..0ca3cd45 --- /dev/null +++ b/src/validator/verification/simulator/new_consensus_simulator.py @@ -0,0 +1,949 @@ +import random +import copy +from pprint import pprint +from datetime import datetime, timedelta + +################################################################################################### +# Constants +################################################################################################### +SIMULATION_TIME = timedelta(minutes=10) # time of the simulation +SIMULATION_STEP = timedelta(milliseconds=100) # simulation step period +DUMP_STEP = timedelta(milliseconds=500) # simulation dump period + +SHARD_VALIDATORS_COUNT = 7 # number of validators in shard +MC_VALIDATORS_COUNT = 100 # number of validators in masterchain +SHARDS_COUNT = 8 # number of shards in workchain +WORKCHAINS_COUNT = 3 # number of workchains + +CANDIDATE_DELAY = 3000 # time for candidate collation +MAX_CANDIDATES_COUNT = 3 # maximum number of candidates per BFT round +EMPTY_BLOCK_DELAY = CANDIDATE_DELAY * MAX_CANDIDATES_COUNT # delay for skip block event +MIN_CANDIDATE_GENERATION_DELAY = 0.3 +MAX_CANDIDATE_GENERATION_DELAY = 1.0 +VALID_BLOCK_THRESHOLD = 0.95 # probability to select block as valid in BFT +MALLICIOUS_BLOCK_THRESHOLD = 0.5 # probability to generate mallision block which will be ignored by shard +MIN_CANDIDATE_SHARD_DELIVERY_DELAY = 0.2 # min delay for delivery candidate within shard +MAX_CANDIDATE_SHARD_DELIVERY_DELAY = 0.8 # max delay for delivery candidate within shard +MIN_CANDIDATE_WORKCHAIN_DELIVERY_DELAY = MIN_CANDIDATE_SHARD_DELIVERY_DELAY # min delay for delivery candidate within workchain +MAX_CANDIDATE_WORKCHAIN_DELIVERY_DELAY = MAX_CANDIDATE_SHARD_DELIVERY_DELAY # max delay for delivery candidate within workchain +MIN_CANDIDATE_WORKCHAIN_DELIVERY_RELIABILITY = 0.9 +MAX_CANDIDATE_WORKCHAIN_DELIVERY_RELIABILITY = 1.0 +BFT_CUTOFF_WEIGHT = 0.67 + +CANDIDATE_DELIVERY_ACK_NEIGHBOURS_ROTATION_PERIOD = 3000 # neighbours of broadcast protection flow rotation period +CANDIDATE_DELIVERY_ACK_NEIGHBOURS_COUNT = int(SHARDS_COUNT * SHARD_VALIDATORS_COUNT / 5) # number of neighbours for broadcast protection delivery + +MIN_CANDIDATE_DELIVERY_ACK_DELAY = MIN_CANDIDATE_SHARD_DELIVERY_DELAY +MAX_CANDIDATE_DELIVERY_ACK_DELAY = MAX_CANDIDATE_SHARD_DELIVERY_DELAY +MIN_CANDIDATE_DELIVERY_ACK_RELIABILITY = 0.9 +MAX_CANDIDATE_DELIVERY_ACK_RELIABILITY = 1.0 + +DELIVERY_CUTOFF_WEIGHT = 0.5001 # weight for validators received broadcast + +MIN_SHARD_BLOCK_MC_DELIVERY_DELAY = 0.1 # min time for delivery committed block from shard to MC +MAX_SHARD_BLOCK_MC_DELIVERY_DELAY = 0.4 +MIN_CANDIDATE_STATUS_MC_DELIVERY_DELAY = MIN_SHARD_BLOCK_MC_DELIVERY_DELAY +MAX_CANDIDATE_STATUS_MC_DELIVERY_DELAY = MAX_SHARD_BLOCK_MC_DELIVERY_DELAY +WORKCHAIN_TO_MASTERCHAIN_ACK_MODULO = 3 + +SELF_RECHECK_DELAY = 0.1 #delay for rechecking a message on the same node (like broadcast delivery results) +SELF_MAX_RECHECK_ATTEMPTS = 5 #max number of attempts for self rechecking + +VERIFICATOR_SELECTION_THRESHOLD = 0.01 #probability to be a verificator +MIN_VERIFICATION_DELIVERY_DELAY = MIN_CANDIDATE_SHARD_DELIVERY_DELAY +MAX_VERIFICATION_DELIVERY_DELAY = MAX_CANDIDATE_SHARD_DELIVERY_DELAY +MIN_VERIFICATION_DELIVERY_RELIABILITY = MIN_CANDIDATE_DELIVERY_ACK_RELIABILITY +MAX_VERIFICATION_DELIVERY_RELIABILITY = MAX_CANDIDATE_DELIVERY_ACK_RELIABILITY +MALLICIOUS_VERIFIER_THRESHOLD = 0.05 + +################################################################################################### +# Global context +################################################################################################### +global_time = datetime.fromtimestamp(0) + +################################################################################################### +# Utils +################################################################################################### + +# Log +def log_print(s): + print("{:10.3f} | {}".format(time_to_seconds(get_time() - datetime.fromtimestamp(0)), s)) + +# Convert time to seconds +def time_to_seconds(time): + return time.microseconds / 1000000.0 + time.seconds + +# Get scaled random value +def scaled_rand(min_value, max_value): + if max_value == min_value: + return min_value + normalized_value = random.uniform(0, 1) + return min_value + (max_value - min_value) * normalized_value + +# Global time +def get_time(): + return global_time + +# Create node messages queue +def create_message_queue(): + return [] + +# Put message to queue +def put_message_to_queue(queue, msg): + queue.append(msg) + +# Run messages in message queue +def process_messages(queue, node, time): + msg_count = len(queue) + + for i in range(msg_count): + msg = queue.pop(0) + msg(node) + +################################################################################################### +# MC state +################################################################################################### + +def create_mc_state(): + state = { + 'workchains': {}, + 'verifications': {}, + } + return state + +def get_mc_workchain_state(mc_state, workchain_id): + if workchain_id in mc_state['workchains']: + return mc_state['workchains'][workchain_id] + + wc_state = { + 'shards': {} + } + mc_state['workchains'][workchain_id] = wc_state + + return wc_state + +def get_mc_shard_state(mc_state, workchain_id, shard_id): + workchain = get_mc_workchain_state(mc_state, workchain_id) + + if shard_id in workchain['shards']: + return workchain['shards'][shard_id] + + shard = { + 'top_block': None, + 'height': -1, + 'block_by_height': {}, + 'blocks': {}, + } + + workchain['shards'][shard_id] = shard + + return shard + +################################################################################################### +# Node +################################################################################################### + +# Create shard node +def create_shard_node(shard_id, source_id, shard, workchain, simulator_state): + node_id = simulator_state['node_id_counter'] + simulator_state['node_id_counter'] += 1 + node = { + 'shard_id' : shard_id, + 'source_id': source_id, + 'id' : node_id, + 'shard': shard, + 'workchain': workchain, + 'simulator_state': simulator_state, + 'block_number' : 0, + 'message_queue': create_message_queue(), + 'delayed_actions': [], + 'round': -1, + 'candidate_generated': False, + 'block_approvers': {}, + 'block_delivered_sources': {}, + 'block_delivery_mc_notified': {}, + 'candidate_delivery_neighbours': None, + 'candidate_delivery_neighbours_next_rotation': get_time(), + } + + if is_masterchain(node): + node['mc_state'] = create_mc_state() + + node['workchain']['nodes'].append(node) + node['workchain']['node_by_id'][node_id] = node + + return node + +# Put message to node's message queue +def put_message_to_node_queue(node, msg, delay=0.0, reliability=1.0): + reliability_rnd = random.uniform(0, 1) + if reliability_rnd < 1.0 - reliability: + return #ignore delivery + + if delay > 0.0: + delayed_action = lambda node: put_message_to_node_queue(node, msg) + delayed_action_time = get_time() + timedelta(milliseconds=int(delay * 1000)) + node['delayed_actions'].append({ + 'time': delayed_action_time, + 'action': delayed_action, + }) + return + + #todo: add node messages limit + node['message_queue'].append(msg) + +# Put message to shard nodes queues +def put_message_to_neighbours_queue(nodes, msg, min_delay=0.0, max_delay=0.0, min_reliability=1.0, max_reliability=1.0): + for node in nodes: + delay = scaled_rand(min_delay, max_delay) + reliability = scaled_rand(min_reliability, max_reliability) + put_message_to_node_queue(node, msg, delay=delay, reliability=reliability) + +# Process delayed actions +def process_delayed_actions(node, time): + actions = node['delayed_actions'] + node['delayed_actions'] = [] + + for action_decl in actions: + if action_decl['time'] <= time: + node['delayed_actions'].append(action_decl) + continue + action_decl['action'](node) + +# New round for node +def new_bft_round(node, time): + node['start_round_time'] = time + node['round'] += 1 + node['candidate_generated'] = False + + sources_count = len(node['shard']['nodes']) + priority = node['round'] - node['source_id'] + if priority < 0: + priority += sources_count + priority = priority % sources_count + + node['end_round_time'] = time + timedelta(milliseconds=EMPTY_BLOCK_DELAY) + node['collation_time'] = time + timedelta(milliseconds=CANDIDATE_DELAY * priority) + + #log_print("new round={} for {}:{}:{} node={} priority={}".format(node['round'], node['workchain']['id'], node['shard']['id'], node['block_number'], + # node['id'], priority)) + +# Commit block +def commit_block(node, block): + round = block['round'] + + if round != node['round']: + return + + shard = node['shard'] + + #log_print("commit block #{} for shard {}:{}:{} from node #{}; round={}".format(block['id'], shard['workchain']['id'], shard['id'], block['height'], block['created_by']['id'], round)) + + if round in shard['round_candidate_selection']: + assert shard['round_candidate_selection'][round] != 'empty_block' + assert shard['round_candidate_selection'][round]['id'] == block['id'] + else: + assert round == shard['round'] + + shard['round_candidate_selection'][round] = block + shard['top_block'] = block + shard['block_height'] += 1 + shard['round'] += 1 + node['block_number'] += 1 + + new_bft_round(node, get_time()) + + if is_masterchain(node): + apply_mc_block(node, block) + #todo: broadcast block in MC + else: + # Send block to MC + put_message_to_workchain_queues(node['simulator_state']['workchains'][-1], lambda mc_node: new_shard_block_committed(mc_node, block), + min_delay=MIN_SHARD_BLOCK_MC_DELIVERY_DELAY, max_delay=MAX_SHARD_BLOCK_MC_DELIVERY_DELAY) + +# Commit empty block +def try_commit_empty_block(node, shard, round): + if round in shard['round_candidate_selection']: + return False + + if round != node['round']: + return False + + log_print("commit empty block for node #{} in round {}".format(node['id'], round)) + + shard['round_candidate_selection'][round] = 'empty_block' + shard['round'] += 1 + + new_bft_round(node, get_time()) + + return True + +def is_fork(node, block): + round = block['round'] + shard = node['shard'] + + if round in shard['round_candidate_selection']: + if shard['round_candidate_selection'][round] == 'empty_block': + return True + + if shard['round_candidate_selection'][round]['id'] != block['id']: + return True + + return False + +# Validate candidate +def validate_candidate(node, block): + if not block['is_valid']: + return False, False + + if is_fork(node, block): + return False, False + + if is_masterchain(node): + is_valid, need_to_wait = validate_mc_candidate(node, block) + + if not is_valid: + return False, need_to_wait + + return True, False + +# New candidate received within shard +def new_candidate_received_within_shard(node, block, attempt=0): + #log_print('new candidate {} from node #{} has been received via shard by node #{}'.format(block['id'], block['created_by']['id'], node['id'])) + round = block['round'] + if round < node['round']: + return + + is_valid, need_to_wait = validate_candidate(node, block) + + if not is_valid: + if need_to_wait and attempt < SELF_MAX_RECHECK_ATTEMPTS: + log_print("recheck block candidate {}:{}:{} with ID #{} in {:.3f}s".format(block['workchain_id'], block['shard_id'], block['height'], block['id'], SELF_RECHECK_DELAY)) + put_message_to_node_queue(node, lambda receiver_node: SELF_RECHECK_DELAY(receiver_node, block, attempt + 1), delay=SELF_RECHECK_DELAY) + return + + put_message_to_shard_queues(node['shard'], lambda receiver_node: new_candidate_approved(receiver_node, node, block), min_delay=MIN_CANDIDATE_SHARD_DELIVERY_DELAY, max_delay=MAX_CANDIDATE_SHARD_DELIVERY_DELAY) + +# New candidated approved by node +def new_candidate_approved(receiver_node, sender_node, block): + round = block['round'] + + if round < receiver_node['round']: + return + + if round > receiver_node['round']: + put_message_to_node_queue(receiver_node, lambda receiver_node: new_candidate_approved(receiver_node, sender_node, block), delay=SELF_RECHECK_DELAY) + return + + approvers = {} + block_id = block['id'] + sender_id = sender_node['id'] + + if block_id in receiver_node['block_approvers']: + approvers = receiver_node['block_approvers'][block_id] + else: + receiver_node['block_approvers'][block_id] = approvers + + if sender_id in approvers: + return + + approvers[sender_id] = True + + approves_count = len(approvers) + total_approvers_count = len(receiver_node['shard']['nodes']) + weight = approves_count / float(total_approvers_count) + + if weight < BFT_CUTOFF_WEIGHT: + #log_print("node {}:{}:{} with ID #{} is waiting for consensus in round {} for block #{} created by node #{}; current weight is {:.2f}%".format(block['workchain_id'], block['shard_id'], block['height'], receiver_node['id'], + # round, block['id'], block['created_by']['id'], weight * 100.0)) + return + + if is_fork(receiver_node, block): + return + + commit_block(receiver_node, block) + +# New candidate received within workchain +def new_candidate_received_within_workchain(node, block): + #log_print('new candidate {} from node #{} has been received via workchain by node #{}'.format(block['id'], block['created_by']['id'], node['id'])) + put_message_to_neighbours_queue(node['candidate_delivery_neighbours'], lambda receiver_node: new_candidate_delivery_accepted(receiver_node, [node], block), + min_delay=MIN_CANDIDATE_DELIVERY_ACK_DELAY, max_delay=MAX_CANDIDATE_DELIVERY_ACK_DELAY, + min_reliability=MIN_CANDIDATE_DELIVERY_ACK_RELIABILITY, max_reliability=MAX_CANDIDATE_DELIVERY_ACK_RELIABILITY) + + # select verificator + if random.uniform(0, 1) < VERIFICATOR_SELECTION_THRESHOLD: + verify_block(node, block) + +# Delivery acceptance by node within workchain +def new_candidate_delivery_accepted(receiver_node, sender_nodes, block): + #log_print('new candidate {} ACK has been received via workchain by node #{} from {} senders'.format(block['id'], receiver_node['id'], len(sender_nodes))) + + delivery_sources = {} + block_id = block['id'] + if block_id in receiver_node['block_delivered_sources']: + delivery_sources = receiver_node['block_delivered_sources'][block_id] + else: + receiver_node['block_delivered_sources'][block_id] = delivery_sources + + # Compute delivery sources before applying update + start_delivery_sources = 0 + for node_id, status in delivery_sources.items(): + if not status: + continue + start_delivery_sources += 1 + + # Apply update + for sender_node in sender_nodes: + if sender_node == receiver_node: + return + sender_id = sender_node['id'] + if sender_id in delivery_sources: + continue + delivery_sources[sender_id] = True + + delivery_sources[receiver_node['id']] = True + + # Compute delivery sources after applying update + sender_nodes = [] + end_delivery_sources = 0 + + for node_id, status in delivery_sources.items(): + if not status: + continue + sender_nodes.append(receiver_node['workchain']['node_by_id'][node_id]) + end_delivery_sources += 1 + + # Continue propagation in case of new source appeared + if end_delivery_sources > start_delivery_sources: + put_message_to_neighbours_queue(receiver_node['candidate_delivery_neighbours'], lambda receiver_node: new_candidate_delivery_accepted(receiver_node, sender_nodes, block), + min_delay=MIN_CANDIDATE_DELIVERY_ACK_DELAY, max_delay=MAX_CANDIDATE_DELIVERY_ACK_DELAY, + min_reliability=MIN_CANDIDATE_DELIVERY_ACK_RELIABILITY, max_reliability=MAX_CANDIDATE_DELIVERY_ACK_RELIABILITY) + + block_delivery_nodes_count = len(delivery_sources) + block_delivery_total_nodes_count = len(receiver_node['workchain']['nodes']) + weight = block_delivery_nodes_count / float(block_delivery_total_nodes_count) + + if weight >= DELIVERY_CUTOFF_WEIGHT: + #log_print("block #{} has been received by {:.1f}% of workchain ({}/{})".format(block['id'], weight * 100.0, block_delivery_nodes_count, block_delivery_total_nodes_count)) + + if not block_id in receiver_node['block_delivery_mc_notified'] and receiver_node['source_id'] % WORKCHAIN_TO_MASTERCHAIN_ACK_MODULO == 0: + #log_print("block #{} has been received by {:.1f}% of workchain; send ACK to MC".format(block['id'], weight * 100.0)) + + receiver_node['block_delivery_mc_notified'][block_id] = True + put_message_to_workchain_queues(receiver_node['simulator_state']['workchains'][-1], lambda node: candidate_delivered_to_workchain(node, block, weight), + min_delay=MIN_CANDIDATE_STATUS_MC_DELIVERY_DELAY, max_delay=MAX_CANDIDATE_STATUS_MC_DELIVERY_DELAY) + +# Generate candidate +def generate_candidate(node, round, time): + # generate new block + block = { + 'id': node['simulator_state']['block_id_counter'], + 'workchain_id': node['workchain']['id'], + 'shard_id': node['shard']['id'], + 'created_by': node, + 'height': node['shard']['block_height'] + 1, + 'prev': node['shard']['top_block'], + 'round': round, + 'is_valid': random.uniform(0, 1) < VALID_BLOCK_THRESHOLD, + 'is_mallicious': random.uniform(0, 1) < MALLICIOUS_BLOCK_THRESHOLD, + 'creation_time': get_time(), + } + + log_print('generate candidate {}:{}:{} by collator node #{}; round={}'.format(block['workchain_id'], block['shard_id'], block['height'], node['id'], round)) + + node['simulator_state']['block_id_counter'] += 1 + + if is_masterchain(node): + generate_mc_candidate(node, block) + + put_message_to_shard_queues(node['shard'], lambda node: new_candidate_received_within_shard(node, block), min_delay=MIN_CANDIDATE_SHARD_DELIVERY_DELAY, max_delay=MAX_CANDIDATE_SHARD_DELIVERY_DELAY) + + if not is_masterchain(node): + put_message_to_workchain_queues(node['workchain'], lambda node: new_candidate_received_within_workchain(node, block), + min_delay=MIN_CANDIDATE_WORKCHAIN_DELIVERY_DELAY, max_delay=MAX_CANDIDATE_WORKCHAIN_DELIVERY_DELAY, + min_reliability=MIN_CANDIDATE_WORKCHAIN_DELIVERY_RELIABILITY, max_reliability=MAX_CANDIDATE_WORKCHAIN_DELIVERY_RELIABILITY) + +# Check if not is in masterchain +def is_masterchain(node): + return node['workchain']['id'] == -1 + +# Get new block within MC state for shard +def get_new_shard_block(node, block): + assert is_masterchain(node) + + block_id = block['id'] + block_height = block['height'] + workchain_id = block['workchain_id'] + shard_id = block['shard_id'] + + mc_state = node['mc_state'] + shard_state = get_mc_shard_state(mc_state, workchain_id, shard_id) + + if block_id in shard_state['blocks']: + return shard_state['blocks'][block_id], False; + + block_decl = { + 'id': block_id, + 'block': block, + 'delivery_weight': 0.0, + } + + shard_state['blocks'][block_id] = block_decl + + return block_decl, True; + +# New shard block appeared +def new_shard_block_committed(node, block): + block_decl, is_created = get_new_shard_block(node, block) + + #log_print("new shard block {}:{}:{} with ID #{} appeared in MC (mcnode is #{})".format(block['workchain_id'], block['shard_id'], block['height'], block['id'], node['id'])) + + if not is_created: + assert block['id'] == block_decl['id'] + + block_id = block['id'] + block_height = block['height'] + workchain_id = block['workchain_id'] + shard_id = block['shard_id'] + + mc_state = node['mc_state'] + shard_state = get_mc_shard_state(mc_state, workchain_id, shard_id) + + if block_height in shard_state['block_by_height']: + decl = shard_state['block_by_height'][block_height] + + if block_id != decl['id']: + log_print("conflict for shard {}:{}:{} with ID {} and candidate {}:{}:{} with ID {}".format(decl['block']['workchain_id'], decl['block']['shard_id'], + decl['block']['height'], decl['id'], block['workchain_id'], block['shard_id'], block['height'], block['id'])) + assert block_id == decl['id'] + + return + + shard_state['block_by_height'][block_height] = block_decl + + #log_print("new shard block {}:{}:{} with ID #{} appeared in MC (mcnode is #{}); {} heights are registered".format(block['workchain_id'], block['shard_id'], + # block['height'], block['id'], node['id'], len(shard_state['block_by_height']))) + +def candidate_delivered_to_workchain(node, block, weight): + block_decl, is_created = get_new_shard_block(node, block) + + if weight > block_decl['delivery_weight']: + block_decl['delivery_weight'] = weight + +def get_blocks_heights(blocks, min_height): + heights = [] + + for block_height, block_decl in blocks.items(): + if block_height <= min_height: + continue + heights.append(block_height) + + heights.sort(reverse=True) + + return heights + +def dump_mc_block_state(state): + for workchain_id, workchain in state['workchains'].items(): + log_print(" workchain {}:".format(workchain_id)) + for shard_id, shard in workchain['shards'].items(): + top_block = shard['top_block'] + + if top_block: + top_block_string = "#{}".format(top_block['id']) + blamed_string = "" + + if 'is_blamed' in shard and shard['is_blamed']: + blamed_string = "; BLAMED!" + log_print(" shard {}: top block {:5} with height {}{}".format(shard_id, top_block_string, top_block['height'], blamed_string)) + else: + log_print(" shard {}: N/A".format(shard_id)) + +def dump_mc_node_state(state): + dump_mc_block_state(state) + +def apply_mc_block(node, mc_block): + #log_print("applying MC block {}:{}:{} with ID #{} for node #{}".format(mc_block['workchain_id'], mc_block['shard_id'], mc_block['height'], mc_block['id'], node['id'])) + + mc_node_state = node['mc_state'] + mc_block_state = mc_block['mc_state'] + min_creation_time = get_time() + + for workchain_id, block_workchain in mc_block_state['workchains'].items(): + node_workchain = mc_node_state['workchains'][workchain_id] + + for shard_id, block_shard in block_workchain['shards'].items(): + node_shard = node_workchain['shards'][shard_id] + + node_shard['top_block'] = block_shard['top_block'] + + if block_shard['top_block']: + if node_shard['top_block']: + block_it = block_shard['top_block'] + block_it_end = node_shard['top_block'] + prev_height = node_shard['top_block']['height'] + + while block_it != block_it_end: + if min_creation_time > block_it['creation_time']: + min_creation_time = block_it['creation_time'] + + block_it = block_it['prev'] + + if min_creation_time > block_shard['top_block']['creation_time']: + min_creation_time = block_shard['top_block']['creation_time'] + + node_shard['height'] = block_shard['top_block']['height'] + else: + node_shard['height'] = -1 + + #dump with minimized spam + if not mc_block['id'] in node['shard']['shard_mc_dumps']: + log_print("dump MC state {}:{}:{} after applying of block #{} on node #{} (max_latency={:.3f}s):".format(mc_block['workchain_id'], mc_block['shard_id'], mc_block['height'], mc_block['id'], node['id'], + time_to_seconds(get_time() - min_creation_time))) + + node['shard']['shard_mc_dumps'][mc_block['id']] = True + dump_mc_node_state(mc_node_state) + +def check_verifications(mc_node, mc_block, cur_block, new_block_it, check_only=False): + mc_state = mc_node['mc_state'] + verifications = mc_state['verifications'] + + while new_block_it: + new_block = new_block_it + new_block_it = new_block_it['prev'] + block_verifications = None + block_id = new_block['id'] + + if cur_block and cur_block['id'] == new_block['id']: + break + + if not block_id in verifications: + continue #no rejects from verifiers; todo: wait for at least 1 approve? + + block_verifications = verifications[block_id] + + if block_verifications['is_suspicious']: + if not check_only: + mallicious_block_detected(mc_node, mc_block, new_block, block_verifications['verifiers']) + return False + else: + if new_block['is_mallicious'] and not check_only: + log_print("!!! mallicious shard block {}:{}:{} with ID #{} to MC block {}:{}:{} has NOT been detected on node #{} (collator is node #{})".format( + new_block['workchain_id'], new_block['shard_id'], new_block['height'], new_block['id'], + mc_block['workchain_id'], mc_block['shard_id'], mc_block['height'], mc_node['id'], new_block['created_by']['id'])) + + return True + +def mallicious_block_detected(mc_node, mc_block, shard_block, verifications): + log_print("!!! attempt to commit mallicious shard block {}:{}:{} with ID #{} to MC block {}:{}:{} has been detected on node #{} (collator is node #{})".format( + shard_block['workchain_id'], shard_block['shard_id'], shard_block['height'], shard_block['id'], + mc_block['workchain_id'], mc_block['shard_id'], mc_block['height'], mc_node['id'], shard_block['created_by']['id'])) + + for verifier_id, verification in verifications.items(): + status_string = None + + if verification['status']: + status_string = "approved" + else: + status_string = "rejected" + + log_print(" {} by verifier node #{}".format(status_string, verifier_id)) + + mc_node['mc_state']['workchains'][shard_block['workchain_id']]['shards'][shard_block['shard_id']]['is_blamed'] = True + + #todo: reset shard!!! + #todo: block verification for wide consensus + #todo: slashing events + +def validate_mc_candidate(node, mc_block): + #log_print("validating MC block {}:{}:{} with ID #{} on node #{}".format(mc_block['workchain_id'], mc_block['shard_id'], mc_block['height'], mc_block['id'], node['id'])) + + mc_node_state = node['mc_state'] + mc_block_state = mc_block['mc_state'] + + for workchain_id, block_workchain in mc_block_state['workchains'].items(): + node_workchain = mc_node_state['workchains'][workchain_id] + + for shard_id, block_shard in block_workchain['shards'].items(): + node_shard = node_workchain['shards'][shard_id] + cur_block = node_shard['top_block'] + new_block = block_shard['top_block'] + + # check verifications + if not check_verifications(node, mc_block, cur_block, new_block): + return False, False + + if not cur_block: + continue + + if not new_block: + return False, False + + block_id = new_block['id'] + new_block_decl = node_shard['blocks'][block_id] + + if not new_block_decl: + return False, True # block has not been delivered to this node yet + + if new_block_decl['delivery_weight'] < DELIVERY_CUTOFF_WEIGHT: + return False, True #reject shard blocks which have no cutoff broadcast receive ACKs + + if new_block['height'] < cur_block['height']: + return False, False + + return True, False + +def generate_mc_candidate(node, mc_block): + mc_state = node['mc_state'] + workchains_block_state = {} + mc_block_state = {'workchains': workchains_block_state} + + mc_block['mc_state'] = mc_block_state + + for workchain_id, workchain in mc_state['workchains'].items(): + shards_block_state = {} + workchain_block_state = {'shards' : shards_block_state} + workchains_block_state[workchain_id] = workchain_block_state + + for shard_id, shard in workchain['shards'].items(): + shard_block_state = {'top_block': None} + shards_block_state[shard_id] = shard_block_state + unprocessed_heights = get_blocks_heights(shard['block_by_height'], shard['height']) + cur_block = mc_state['workchains'][workchain_id]['shards'][shard_id]['top_block'] + + while len(unprocessed_heights) > 0: + height = unprocessed_heights.pop(0) + block_decl = shard['block_by_height'][height] + + if block_decl['delivery_weight'] < DELIVERY_CUTOFF_WEIGHT: + continue #ignore shard blocks which have no cutoff broadcast receive ACKs + + if not check_verifications(node, mc_block, cur_block, block_decl['block']): + continue + + shard_block_state['top_block'] = block_decl['block'] + + break + + log_print("dump MC state for block #{} during generation on node #{}:".format(mc_block['id'], node['id'])) + dump_mc_block_state(mc_block_state) + +# MC node notification about verification status +# todo: garbage collection for verification statuses +def mc_notify_verification_status(mc_node, verifier_node, block, status): + mc_state = mc_node['mc_state'] + verifications = mc_state['verifications'] + block_verifications = None + block_id = block['id'] + + if block_id in verifications: + block_verifications = verifications[block_id] + else: + block_verifications = { + 'is_suspicious': False, + 'verifiers': {}, + } + verifications[block_id] = block_verifications + + verifiers = block_verifications['verifiers'] + verifier_id = verifier_node['id'] + + if verifier_id in verifiers: + return #ignore duplicate verifications + + verification = { + 'status': status, + } + + verifiers[verifier_id] = verification + + if not status: + block_verifications['is_suspicious'] = True + +# Verification +def verify_block(node, block): + log_print("verifying block {}:{}:{} with ID #{} on node #{} (collated on node #{})".format(block['workchain_id'], block['shard_id'], block['height'], block['id'], node['id'], block['created_by']['id'])) + + status = validate_candidate(node, block) and not block['is_mallicious'] + + if random.uniform(0, 1) < MALLICIOUS_VERIFIER_THRESHOLD: + status = True + + put_message_to_workchain_queues(node['simulator_state']['workchains'][-1], lambda mc_node: mc_notify_verification_status(mc_node, node, block, status), + min_delay=MIN_VERIFICATION_DELIVERY_DELAY, max_delay=MAX_VERIFICATION_DELIVERY_DELAY, min_reliability=MIN_VERIFICATION_DELIVERY_RELIABILITY, max_reliability=MAX_VERIFICATION_DELIVERY_RELIABILITY) + +# Do simulation step for node +def update_node(node, time): + # Update candidate delivery neighbours + + if not node['candidate_delivery_neighbours'] or node['candidate_delivery_neighbours_next_rotation'] <= time: + # rotate neighbours + neighbours = [] + workchain = node['workchain'] + nodes_count = len(workchain['nodes']) + for i in range(CANDIDATE_DELIVERY_ACK_NEIGHBOURS_COUNT): + neighbour_id = random.randint(0, nodes_count-1) + neighbours.append(workchain['nodes'][neighbour_id]) + node['candidate_delivery_neighbours_next_rotation'] = get_time() + timedelta(milliseconds=CANDIDATE_DELIVERY_ACK_NEIGHBOURS_ROTATION_PERIOD) + node['candidate_delivery_neighbours'] = neighbours + + # Process queues + process_delayed_actions(node, time) + process_messages(node['message_queue'], node, time) + + # Do simplified BFT consensus iteration + round = node['round'] + + if not 'collation_time' in node: + #log_print("start first round for node #{}".format(node['id'])) + new_bft_round(node, time) + + round = node['round'] + + #while round in node['shard']['round_candidate_selection']: + # new_bft_round(node, time) + # round = node['round'] + + if time >= node['end_round_time']: + try_commit_empty_block(node, node['shard'], round) + elif time >= node['collation_time'] and not node['candidate_generated']: + node['candidate_generated'] = True + put_message_to_node_queue(node, lambda node: generate_candidate(node, round, get_time()), delay=scaled_rand(MIN_CANDIDATE_GENERATION_DELAY, MAX_CANDIDATE_GENERATION_DELAY)) + +################################################################################################### +# Shard +################################################################################################### + +# Create shard +def create_shard(shard_id, workchain, validators_count, simulator_state): + nodes = {} + shard = { + 'id': shard_id, + 'nodes': nodes, + 'workchain': workchain, + 'simulator_state': simulator_state, + 'block_height': -1, + 'round': 0, + 'round_candidate_selection': {}, #fake map to simplify BFT consensus phases + 'top_block': None, + 'shard_mc_dumps': {}, + } + + for i in range(validators_count): + source_id = i + nodes[i] = create_shard_node(shard_id, source_id, shard, workchain, simulator_state) + + return shard + +# Put message to shard nodes queues +def put_message_to_shard_queues(shard, msg, min_delay=0.0, max_delay=0.0, min_reliability=1.0, max_reliability=1.0): + for _, node in shard['nodes'].items(): + delay = scaled_rand(min_delay, max_delay) + reliability = scaled_rand(min_reliability, max_reliability) + put_message_to_node_queue(node, msg, delay=delay, reliability=reliability) + +# Do simulation step for shard +def update_shard(shard, time): + # Process nodes + for _, node in shard['nodes'].items(): + update_node(node, time) + +################################################################################################### +# Workchain +################################################################################################### + +# Create workchain +def create_workchain(workchain_id, shards_count, validators_count, simulator_state): + shards = {} + workchain = { + 'id' : workchain_id, + 'shards' : shards, + 'nodes': [], + 'node_by_id': {}, + 'simulator_state': simulator_state, + } + + # Initialize shards + + for i in range(shards_count): + shard_id = i + shards[i] = create_shard(shard_id, workchain, validators_count, simulator_state) + + return workchain + +# Put message to workchain's nodes' queues +def put_message_to_workchain_queues(workchain, msg, min_delay=0.0, max_delay=0.0, min_reliability=1.0, max_reliability=1.0): + for _, shard in workchain['shards'].items(): + put_message_to_shard_queues(shard, msg, min_delay=min_delay, max_delay=max_delay, min_reliability=min_reliability, max_reliability=max_reliability) + +# Do simulation step for workchain +def update_workchain(workchain, time): + for _, shard in workchain['shards'].items(): + update_shard(shard, time) + +################################################################################################### +# Simulator +################################################################################################### + +# Simulator initialization +def create_simulator(): + workchains = {} + simulator_state = { + 'workchains' : workchains, + 'node_id_counter': 0, + 'block_id_counter': 0, + } + + # Initialize workchains + + for i in range(WORKCHAINS_COUNT): + workchain_id = i + workchains[i] = create_workchain(workchain_id, SHARDS_COUNT, SHARD_VALIDATORS_COUNT, simulator_state) + + workchains[-1] = create_workchain(-1, 1, MC_VALIDATORS_COUNT, simulator_state) + + # Create simulation state + + return simulator_state + +# Simulation step +def update_simulator(state, time): + # Update workchains + for _, workchain in state['workchains'].items(): + update_workchain(workchain, time) + +################################################################################################### +# Main +################################################################################################### + +def main(): + global global_time + + # initialize simulator + state = create_simulator() + + # do simulation loops + start_time = datetime.fromtimestamp(0) + end_time = start_time + SIMULATION_TIME + time = start_time + next_dump_time = time + global_time = time + + while time <= end_time: + # Update simulator + update_simulator(state, time) + + # Dump simulator step + if time >= next_dump_time or time + SIMULATION_STEP > end_time: + time_delta = time_to_seconds(time - start_time) + end_time_delta = time_to_seconds(end_time - start_time) + log_print("{:.2f}% time is {:.2f}s".format(100.0 * time_delta / end_time_delta, time_delta)) + #plog_print(state) + next_dump_time += DUMP_STEP + #put_message_to_workchain_queues(state['workchains'][0], lambda node: log_print("node={} time={:.3f}s".format(node['id'], time_to_seconds(get_time()-start_time))), + # min_delay=0.2, max_delay=1.0, min_reliability=0.1, max_reliability=0.2) + + # Increment simulation time + time += SIMULATION_STEP + global_time = time + +main() diff --git a/src/validator/verification/utils.rs b/src/validator/verification/utils.rs new file mode 100644 index 00000000..a58032fe --- /dev/null +++ b/src/validator/verification/utils.rs @@ -0,0 +1,112 @@ +/* +* Copyright (C) 2019-2021 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +pub use super::*; + +use tokio::time::sleep; +use std::time::Duration; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use log::warn; +use ton_api::IntoBoxed; +use ton_types::KeyOption; + +/* + Constants +*/ + +const HANG_CHECKER_WARN_DUMP_PERIOD: Duration = Duration::from_millis(2000); //latency warning dump period + +/* +=================================================================================================== + HangChecker +=================================================================================================== +*/ + +pub struct HangCheck { + is_running: Arc, //is code still running +} + +impl HangCheck { + pub fn new(runtime: tokio::runtime::Handle, name: String, warn_delay: std::time::Duration) -> Self { + let is_running = Arc::new(AtomicBool::new(true)); + let is_running_clone = is_running.clone(); + let start_time = std::time::SystemTime::now(); + let warn_time = start_time + warn_delay; + + runtime.spawn(async move { + if let Ok(delay) = warn_time.duration_since(std::time::SystemTime::now()) { + sleep(delay).await; + } + + loop { + if !is_running.load(Ordering::Relaxed) { + break; + } + + let processing_delay = match start_time.elapsed() { + Ok(elapsed) => elapsed, + Err(_err) => std::time::Duration::default(), + }; + + warn!(target: "verificator", "{} is hanging for {:.3}s", name, processing_delay.as_secs_f64()); + + sleep(HANG_CHECKER_WARN_DUMP_PERIOD).await; + } + }); + + Self { + is_running: is_running_clone, + } + } +} + +impl Drop for HangCheck { + fn drop(&mut self) { + self.is_running.store(false, Ordering::Release); + } +} + +/* + Utils +*/ + +pub(crate) fn get_adnl_id(validator: &ValidatorDescr) -> Arc { + super::super::validator_utils::get_adnl_id(validator) + //ever_crypto::KeyId::from_data(validator.compute_node_id_short().inner()) +} + +pub(crate) fn into_public_key_tl(opt: &Arc) -> ton_types::Result { + let pub_key = opt.pub_key()?; + use ton_api::ton::pub_::publickey::Bls; + Ok(Bls { + bls_key: pub_key.to_vec() + }.into_boxed()) +} + +pub(crate) fn generate_test_bls_key(public_key: &Arc) -> Result> { + use ton_types::BlsKeyOption; + use ton_types::BLS_KEY_MATERIAL_LEN; + + log::debug!(target: "verificator", "Generate BLS key from public validator's key {}", public_key.id()); + + let public_key_data = public_key.pub_key()?; + let mut ikm: [u8; BLS_KEY_MATERIAL_LEN] = [0; BLS_KEY_MATERIAL_LEN]; + ikm.copy_from_slice(&public_key_data); + + let bls_key = Arc::new(BlsKeyOption::from_key_material(&ikm)?); + + log::debug!(target: "verificator", "BLS key generation is done for public validator's key {}", public_key.id()); + + Ok(bls_key) +} diff --git a/src/validator/verification/verification_manager.rs b/src/validator/verification/verification_manager.rs new file mode 100644 index 00000000..50c408b9 --- /dev/null +++ b/src/validator/verification/verification_manager.rs @@ -0,0 +1,510 @@ + +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +use super::workchain::Workchain; +use super::workchain::WorkchainPtr; +use super::utils::HangCheck; +use super::*; +use log::*; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::SystemTime; +use std::time::Duration; +use std::time::UNIX_EPOCH; +use spin::mutex::SpinMutex; +use ton_api::ton::ton_node::broadcast::BlockCandidateBroadcast; +use ton_types::Result; +use catchain::utils::MetricsDumper; +use catchain::utils::MetricsHandle; +use catchain::PublicKeyHash; +use metrics::Recorder; +use catchain::utils::compute_instance_counter; +use catchain::check_execution_time; +use catchain::utils::get_elapsed_time; + +/* + Constants +*/ + +const METRICS_DUMP_PERIOD_MS: u64 = 60000; //time for verification manager metrics dump + +/* +=============================================================================== + VerificationManagerImplImpl +=============================================================================== +*/ + +/// Hash map for workchains +type WorkchainMap = HashMap; + +/// Pointer to workchains map +type WorkchainMapPtr = Arc; + +/// Verficator manager +pub struct VerificationManagerImpl { + engine: EnginePtr, //pointer to engine + runtime: tokio::runtime::Handle, //runtime for spawns + workchains: Arc>, //map of workchains + should_stop_flag: Arc, //flag to indicate manager should be stopped + dump_thread_is_stopped_flag: Arc, //flag to indicate dump thread is stopped + metrics_receiver: MetricsHandle, //metrics receiver + blocks_instance_counter: Arc, //instance counter for blocks + workchains_instance_counter: Arc, //instance counter for workchains + wc_overlays_instance_counter: Arc, //instance counter for workchains WC overlays + mc_overlays_instance_counter: Arc, //instance counter for workchains MC overlays + send_new_block_candidate_counter: metrics::Counter, //counter for new candidates invocations + update_workchains_counter: metrics::Counter, //counter for workchains update invocations +} + +#[async_trait::async_trait] +impl VerificationManager for VerificationManagerImpl { + /* + Block candidates management + */ + + /// New block broadcast has been generated + async fn send_new_block_candidate(&self, candidate: &BlockCandidate) { + check_execution_time!(20_000); + + log::debug!(target:"verificator", "New block candidate has been generated {:?}", candidate.block_id); + + let _hang_checker = HangCheck::new(self.runtime.clone(), format!("VerificationManagerImpl::send_new_block_candidate: {:?}", candidate.block_id), Duration::from_millis(1000)); + + let workchain_id = candidate.block_id.shard_id.workchain_id(); + let workchain = match self.workchains.lock().get(&workchain_id) { + Some(workchain) => Some(workchain.clone()), + None => None, + }; + + if let Some(workchain) = workchain { + let block_id = candidate.block_id.clone(); + let timestamp = match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(n) => n.as_millis(), + Err(_) => panic!("SystemTime before UNIX EPOCH!"), + }; + let candidate = BlockCandidateBroadcast { + id: candidate.block_id.clone(), + data: candidate.data.clone().into(), + collated_data: candidate.collated_data.clone().into(), + collated_data_file_hash: candidate.collated_file_hash.clone().into(), + created_by: candidate.created_by.clone(), + created_timestamp: timestamp as i64, + }; + let workchain = workchain.clone(); + + self.send_new_block_candidate_counter.increment(1); + + if let Err(err) = self.runtime + .spawn(async move { + workchain.send_new_block_candidate(candidate); + }) + .await + { + error!(target: "verificator", "Error during new candidate {:?} send: {:?}", block_id, err); + } + } + } + + /// Get block status (delivered, rejected) + fn get_block_status( + &self, + block_id: &BlockIdExt, + ) -> (bool, bool) { + let workchain_id = block_id.shard_id.workchain_id(); + let workchain = match self.workchains.lock().get(&workchain_id) { + Some(workchain) => Some(workchain.clone()), + None => None, + }; + + if let Some(workchain) = workchain { + let candidate_id = + Workchain::get_candidate_id_impl(block_id); + + if let Some(block) = workchain.get_block_by_id(&candidate_id) { + return workchain.get_block_status(&block); + } + } + + (false, false) + } + + /// Wait for block verification + fn wait_for_block_verification( + &self, + block_id: &BlockIdExt, + timeout: &std::time::Duration, + ) -> bool { + log::trace!(target: "verificator", "Start block {} verification", block_id); + + let workchain_id = block_id.shard_id.workchain_id(); + let workchain = match self.workchains.lock().get(&workchain_id) { + Some(workchain) => Some(workchain.clone()), + None => None, + }; + + if let Some(workchain) = workchain { + let candidate_id = Workchain::get_candidate_id_impl(block_id); + let start_time = SystemTime::now(); + + loop { + //check block status + + if let Some(block) = workchain.get_block_by_id(&candidate_id) { + let (delivered, rejected) = workchain.get_block_status(&block); + + workchain.update_block_external_delivery_metrics(&candidate_id, &start_time); + + if rejected { + log::warn!(target: "verificator", "Finish block {} verification - NACK detected", block_id); + //TODO: initiate arbitrage & wait for result instead of returning false + return false; + } + + if delivered && !rejected { + log::trace!(target: "verificator", "Finish block {} verification - DELIVERED", block_id); + return true; + } + } + + //check for timeout + + let elapsed_time = get_elapsed_time(&start_time); + if elapsed_time > *timeout { + log::warn!(target: "verificator", "Finish block {} verification - timeout {}ms expired", block_id, elapsed_time.as_millis()); + workchain.update_block_external_delivery_metrics(&candidate_id, &start_time); + return false; + } + } + } + + log::warn!(target: "verificator", "Finish block {} verification - no workchain found", block_id); + + false + } + + /* + Workchains management + */ + + /// Reset workchains + async fn reset_workchains<'a>( + &'a self, + ) { + check_execution_time!(100_000); + + let _hang_checker = HangCheck::new(self.runtime.clone(), "VerificationManagerImpl::reset_workchains".to_string(), Duration::from_millis(1000)); + + trace!(target: "verificator", "Reset workchains"); + + self.update_workchains_counter.increment(1); + + self.set_workchains(Arc::new(HashMap::new())); + } + + /// Update workchains + async fn update_workchains<'a>( + &'a self, + local_key_id: PublicKeyHash, + local_bls_key: PrivateKey, + workchain_id: i32, + utime_since: u32, + workchain_validators: &'a Vec, + mc_validators: &'a Vec, + listener: &'a VerificationListenerPtr, + ) { + check_execution_time!(100_000); + + let _hang_checker = HangCheck::new(self.runtime.clone(), "VerificationManagerImpl::update_workchains".to_string(), Duration::from_millis(1000)); + + trace!(target: "verificator", "Update workchains"); + + self.update_workchains_counter.increment(1); + + //create workchains + + let engine = self.engine.clone(); + let current_workchains = self.get_workchains(); + let mut new_workchains = HashMap::new(); + + match Self::get_workchain_by_validator_set( + &engine, + self.runtime.clone(), + &local_key_id, + &local_bls_key, + ¤t_workchains, + workchain_id, + utime_since, + &workchain_validators, + mc_validators, + listener, + self.metrics_receiver.clone(), + self.workchains_instance_counter.clone(), + self.blocks_instance_counter.clone(), + self.wc_overlays_instance_counter.clone(), + self.mc_overlays_instance_counter.clone(), + ) + .await + { + Ok(workchain) => { + new_workchains.insert(workchain_id, workchain); + } + Err(err) => { + error!(target: "verificator", "Can't create workchain: {:?}", err); + } + }; + + //replace workchains cache with a new one + + self.set_workchains(Arc::new(new_workchains)); + } +} + +impl VerificationManagerImpl { + /* + Workchains management + */ + + /// Get workchains map + fn get_workchains(&self) -> WorkchainMapPtr { + self.workchains.lock().clone() + } + + /// Set workchains map + fn set_workchains(&self, workchains: WorkchainMapPtr) { + *self.workchains.lock() = workchains; + } + + /// Compute validator set hash based on a validators list + fn compute_validator_set_hash(utime_since: u32, validators: &Vec) -> UInt256 { + let mut result = Vec::::with_capacity(validators.len() * 32); + + for validator in validators { + result.extend(validator.public_key.key_bytes()); + } + + result.extend(utime_since.to_le_bytes()); + + UInt256::calc_file_hash(&result) + } + + async fn get_workchain_by_validator_set( + engine: &EnginePtr, + runtime: tokio::runtime::Handle, + local_key_id: &PublicKeyHash, + local_bls_key: &PrivateKey, + workchains: &WorkchainMapPtr, + workchain_id: i32, + utime_since: u32, + wc_validators: &Vec, + mc_validators: &Vec, + listener: &VerificationListenerPtr, + metrics_receiver: MetricsHandle, + workchains_instance_counter: Arc, + blocks_instance_counter: Arc, + wc_overlays_instance_counter: Arc, + mc_overlays_instance_counter: Arc, + ) -> Result { + let wc_validator_set_hash = Self::compute_validator_set_hash(utime_since, wc_validators); + let mc_validator_set_hash = Self::compute_validator_set_hash(utime_since, mc_validators); + + //try to find workchain in a cache based on its ID and hash + + if let Some(workchain) = workchains.get(&workchain_id) { + if workchain.get_wc_validator_set_hash() == &wc_validator_set_hash && + workchain.get_mc_validator_set_hash() == &mc_validator_set_hash { + return Ok(workchain.clone()); + } + } + + //create new workchain + + let workchain = Workchain::create( + engine.clone(), + runtime, + workchain_id, + wc_validators.clone(), + mc_validators.clone(), + wc_validator_set_hash, + mc_validator_set_hash, + local_key_id, + local_bls_key, + listener.clone(), + metrics_receiver, + workchains_instance_counter, + blocks_instance_counter, + wc_overlays_instance_counter, + mc_overlays_instance_counter, + ) + .await?; + + Ok(workchain) + } + + /* + Self-diagnostic + */ + + fn run_metrics_dumper( + should_stop_flag: Arc, + is_stopped_flag: Arc, + metrics_receiver: MetricsHandle, + workchains: Arc>, + ) { + let builder = std::thread::Builder::new(); + + let _ = builder.spawn(move || { + let mut metrics_dumper = MetricsDumper::new(); + let mut workchain_metrics_dumpers: HashMap = HashMap::new(); + + metrics_dumper.add_compute_handler("smft_block".to_string(), &compute_instance_counter); + metrics_dumper.add_compute_handler("smft_workchains".to_string(), &compute_instance_counter); + metrics_dumper.add_compute_handler("smft_wc_overlays".to_string(), &compute_instance_counter); + metrics_dumper.add_compute_handler("smft_mc_overlays".to_string(), &compute_instance_counter); + + metrics_dumper.add_derivative_metric("smft_block".to_string()); + metrics_dumper.add_derivative_metric("smft_workchains".to_string()); + + let mut next_metrics_dump_time = SystemTime::now() + Duration::from_millis(METRICS_DUMP_PERIOD_MS); + let mut loop_idx = 0; + + loop { + if should_stop_flag.load(Ordering::SeqCst) { + break; + } + + const CHECKING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(300); + + std::thread::sleep(CHECKING_INTERVAL); + + if let Err(_err) = next_metrics_dump_time.elapsed() { + continue; + } + + if !log_enabled!(target: "verificator", log::Level::Debug) { + continue; + } + + metrics_dumper.update(&metrics_receiver); + + debug!(target: "verificator", "Verification manager metrics:"); + + metrics_dumper.dump(|string| { + debug!(target: "verificator", "{}", string); + }); + + metrics_dumper.enumerate_as_f64(|key, value| { + metrics::gauge!(key.replace(".", "_"), value); + }); + + let current_workchains = workchains.lock().clone(); + + for (workchain_id, workchain) in current_workchains.iter() { + let mut dumper = workchain_metrics_dumpers.get_mut(workchain_id); + + if dumper.is_none() { + let mut metrics_dumper = MetricsDumper::new(); + + workchain.configure_dumper(&mut metrics_dumper); + + workchain_metrics_dumpers.insert(*workchain_id, (metrics_dumper, loop_idx)); + dumper = workchain_metrics_dumpers.get_mut(workchain_id); + } + + let dumper = dumper.unwrap(); + + let (workchain_metrics_dumper, updated_loop_idx) = dumper; + + *updated_loop_idx = loop_idx; + + workchain_metrics_dumper.update(&metrics_receiver); + + workchain_metrics_dumper.dump(|string| { + debug!(target: "verificator", "{}", string); + }); + } + + workchain_metrics_dumpers.retain(|_, dumper| dumper.1 == loop_idx); + + next_metrics_dump_time = SystemTime::now() + Duration::from_millis(METRICS_DUMP_PERIOD_MS); + loop_idx += 1; + } + + debug!(target: "verificator", "Verification manager dump loop is finished"); + + is_stopped_flag.store(true, Ordering::SeqCst); + }).unwrap(); + } + + /* + Stopping + */ + + pub fn stop(&self) { + log::info!(target: "verificator", "Stopping verification manager"); + + self.should_stop_flag.store(true, Ordering::SeqCst); + + loop { + if self.dump_thread_is_stopped_flag.load(Ordering::SeqCst) { + break; + } + + const MAX_WAIT : std::time::Duration = std::time::Duration::from_millis(500); + + std::thread::sleep(MAX_WAIT); + + debug!(target: "verificator", "...waiting for verification manager stopping"); + } + + log::info!(target: "verificator", "Verification manager has been stopped"); + } + + /* + Constructor + */ + + pub fn create(engine: EnginePtr, runtime: tokio::runtime::Handle) -> Arc { + log::info!(target: "verificator", "Creating verification manager"); + + let workchains = Arc::new(SpinMutex::new(Arc::new(HashMap::new()))); + let metrics_receiver = MetricsHandle::new(Some(Duration::from_secs(30))); + let should_stop_flag = Arc::new(AtomicBool::new(false)); + let dump_thread_is_stopped_flag = Arc::new(AtomicBool::new(false)); + + let body = Self { + engine, + runtime, + workchains: workchains.clone(), + metrics_receiver: metrics_receiver.clone(), + send_new_block_candidate_counter: metrics_receiver.sink().register_counter(&"smft_candidates".into()), + update_workchains_counter: metrics_receiver.sink().register_counter(&"smft_workchains_updates".into()), + blocks_instance_counter: Arc::new(InstanceCounter::new(&metrics_receiver, &"smft_block".to_string())), + workchains_instance_counter: Arc::new(InstanceCounter::new(&metrics_receiver, &"smft_workchains".to_string())), + wc_overlays_instance_counter: Arc::new(InstanceCounter::new(&metrics_receiver, &"smft_wc_overlays".to_string())), + mc_overlays_instance_counter: Arc::new(InstanceCounter::new(&metrics_receiver, &"smft_mc_overlays".to_string())), + should_stop_flag: should_stop_flag.clone(), + dump_thread_is_stopped_flag: dump_thread_is_stopped_flag.clone(), + }; + + Self::run_metrics_dumper(should_stop_flag, dump_thread_is_stopped_flag, metrics_receiver, workchains); + + Arc::new(body) + } +} + +impl Drop for VerificationManagerImpl { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/src/validator/verification/workchain.rs b/src/validator/verification/workchain.rs new file mode 100644 index 00000000..41bb3534 --- /dev/null +++ b/src/validator/verification/workchain.rs @@ -0,0 +1,1206 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +use super::block::Block; +use super::block::BlockCandidateBody; +use super::block::BlockPtr; +use super::block::MultiSignature; +use super::workchain_overlay::WorkchainOverlay; +use super::workchain_overlay::WorkchainOverlayListener; +use super::*; +use super::utils::HangCheck; +use super::utils::get_adnl_id; +use crate::validator::validator_utils::sigpubkey_to_publickey; +use catchain::BlockPayloadPtr; +use catchain::PublicKeyHash; +use catchain::profiling::ResultStatusCounter; +use catchain::profiling::InstanceCounter; +use catchain::check_execution_time; +use log::*; +use rand::Rng; +use tokio::time::sleep; +use std::time::SystemTime; +use std::time::Duration; +use spin::mutex::SpinMutex; +use ton_api::ton::ton_node::blockcandidatestatus::BlockCandidateStatus; +use ton_api::ton::ton_node::broadcast::BlockCandidateBroadcast; +use ton_types::Result; +use validator_session::ValidatorWeight; +use catchain::utils::MetricsDumper; +use catchain::utils::MetricsHandle; +use metrics::Recorder; +use catchain::utils::add_compute_relative_metric; +use catchain::utils::add_compute_result_metric; +use ton_types::bls::BLS_PUBLIC_KEY_LEN; + +//TODO: cutoff weight configuration +//TODO: neighbours mode configuration +//TODO: force request block from MC + +/* +=============================================================================== + Constants +=============================================================================== +*/ + +const BLOCK_SYNC_MIN_PERIOD_MS: u64 = 300; //min time for block sync +const BLOCK_SYNC_MAX_PERIOD_MS: u64 = 400; //max time for block sync +const NEIGHBOURS_SYNC_MIN_PERIOD_MS: u64 = 10000; //min time for sync with neighbour nodes +const NEIGHBOURS_SYNC_MAX_PERIOD_MS: u64 = 20000; //max time for sync with neighbour nodes +const BLOCK_LIFETIME_PERIOD: Duration = Duration::from_secs(60); //block's lifetime +const VERIFICATION_OBLIGATION_CUTOFF: f64 = 0.2; //cutoff for validator obligation to verify [0..1] + +/* +=============================================================================== + Workchain +=============================================================================== +*/ + +pub type WorkchainPtr = Arc; + +//todo: hide fields within module +pub struct Workchain { + runtime: tokio::runtime::Handle, //runtime handle for spawns + wc_validator_set_hash: UInt256, //hash of validators set for WC + mc_validator_set_hash: UInt256, //hash of validators set for MC + wc_validators: Vec, //WC validators + wc_pub_keys: Vec<[u8; BLS_PUBLIC_KEY_LEN]>, //WC validators pubkeys + local_adnl_id: PublicKeyHash, //ADNL ID for this node + wc_local_idx: i16, //local index in WC validator set + mc_local_idx: i16, //local index in MC validator set + workchain_id: i32, //workchain identifier + self_weak_ref: SpinMutex>>, //self weak reference + wc_cutoff_weight: ValidatorWeight, //cutoff weight for consensus in WC + local_bls_key: PrivateKey, //private BLS key + local_id: PublicKeyHash, //local ID for this node + workchain_overlay: SpinMutex>>, //workchain overlay + mc_overlay: SpinMutex>>, //MC overlay + blocks: SpinMutex>, //blocks + listener: VerificationListenerPtr, //verification listener + node_debug_id: Arc, //node debug ID for workchain + _workchains_instance_counter: InstanceCounter, //workchain instances counter + blocks_instance_counter: Arc, //instance counter for blocks + merge_block_status_counter: metrics::Counter, //counter for block updates (via merge with other nodes statuses) + set_block_status_counter: metrics::Counter, //counter for set block status (by local node) + process_block_candidate_counter: metrics::Counter, //counter for block candidates processings + process_block_status_counter: metrics::Counter, //counter for block statuses processings + new_block_candidate_counter: metrics::Counter, //counter for new block candidates + send_block_status_to_mc_counter: metrics::Counter, //counter of sendings block status to MC + send_block_status_counter: metrics::Counter, //counter of sendings block status within workchain + external_request_counter: metrics::Counter, //counter of external requests for a block + external_request_delivered_blocks_counter: metrics::Counter, //counter of external requests for delivered blocks + external_request_approved_blocks_counter: metrics::Counter, //counter of external requests for approved block + external_request_rejected_blocks_counter: metrics::Counter, //counter of external requests for rejected blocks + verify_block_counter: ResultStatusCounter, //counter for block verifications + block_status_received_in_mc_counter: ResultStatusCounter, //counter for block receivings in MC + block_status_send_to_mc_latency_histogram: metrics::Histogram, //histogram for block candidate sending to MC + block_status_received_in_mc_latency_histogram: metrics::Histogram, //histogram for block candidate receiving in MC + candidate_delivered_to_wc_latency_histogram: metrics::Histogram, //histogram for block candidate receiving in WC + block_status_merges_count_histogram: metrics::Histogram, //histogram for block candidate merges count (hops count) + block_external_request_delays_histogram: metrics::Histogram, //histogram for block external request delays +} + +impl Workchain { + /* + Initialization + */ + + fn bls_key_to_string(key: &Option<[u8; BLS_PUBLIC_KEY_LEN]>) -> String { + match key { + None => "N/A".to_string(), + Some(key) => hex::encode(&key), + } + } + + fn get_overlay_id(workchain_id: i32, wc_validator_set_hash: &UInt256, mc_validator_set_hash: &UInt256, tag: u32) -> UInt256 { + let magic_suffix = [0xff, 0xbe, 0x45, 0x23]; //magic suffix to create unique hash different from public overlay hashes + let mut overlay_id = Vec::new(); + + overlay_id.extend_from_slice(&magic_suffix); + overlay_id.extend_from_slice(&tag.to_le_bytes()); + overlay_id.extend_from_slice(&workchain_id.to_le_bytes()); + overlay_id.extend_from_slice(wc_validator_set_hash.as_slice()); + overlay_id.extend_from_slice(mc_validator_set_hash.as_slice()); + + UInt256::calc_file_hash(&overlay_id) + } + + pub async fn create( + engine: EnginePtr, + runtime: tokio::runtime::Handle, + workchain_id: i32, + mut wc_validators: Vec, + mc_validators: Vec, + wc_validator_set_hash: UInt256, + mc_validator_set_hash: UInt256, + local_id: &PublicKeyHash, + local_bls_key: &PrivateKey, + listener: VerificationListenerPtr, + metrics_receiver: MetricsHandle, + workchains_instance_counter: Arc, + blocks_instance_counter: Arc, + wc_overlays_instance_counter: Arc, + mc_overlays_instance_counter: Arc, + ) -> Result> { + let mut wc_local_idx = -1; + let mut mc_local_idx = -1; + let mut local_adnl_id = None; + + for (idx, desc) in wc_validators.iter().enumerate() { + let public_key = sigpubkey_to_publickey(&desc.public_key); + + if public_key.id() == local_id { + wc_local_idx = idx as i16; + local_adnl_id = Some(get_adnl_id(&desc)); + break; + } + } + + for (idx, desc) in mc_validators.iter().enumerate() { + let public_key = sigpubkey_to_publickey(&desc.public_key); + + if public_key.id() == local_id { + mc_local_idx = idx as i16; + local_adnl_id = Some(get_adnl_id(&desc)); + break; + } + } + + if local_adnl_id.is_none() { + failure::bail!("local_adnl_id must exist for workchain {}", workchain_id); + } + + let local_adnl_id = local_adnl_id.as_ref().expect("local_adnl_id must exist").clone(); + let node_debug_id = Arc::new(format!("#{}.{}", workchain_id, local_adnl_id)); + + let wc_total_weight: ValidatorWeight = wc_validators.iter().map(|desc| desc.weight).sum(); + let wc_cutoff_weight = wc_total_weight * 2 / 3 + 1; + + let mut wc_pub_keys = Vec::new(); + + log::info!(target: "verificator", "Creating verification workchain {} (wc_validator_set_hash={}, mc_validator_set_hash={}) with {} workchain nodes (total_weight={}, cutoff_weight={}, wc_local_idx={}, mc_local_idx={})", + node_debug_id, + wc_validator_set_hash.to_hex_string(), + mc_validator_set_hash.to_hex_string(), + wc_validators.len(), + wc_total_weight, + wc_cutoff_weight, + wc_local_idx, + mc_local_idx); + + let wc_validators_count = wc_validators.len(); + + for (i, desc) in wc_validators.iter_mut().enumerate() { + let adnl_id = get_adnl_id(&desc); + //let adnl_id = desc.adnl_addr.clone().map_or("** no-addr **".to_string(), |x| x.to_hex_string()); + let public_key = sigpubkey_to_publickey(&desc.public_key); + let mut bls_public_key = desc.bls_public_key.clone(); + + if bls_public_key.is_none() && GENERATE_MISSING_BLS_KEY { + match utils::generate_test_bls_key(&public_key) { + Ok(bls_key) => { + match bls_key.pub_key() { + Ok(bls_key) => { + let bls_key_data: [u8; BLS_PUBLIC_KEY_LEN] = bls_key.to_vec().try_into().unwrap_or_else(|v: Vec| panic!("Expected a Vec of length {} but it was {}", BLS_PUBLIC_KEY_LEN, v.len())); + bls_public_key = Some(bls_key_data); + desc.bls_public_key = bls_public_key; + }, + Err(err) => log::error!(target: "verificator", "Can't generate test BLS key (can't extract pub key data): {:?}", err), + } + }, + Err(err) => log::error!(target: "verificator", "Can't generate test BLS key: {:?}", err), + } + } + + if log::log_enabled!(target: "verificator", log::Level::Debug) { + let serialized_pub_key_tl : ton_api::ton::bytes = catchain::serialize_tl_boxed_object!(&utils::into_public_key_tl(&public_key).unwrap()); + + log::debug!(target: "verificator", "...node {}#{}/{} for workchain {}: public_key={}, public_key_bls={}, adnl_id={}, weight={} ({:.2}%)", + if local_id == public_key.id() { ">" } else { " " }, + i, wc_validators_count, node_debug_id, + &hex::encode(&serialized_pub_key_tl), + Self::bls_key_to_string(&bls_public_key), + adnl_id, + desc.weight, + desc.weight as f64 / wc_total_weight as f64 * 100.0); + } + + wc_pub_keys.push(match bls_public_key { + Some(bls_public_key) => bls_public_key.clone().into(), + None => [0; BLS_PUBLIC_KEY_LEN], + }); + } + + let mc_total_weight: ValidatorWeight = mc_validators.iter().map(|desc| desc.weight).sum(); + let mc_cutoff_weight = mc_total_weight * 2 / 3 + 1; + + log::debug!(target: "verificator", "Workchain {} (wc_validator_set_hash={}, mc_validator_set_hash={}) has {} linked MC nodes (total_weight={}, cutoff_weight={})", + node_debug_id, + wc_validator_set_hash.to_hex_string(), + mc_validator_set_hash.to_hex_string(), + mc_validators.len(), + mc_total_weight, + mc_cutoff_weight); + + for (i, desc) in mc_validators.iter().enumerate() { + let adnl_id = get_adnl_id(&desc); + //let adnl_id = desc.adnl_addr.clone().map_or("** no-addr **".to_string(), |x| x.to_hex_string()); + let public_key = sigpubkey_to_publickey(&desc.public_key); + + if log::log_enabled!(target: "verificator", log::Level::Debug) { + let serialized_pub_key_tl : ton_api::ton::bytes = catchain::serialize_tl_boxed_object!(&utils::into_public_key_tl(&public_key).unwrap()); + + log::debug!(target: "verificator", "...MC node {}#{}/{} for workchain {}: public_key={}, adnl_id={}, weight={} ({:.2}%)", + if local_id == public_key.id() { ">" } else { " " }, + i, mc_validators.len(), node_debug_id, + &hex::encode(&serialized_pub_key_tl), + adnl_id, + desc.weight, + desc.weight as f64 / mc_total_weight as f64 * 100.0); + } + } + + let workchain = Self { + workchain_id, + node_debug_id, + runtime: runtime.clone(), + wc_validators, + wc_validator_set_hash, + mc_validator_set_hash, + wc_cutoff_weight, + local_bls_key: local_bls_key.clone(), + local_adnl_id: local_adnl_id.clone().into(), + local_id: local_id.clone(), + wc_local_idx, + mc_local_idx, + wc_pub_keys, + blocks: SpinMutex::new(HashMap::new()), + mc_overlay: SpinMutex::new(None), + workchain_overlay: SpinMutex::new(None), + listener, + self_weak_ref: SpinMutex::new(None), + _workchains_instance_counter: (*workchains_instance_counter).clone(), + blocks_instance_counter, + merge_block_status_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_status_merges", workchain_id).into()), + set_block_status_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_status_sets", workchain_id).into()), + process_block_candidate_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_candidate_processings", workchain_id).into()), + process_block_status_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_status_processings", workchain_id).into()), + new_block_candidate_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_new_block_candidates", workchain_id).into()), + send_block_status_to_mc_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_status_to_mc_sends", workchain_id).into()), + send_block_status_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_block_status_within_wc_sends", workchain_id).into()), + external_request_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_mc_requests", workchain_id).into()), + external_request_delivered_blocks_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_mc_delivered", workchain_id).into()), + external_request_approved_blocks_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_mc_approved", workchain_id).into()), + external_request_rejected_blocks_counter: metrics_receiver.sink().register_counter(&format!("smft_wc{}_mc_rejected", workchain_id).into()), + block_status_received_in_mc_counter: ResultStatusCounter::new(&metrics_receiver, &format!("smft_wc{}_block_status_received_in_mc", workchain_id)), + verify_block_counter: ResultStatusCounter::new(&metrics_receiver, &format!("smft_wc{}_block_candidate_verifications", workchain_id)), + candidate_delivered_to_wc_latency_histogram: metrics_receiver.sink().register_histogram(&format!("time:smft_wc{}_stage1_block_candidate_delivered_in_wc", workchain_id).into()), + block_status_send_to_mc_latency_histogram: metrics_receiver.sink().register_histogram(&format!("time:smft_wc{}_stage2_block_status_send_to_mc_latency", workchain_id).into()), + block_status_received_in_mc_latency_histogram: metrics_receiver.sink().register_histogram(&format!("time:smft_wc{}_stage3_block_status_received_in_mc_latency", workchain_id).into()), + block_status_merges_count_histogram: metrics_receiver.sink().register_histogram(&format!("smft_wc{}_block_status_merges_count", workchain_id).into()), + block_external_request_delays_histogram: metrics_receiver.sink().register_histogram(&format!("time:smft_wc{}_block_mc_delay", workchain_id).into()), + }; + let workchain = Arc::new(workchain); + + //set self weak reference + + *workchain.self_weak_ref.lock() = Some(Arc::downgrade(&workchain)); + + //start overlay for interactions with MC + + const WC_OVERLAY_TAG: u32 = 1; + const MC_OVERLAY_TAG: u32 = 2; + + let wc_overlay_id = Self::get_overlay_id(workchain.workchain_id, &workchain.wc_validator_set_hash, &workchain.mc_validator_set_hash, WC_OVERLAY_TAG); + let mc_overlay_id = Self::get_overlay_id(workchain.workchain_id, &workchain.wc_validator_set_hash, &workchain.mc_validator_set_hash, MC_OVERLAY_TAG); + + let mut full_validators = mc_validators.clone(); + full_validators.append(&mut workchain.wc_validators.clone()); + + //TODO: exclude duplicates from full_validators list!!! + + let mc_overlay_listener: Arc = workchain.clone(); + let mc_overlay = WorkchainOverlay::create( + workchain.workchain_id, + format!("MC[{}]{}", workchain.mc_local_idx, *workchain.node_debug_id), + mc_overlay_id, + &full_validators, + mc_validators.len(), //only part of nodes are active + workchain.local_adnl_id.clone(), + Arc::downgrade(&mc_overlay_listener), + &engine, + runtime.clone(), + metrics_receiver.clone(), + mc_overlays_instance_counter, + format!("smft_mc{}_overlay", workchain_id), + true, + ).await?; + *workchain.mc_overlay.lock() = Some(mc_overlay); + + if wc_local_idx != -1 { + //start overlay for private interactions + + let workchain_overlay_listener: Arc = workchain.clone(); + let workchain_overlay = WorkchainOverlay::create( + workchain.workchain_id, + format!("WC[{}]{}", workchain.wc_local_idx, *workchain.node_debug_id), + wc_overlay_id, + &workchain.wc_validators, + workchain.wc_validators.len(), + workchain.local_adnl_id.clone(), + Arc::downgrade(&workchain_overlay_listener), + &engine, + runtime.clone(), + metrics_receiver, + wc_overlays_instance_counter, + format!("smft_wc{}_overlay", workchain_id), + false, + ).await?; + *workchain.workchain_overlay.lock() = Some(workchain_overlay); + } + + Ok(workchain) + } + + /* + Dumper + */ + + pub fn configure_dumper(&self, metrics_dumper: &mut MetricsDumper) { + log::debug!(target: "verificator", "Creating verification workchain {} metrics dumper", self.node_debug_id); + + let workchain_id = self.workchain_id; + + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_candidate_verifications.total", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_candidate_verifications.success", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_candidate_verifications.failure", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_new_block_candidates", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_mc{}_overlay_in_queries", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_mc{}_overlay_out_queries.total", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_mc{}_overlay_in_messages", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_mc{}_overlay_out_messages", workchain_id)); + + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_status_processings", workchain_id)); + + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_status_received_in_mc.total", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_status_received_in_mc.success", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_block_status_received_in_mc.failure", workchain_id)); + + add_compute_result_metric(metrics_dumper, &format!("smft_wc{}_block_candidate_verifications", workchain_id)); + add_compute_result_metric(metrics_dumper, &format!("smft_mc{}_overlay_out_queries", workchain_id)); + + add_compute_relative_metric( + metrics_dumper, + &format!("smft_wc{}_merges_per_block", workchain_id), + &format!("smft_wc{}_block_status_merges", workchain_id), + &format!("smft_wc{}_new_block_candidates", workchain_id), + 0.0, + ); + + add_compute_relative_metric( + metrics_dumper, + &format!("smft_wc{}_updates_per_mc_send", workchain_id), + &format!("smft_wc{}_block_status_processings", workchain_id), + &format!("smft_wc{}_block_status_to_mc_sends", workchain_id), + 0.0, + ); + + add_compute_relative_metric( + metrics_dumper, + &format!("smft_wc{}_mc_sends_per_block_candidate", workchain_id), + &format!("smft_wc{}_block_status_to_mc_sends", workchain_id), + &format!("smft_wc{}_new_block_candidates", workchain_id), + 0.0, + ); + + use catchain::utils::add_compute_percentage_metric; + + add_compute_percentage_metric( + metrics_dumper, + &format!("smft_wc{}_mc_delivered.frequency", workchain_id), + &format!("smft_wc{}_mc_delivered", workchain_id), + &format!("smft_wc{}_mc_requests", workchain_id), + 0.0, + ); + + add_compute_percentage_metric( + metrics_dumper, + &format!("smft_wc{}_mc_rejected.frequency", workchain_id), + &format!("smft_wc{}_mc_rejected", workchain_id), + &format!("smft_wc{}_mc_requests", workchain_id), + 0.0, + ); + + add_compute_percentage_metric( + metrics_dumper, + &format!("smft_wc{}_mc_approved.frequency", workchain_id), + &format!("smft_wc{}_mc_approved", workchain_id), + &format!("smft_wc{}_mc_requests", workchain_id), + 0.0, + ); + + if workchain_id != -1 { + add_compute_result_metric(metrics_dumper, &format!("smft_wc{}_overlay_out_queries", workchain_id)); + + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_in_broadcasts", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_out_broadcasts", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_in_queries", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_in_block_candidates", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_out_queries.total", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_send_message_to_neighbours_calls", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_in_messages", workchain_id)); + metrics_dumper.add_derivative_metric(format!("smft_wc{}_overlay_out_messages", workchain_id)); + } + } + + + /* + Common methods + */ + + /// Workchain validator set hash for WC + pub fn get_wc_validator_set_hash(&self) -> &UInt256 { + &self.wc_validator_set_hash + } + + /// Workchain validator set hash for MC + pub fn get_mc_validator_set_hash(&self) -> &UInt256 { + &self.mc_validator_set_hash + } + + /// Get self weak reference + fn get_self(&self) -> WorkchainPtr { + self.self_weak_ref + .lock() + .clone() + .expect("Self ref must be set") + .upgrade() + .expect("Self ref must exist") + } + + /* + Block management + */ + + /// Block status (delivered, rejected) + pub fn get_block_status(&self, block: &BlockPtr) -> (bool, bool) { + let block = block.lock(); + let is_delivered = block.is_delivered(&self.wc_validators, self.wc_cutoff_weight); + let is_rejected = block.is_rejected(); + + (is_delivered, is_rejected) + } + + /// Should block be sent to MC + fn should_send_to_mc(&self, block: &BlockPtr) -> bool { + let (is_delivered, is_rejected) = self.get_block_status(block); + + is_rejected || is_delivered + } + + /// Get block by its ID + pub fn get_block_by_id(&self, candidate_id: &UInt256) -> Option { + Self::get_block_by_id_impl(&self.blocks.lock(), candidate_id) + } + + /// Get block by its ID without lock + fn get_block_by_id_impl(blocks: &HashMap, candidate_id: &UInt256) -> Option { + match blocks.get(&candidate_id) { + Some(block) => Some(block.clone()), + None => None, + } + } + + /// Put new block to map + fn add_block_impl( + &self, + candidate_id: &UInt256, + block_candidate: Option>, + ) -> BlockPtr { + check_execution_time!(1_000); + + let block = { + let mut blocks = self.blocks.lock(); + + match Self::get_block_by_id_impl(&blocks, candidate_id) { + Some(existing_block) => existing_block, + None => { + trace!(target: "verificator", "Creating new block {:?} for workchain {}", candidate_id, self.node_debug_id); + + let new_block = Block::create(candidate_id.clone(), &*self.blocks_instance_counter); + + blocks.insert(candidate_id.clone(), new_block.clone()); + + drop(blocks); //to release lock + + self.start_synchronizing_block(&new_block); + + new_block + } + } + }; + + if let Some(block_candidate) = block_candidate { + let status = block.lock().update_block_candidate(block_candidate.clone()); + + trace!(target: "verificator", "Block {:?} status is {} (node={})", candidate_id, status, self.node_debug_id); + + if status + { + trace!(target: "verificator", "Block candidate {} is delivered (node={})", candidate_id, self.node_debug_id); + + //measure latency for initial delivery + + let latency = block.lock().get_delivery_latency(); + if let Some(latency) = latency { + self.candidate_delivered_to_wc_latency_histogram.record(latency.as_millis() as f64); + } + + //set block status to delivered + + self.set_block_status(&candidate_id, None); + + //initiate verification of the block + + self.verify_block(candidate_id, block_candidate); + } + } + + block + } + + /// Remove block + fn remove_block(&self, candidate_id: &UInt256) { + trace!(target: "verificator", "Remove block {:?} for workchain {}", candidate_id, self.node_debug_id); + + self.blocks.lock().remove(candidate_id); + } + + /// Block update function + fn synchronize_block(workchain_weak: Weak, block_weak: Weak>, neighbours_sync_time: Option) { + let workchain = { + if let Some(workchain) = workchain_weak.upgrade() { + workchain + } else { + return; + } + }; + let block = { + if let Some(block) = block_weak.upgrade() { + block + } else { + return; + } + }; + + let candidate_id = block.lock().get_id().clone(); + + //remove old blocks + + let block_end_of_life_time = *block.lock().get_first_appearance_time() + BLOCK_LIFETIME_PERIOD; + + if let Ok(_) = block_end_of_life_time.elapsed() { + workchain.remove_block(&candidate_id); + return; //prevent all further sync logic because the block is expired + } + + //trace!(target: "verificator", "Synchronize block {:?} for workchain {}", candidate_id, workchain.node_debug_id); + trace!(target: "verificator", "Synchronize block {:?} for workchain {}: {:?}", candidate_id, workchain.node_debug_id, block.lock()); + + let mut rng = rand::thread_rng(); + let delay = Duration::from_millis(rng.gen_range( + BLOCK_SYNC_MIN_PERIOD_MS, + BLOCK_SYNC_MAX_PERIOD_MS + 1, + )); + let next_sync_time = SystemTime::now() + delay; + let neighbours_force_sync = neighbours_sync_time.is_some() && neighbours_sync_time.unwrap().elapsed().is_ok(); + let neighbours_sync_time = { + if neighbours_sync_time.is_none() || neighbours_force_sync { + Some(SystemTime::now() + Duration::from_millis(rng.gen_range(NEIGHBOURS_SYNC_MIN_PERIOD_MS, NEIGHBOURS_SYNC_MAX_PERIOD_MS + 1))) + } else { + neighbours_sync_time + } + }; + + let workchain_id = workchain.workchain_id; + let node_debug_id = workchain.node_debug_id.clone(); + + workchain.runtime.spawn(async move { + if let Ok(timeout) = next_sync_time.duration_since(SystemTime::now()) { + /*trace!( + target: "verificator", + "Next block {:?} synchronization for workchain's #{} private overlay is scheduled at {} (in {:.3}s from now; overlay={})", + candidate_id, + workchain_id, + catchain::utils::time_to_string(&next_sync_time), + timeout.as_secs_f64(), + node_debug_id);*/ + + sleep(timeout).await; + } + + //synchronize block + + Self::synchronize_block(workchain_weak, block_weak, neighbours_sync_time); + }); + + //periodically force push block status to neighbours + + if neighbours_force_sync { + let (is_delivered, _is_rejected) = workchain.get_block_status(&block); + + if !is_delivered { + trace!( + target: "verificator", + "Force block {:?} synchronization for workchain's #{} private overlay (overlay={})", + candidate_id, + workchain_id, + node_debug_id); + + block.lock().toggle_send_ready(true); + } + } + + //TODO: force sync in case of overlay neighbours change + + //check if block updates has to be sent to network (updates buffering) + + let ready_to_send = block.lock().toggle_send_ready(false); + + if ready_to_send { + workchain.send_block_status_impl(&block); + } + } + + /// Start block synchronization + fn start_synchronizing_block(&self, block: &BlockPtr) { + Self::synchronize_block(Arc::downgrade(&self.get_self()), Arc::downgrade(block), None); + } + + /// Put new block to map after delivery + fn add_delivered_block(&self, block_candidate: Arc) -> BlockPtr { + let candidate_id = Self::get_candidate_id(&block_candidate.candidate()); + + //register block + + let block = self.add_block_impl(&candidate_id, Some(block_candidate.clone())); + + block + } + + /// Merge block status + fn merge_block_status( + &self, + candidate_id: &UInt256, + deliveries_signature: &MultiSignature, + approvals_signature: &MultiSignature, + rejections_signature: &MultiSignature, + merges_count: u32, + created_timestamp: i64, + received_from_workchain: bool, + ) -> BlockPtr { + check_execution_time!(5_000); + + self.merge_block_status_counter.increment(1); + + //get existing block or create it + + let block = self.add_block_impl(candidate_id, None); + + //check if block is MC originated + + if !received_from_workchain { + block.lock().mark_as_mc_originated(); + } + + //update status + + let status = block.lock().merge_status(deliveries_signature, approvals_signature, rejections_signature, merges_count, created_timestamp + ); + match status { + Ok(status) => { + if !status { + //block status is the same + return block.clone(); + } + } + Err(err) => { + error!(target: "verificator", "Can't merge block status for block {:?} in workchain {}: {:?}", candidate_id, self.node_debug_id, err); + } + } + + //compute latency for MC deliveries + + if !received_from_workchain { + let was_mc_processed = block.lock().was_mc_processed(); + let (is_delivered, _) = self.get_block_status(&block); + + self.block_status_received_in_mc_counter.total_increment(); + + if is_delivered { + self.block_status_received_in_mc_counter.success(); + } else { + self.block_status_received_in_mc_counter.failure(); + } + + if !was_mc_processed && is_delivered { + let latency = block.lock().get_delivery_latency(); + if let Some(latency) = latency { + self.block_status_received_in_mc_latency_histogram.record(latency.as_millis() as f64); + } + + self.update_block_external_delivery_metrics_impl(&block, None, Some(std::time::SystemTime::now())); + + self.block_status_merges_count_histogram.record(merges_count as f64); + + block.lock().mark_as_mc_processed(); + } + } + + //TODO: do not send block to neighbours if it was received from MC overlay + + //block status was updated + + let send_immediately_to_mc = false; + + self.send_block_status(&block, received_from_workchain, send_immediately_to_mc); + + block + } + + /// Set blocks status (delivered - None, ack - Some(true), nack - Some(false)) + fn set_block_status(&self, candidate_id: &UInt256, status: Option) { + check_execution_time!(5_000); + + self.set_block_status_counter.increment(1); + + //get existing block or create it + + let block = self.add_block_impl(candidate_id, None); + + //update block status + + if self.wc_local_idx != -1 { + let update_status = block.lock().set_status(&self.local_bls_key, self.wc_local_idx as u16, self.wc_validators.len() as u16, status); + + match update_status { + Ok(update_status) => { + if update_status { + //block status was updated + let received_from_workchain = true; + let send_immediately_to_mc = status.is_some(); //send to MC immediately ACK/NACK without buffering + self.send_block_status(&block, received_from_workchain, send_immediately_to_mc); + } + } + Err(err) => { + warn!(target: "verificator", "Can't sign block {} in workchain's node {} private overlay: {:?}", candidate_id, self.node_debug_id, err); + } + } + } + } + + /// Update block delivery metrics + pub fn update_block_external_delivery_metrics( + &self, + candidate_id: &UInt256, + external_request_time: &std::time::SystemTime) { + let block = self.add_block_impl(candidate_id, None); + + self.update_block_external_delivery_metrics_impl(&block, Some(external_request_time.clone()), None); + } + + fn update_block_external_delivery_metrics_impl( + &self, + block: &BlockPtr, + external_request_time: Option, + delivery_state_change_time: Option, + ) { + let (should_update_delivery_metrics, is_first_time_external_request, has_approves, is_rejected, is_delivered, latency) = { + let mut block = block.lock(); + let is_delivered = block.is_delivered(&self.wc_validators, self.wc_cutoff_weight); + + let is_first_time_external_request = block.get_first_external_request_time().is_none() && external_request_time.is_some(); + if let Some(external_request_time) = external_request_time { + block.set_first_external_request_time(&external_request_time); + } + + let is_delivery_state_changed = block.get_delivery_state_change_time().is_none() && delivery_state_change_time.is_some(); + if let Some(delivery_state_change_time) = delivery_state_change_time { + block.set_delivery_state_change_time(&delivery_state_change_time); + } + + let latency = if let Some(external_request_time) = external_request_time { + if let Some(delivery_state_change_time) = delivery_state_change_time { + if let Ok(latency) = delivery_state_change_time.duration_since(external_request_time) { + Some(latency.as_millis()) + } else { + Some(0) //default case - no latency + } + } else { + None + } + } else { + None + }; + + let should_update_delivery_metrics = block.get_first_external_request_time().is_some() && block.get_delivery_state_change_time().is_some() && + (is_first_time_external_request || is_delivery_state_changed); + + (should_update_delivery_metrics, is_first_time_external_request, block.has_approves(), block.is_rejected(), is_delivered, latency) + }; + + if is_first_time_external_request { + self.external_request_counter.increment(1); + } + + if !should_update_delivery_metrics { + return; + } + + if is_delivered { + self.external_request_delivered_blocks_counter.increment(1); + } + + if is_rejected { + self.external_request_rejected_blocks_counter.increment(1); + } + + if has_approves { + self.external_request_approved_blocks_counter.increment(1); + } + + if let Some(latency) = latency { + self.block_external_request_delays_histogram.record(latency as f64); + } + } + + /* + Broadcast delivery protection methods + */ + + /// Get candidate ID + fn get_candidate_id(candidate: &BlockCandidateBroadcast) -> UInt256 { + Self::get_candidate_id_impl(&candidate.id) + } + + /// Get candidate ID + pub fn get_candidate_id_impl(id: &BlockIdExt) -> UInt256 { + id.root_hash.clone() + } + + /// Process new block candidate broadcast + fn process_block_candidate(&self, block_candidate: Arc) { + check_execution_time!(5_000); + + trace!(target: "verificator", "BlockCandidateBroadcast received by verification workchain's node {} private overlay: {:?}", self.node_debug_id, block_candidate.candidate()); + + self.process_block_candidate_counter.increment(1); + + self.add_delivered_block(block_candidate); + } + + /// New block broadcast has been generated + pub fn send_new_block_candidate(&self, candidate: BlockCandidateBroadcast) { + check_execution_time!(5_000); + + let _hang_checker = HangCheck::new(self.runtime.clone(), format!("Workchain::send_new_block_candidate: {:?} for workchain {}", candidate.id, self.node_debug_id), Duration::from_millis(1000)); + + self.new_block_candidate_counter.increment(1); + + //process block candidate + + let block_candidate = Arc::new(BlockCandidateBody::new(candidate)); + let serialized_candidate = block_candidate.serialized_candidate().clone(); + + let candidate_id = self.process_block_candidate(block_candidate); + + //send candidate to other workchain validators + + if let Some(workchain_overlay) = self.get_workchain_overlay() { + trace!(target: "verificator", "Send new block broadcast in workchain {} with candidate_id {:?}", self.node_debug_id, candidate_id); + + workchain_overlay.send_broadcast( + &self.local_adnl_id, + &self.local_id, + serialized_candidate, + ); + } + } + + /// Block status update has been received + pub fn process_block_status(&self, block_status: BlockCandidateStatus, received_from_workchain: bool) -> Result { + check_execution_time!(50_000); + + trace!(target: "verificator", "BlockCandidateStatus received by verification workchain's node {} private overlay: {:?}", self.node_debug_id, block_status); + + self.process_block_status_counter.increment(1); + + let wc_pub_key_refs: Vec<&[u8; BLS_PUBLIC_KEY_LEN]> = self.wc_pub_keys.iter().map(|x| x).collect(); + + let candidate_id: UInt256 = block_status.candidate_id.clone().into(); + let deliveries_signature = MultiSignature::deserialize(1, &candidate_id, &wc_pub_key_refs, &block_status.deliveries_signature); + let approvals_signature = MultiSignature::deserialize(2, &candidate_id, &wc_pub_key_refs, &block_status.approvals_signature); + let rejections_signature = MultiSignature::deserialize(3, &candidate_id, &wc_pub_key_refs, &block_status.rejections_signature); + + if let Err(err) = deliveries_signature { + failure::bail!( + "Can't parse block candidate status (deliveries signature) {:?}: {:?}", + block_status, + err + ); + } + + if let Err(err) = approvals_signature { + failure::bail!( + "Can't parse block candidate status (approvals signature) {:?}: {:?}", + block_status, + err + ); + } + + if let Err(err) = rejections_signature { + failure::bail!( + "Can't parse block candidate status (rejections signature) {:?}: {:?}", + block_status, + err + ); + } + + let deliveries_signature = deliveries_signature.unwrap(); + let approvals_signature = approvals_signature.unwrap(); + let rejections_signature = rejections_signature.unwrap(); + + Ok(self.merge_block_status( + &candidate_id, + &deliveries_signature, + &approvals_signature, + &rejections_signature, + block_status.merges_cnt as u32, + block_status.created_timestamp, + received_from_workchain, + )) + } + + /// Send block for delivery + fn send_block_status(&self, block: &BlockPtr, received_from_workchain: bool, send_immediately_to_mc: bool) { + //serialize block status + + let (serialized_block_status, is_mc_originated, candidate_id, mc_delivered) = { + //this block is needeed to minimize lock of block + let mut block = block.lock(); + let candidate_id = block.get_id().clone(); + + (block.serialize(), block.is_mc_originated(), candidate_id, block.was_mc_delivered()) + }; + + //check if block need to be send to mc + + let should_send_to_mc = self.should_send_to_mc(block) && received_from_workchain && !is_mc_originated && !mc_delivered; + + if send_immediately_to_mc || should_send_to_mc { + if let Some(mc_overlay) = self.get_mc_overlay() { + trace!(target: "verificator", "Send block {:?} to MC after update (node={})", candidate_id, self.node_debug_id); + + let mc_overlay = Arc::downgrade(&mc_overlay); + let serialized_block_status = serialized_block_status.clone(); + let send_block_status_to_mc_counter = self.send_block_status_to_mc_counter.clone(); + + let latency = block.lock().get_delivery_latency(); + if let Some(latency) = latency { + self.block_status_send_to_mc_latency_histogram.record(latency.as_millis() as f64); + } + + if should_send_to_mc { + //prevent double sending of block because of new delivery signatures + //do not mark block as delivered for ACK/NACK signals (because they can appear earlier than cutoff weight for delivery BLS) + block.lock().mark_as_mc_delivered(); + } + + Self::send_block_status_to_mc(mc_overlay, serialized_block_status, send_block_status_to_mc_counter); + } + } + + //mark as ready for send within workchain + + block.lock().toggle_send_ready(true); + } + + /// Send block for delivery implementation + fn send_block_status_impl(&self, block: &BlockPtr) { + self.send_block_status_counter.increment(1); + + //serialize block status + + let serialized_block_status = { + //this block is needeed to minimize lock of block + let mut block = block.lock(); + let candidate_id = block.get_id(); + + trace!(target: "verificator", "Send block {:?} to neighbours after update (node={})", candidate_id, self.node_debug_id); + + block.serialize() + }; + + //send block status to neighbours + + self.send_message_to_private_neighbours(serialized_block_status); + } + + /* + Verification management + */ + + fn should_verify(&self, candidate_id: &UInt256) -> bool { + if let Ok(local_bls_key) = self.local_bls_key.export_key() { + let mut local_key_payload : Vec = local_bls_key.into(); + let mut payload : Vec = candidate_id.as_array().into(); + + payload.append(&mut local_key_payload); + + let payload_ptr = unsafe { std::slice::from_raw_parts(payload.as_ptr() as *const u8, payload.len()) }; + let hash = ton_types::crc32_digest(payload_ptr); + + if self.wc_validators.len () < 1 { + return false; //no verification in empty workchain + } + let random_value = (((hash as usize) % self.wc_validators.len()) as f64) / (self.wc_validators.len() as f64); + let result = random_value < VERIFICATION_OBLIGATION_CUTOFF; + + trace!(target: "verificator", "Verification obligation for block candidate {} is {:.3} < {:.3} -> {} (node={})", candidate_id, random_value, VERIFICATION_OBLIGATION_CUTOFF, result, self.node_debug_id); + + return result; + } + + log::warn!(target: "verificator", "Can't export BLS secret key (node={})", self.node_debug_id); + + false //can't verify without secret BLS key + } + + fn verify_block(&self, candidate_id: &UInt256, block_candidate: Arc) { + trace!(target: "verificator", "Verifying block candidate {} (node={})", candidate_id, self.node_debug_id); + + if !self.should_verify(candidate_id) { + trace!(target: "verificator", "Skipping verification of block candidate {} (node={})", candidate_id, self.node_debug_id); + return; + } + + self.verify_block_counter.total_increment(); + + if let Some(verification_listener) = self.listener.upgrade() { + let candidate_id = candidate_id.clone(); + let workchain = Arc::downgrade(&self.get_self()); + let node_debug_id = self.node_debug_id.clone(); + let runtime = self.runtime.clone(); + let _verification_future = self.runtime.spawn(async move { + if let Some(workchain) = workchain.upgrade() { + check_execution_time!(1_000); + let _hang_checker = HangCheck::new(runtime, format!("Workchain::verify_block: {} for workchain {}", candidate_id, node_debug_id), Duration::from_millis(2000)); + + let candidate = super::BlockCandidate { + block_id: block_candidate.candidate().id.clone().into(), + data: block_candidate.candidate().data.to_vec().into(), + collated_file_hash: block_candidate + .candidate() + .collated_data_file_hash + .clone() + .into(), + collated_data: block_candidate.candidate().collated_data.to_vec().into(), + created_by: block_candidate.candidate().created_by.clone().into(), + }; + + let verification_status = verification_listener.verify(&candidate).await; + + workchain.set_block_verification_status(&candidate_id, verification_status); + } + }); + } + } + + fn set_block_verification_status(&self, candidate_id: &UInt256, verification_status: bool) { + trace!(target: "verificator", "Verified block candidate {:?} status is {} (node={})", candidate_id, verification_status, self.node_debug_id); + + if verification_status { + self.verify_block_counter.success(); + } else { + self.verify_block_counter.failure(); + } + + self.set_block_status(candidate_id, Some(verification_status)); + + if !verification_status { + error!(target: "verificator", "Malicios block candidate {:?} detected (node={})", candidate_id, self.node_debug_id); + } + } + + /* + Private network (for workchains) + */ + + /// Workchain's private overlay + fn get_workchain_overlay(&self) -> Option> { + self.workchain_overlay.lock().clone() + } + + /// Send message to neighbours in a private workchain overlay + fn send_message_to_private_neighbours(&self, data: BlockPayloadPtr) { + if let Some(workchain_overlay) = self.get_workchain_overlay() { + workchain_overlay.send_message_to_private_neighbours(data); + } + } + + /* + Public network (for interaction with MC) + */ + + /// MC public overlay + fn get_mc_overlay(&self) -> Option> { + self.mc_overlay.lock().clone() + } + + fn send_block_status_to_mc(mc_overlay: Weak, data: BlockPayloadPtr, send_block_status_to_mc_counter: metrics::Counter) { + log::trace!(target: "verificator", "Workchain::send_block_status_to_mc"); + + if let Some(mc_overlay) = mc_overlay.upgrade() { + send_block_status_to_mc_counter.increment(1); + + mc_overlay.send_all(data); + } + } +} + +impl WorkchainOverlayListener for Workchain { + /// Block status has been updated + fn on_workchain_block_status_updated( + &self, + block_status: BlockCandidateStatus, + received_from_workchain: bool, + ) -> Result { + let candidate_id: UInt256 = block_status.candidate_id.clone().into(); + let _hang_checker = HangCheck::new(self.runtime.clone(), format!("Workchain::on_workchain_block_status_updated: {} for workchain {}", candidate_id, self.node_debug_id), Duration::from_millis(1000)); + + self.process_block_status(block_status, received_from_workchain) + } + + /// Process new block candidate broadcast + fn on_workchain_block_candidate(&self, block_candidate: Arc) { + let _hang_checker = HangCheck::new(self.runtime.clone(), format!("Workchain::on_workchain_block_candidate: {:?} for workchain {}", block_candidate.candidate().id, self.node_debug_id), Duration::from_millis(1000)); + + self.process_block_candidate(block_candidate); + } +} + +impl Drop for Workchain { + fn drop(&mut self) { + log::info!(target: "verificator", "Dropping verification workchain {}", self.node_debug_id); + } +} diff --git a/src/validator/verification/workchain_overlay.rs b/src/validator/verification/workchain_overlay.rs new file mode 100644 index 00000000..85fa7a6f --- /dev/null +++ b/src/validator/verification/workchain_overlay.rs @@ -0,0 +1,616 @@ +/* +* Copyright (C) 2019-2022 TON Labs. All Rights Reserved. +* +* Licensed under the SOFTWARE EVALUATION License (the "License"); you may not use +* this file except in compliance with the License. +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific TON DEV software governing permissions and +* limitations under the License. +*/ + +use super::block::BlockCandidateBody; +use super::block::BlockPtr; +use super::*; +use super::utils::HangCheck; +use super::utils::get_adnl_id; +use crate::engine_traits::PrivateOverlayOperations; +use crate::validator::validator_utils::sigpubkey_to_publickey; +use catchain::utils::MetricsHandle; +use metrics::Recorder; +use catchain::BlockPayloadPtr; +use catchain::CatchainNode; +use catchain::CatchainOverlayListener; +use catchain::CatchainOverlayLogReplayListener; +use catchain::ExternalQueryResponseCallback; +use catchain::PublicKeyHash; +use catchain::profiling::ResultStatusCounter; +use catchain::profiling::InstanceCounter; +use catchain::check_execution_time; +use log::*; +use adnl::{OverlayShortId, PrivateOverlayShortId}; +use rand::Rng; +use spin::mutex::SpinMutex; +use std::time::Duration; +use std::time::SystemTime; +use tokio::time::sleep; +use ton_api::ton::ton_node::blockcandidatestatus::BlockCandidateStatus; +use ton_api::ton::ton_node::Broadcast; +use ton_types::Result; + +//TODO: traffic metrics & derivatives +//TODO: remove dependency from CatchainClient? (use private overlay directly) + +/* +=============================================================================== + Constants +=============================================================================== +*/ + +const MIN_NEIGHBOURS_COUNT: usize = 16; //min number of neighbours to synchronize +const MAX_NEIGHBOURS_COUNT: usize = 32; //max number of neighbours to synchronize +const NEIGHBOURS_ROTATE_MIN_PERIOD_MS: u64 = 60000; //min time for neighbours rotation +const NEIGHBOURS_ROTATE_MAX_PERIOD_MS: u64 = 120000; //max time for neighbours rotation +const USE_QUERIES_FOR_BLOCK_STATUS: bool = false; //use queries for block status delivery, otherwise use messages + +/* +=============================================================================== + Workchain's private overlay listener +=============================================================================== +*/ + +pub trait WorkchainOverlayListener: Send + Sync { + /// Block status has been updated + fn on_workchain_block_status_updated( + &self, + block_status: BlockCandidateStatus, + received_from_workchain: bool, + ) -> Result; + + /// Process new block candidate broadcast + fn on_workchain_block_candidate(&self, block_candidate: Arc); +} + +/* +=============================================================================== + Workchain's private overlay +=============================================================================== +*/ + +type PrivateOverlayNetworkPtr = Weak; +type OverlayPtr = catchain::CatchainOverlayPtr; + +pub struct WorkchainOverlay { + engine: EnginePtr, //pointer to engine + workchain_id: i32, //workchain identifier + runtime_handle: tokio::runtime::Handle, //runtime handle for further interaction between threads + local_adnl_id: PublicKeyHash, //ADNL ID for this node + network: PrivateOverlayNetworkPtr, //private overlay network + overlay: OverlayPtr, //private overlay + overlay_short_id: Arc, //private overlay short ID + overlay_id: UInt256, //ID of validator list + neighbours_adnl_ids: SpinMutex>, //ADNL IDs of neighbours + active_validators_adnl_ids: Vec, //ADNL IDs of active validators + listener: Arc, //linked listener + node_debug_id: Arc, //debug ID of node + _instance_counter: InstanceCounter, //instance counter + send_message_to_neighbours_counter: metrics::Counter, //number of send message to neighbours calls + out_message_counter: metrics::Counter, //number of outgoing messages + _in_message_counter: metrics::Counter, //number of incoming messages + send_all_counter: metrics::Counter, //number of send to all active nodes calls + out_broadcast_counter: metrics::Counter, //number of outgoing broadcasts + out_query_counter: ResultStatusCounter, //number of outgoing queries + block_status_update_counter: metrics::Counter, //number of block status updates + is_master_chain_overlay: bool, //is this overlay for communication with MC +} + +impl WorkchainOverlay { + /* + Constructor + */ + + /// Create new overlay + pub async fn create( + workchain_id: i32, + debug_id_prefix: String, + overlay_id: UInt256, + validators: &Vec, + active_validators_count: usize, + local_adnl_id: PublicKeyHash, + listener: Weak, + engine: &EnginePtr, + runtime: tokio::runtime::Handle, + metrics_receiver: MetricsHandle, + overlays_instance_counter: Arc, + metrics_prefix: String, + is_master_chain_overlay: bool, + ) -> Result> { + let overlay_short_id = + PrivateOverlayShortId::from_data(overlay_id.as_slice().clone()); + let node_debug_id = Arc::new(format!("{}.{}", debug_id_prefix, overlay_short_id)); + + log::debug!(target: "verificator", "Creating verification workchain's #{} private overlay - overlay_id={}, overlay={}", workchain_id, overlay_id.to_hex_string(), node_debug_id); + + let network = engine.validator_network(); + let nodes: Vec = validators + .iter() + .map(|desc| CatchainNode { + adnl_id: get_adnl_id(&desc), + public_key: sigpubkey_to_publickey(&desc.public_key), + }) + .collect(); + + let result = network.set_validator_list(overlay_id.clone(), &nodes).await?; + + if result.is_none() { + failure::bail!("Can't resolve ADNL IDs for overlay_id={}", overlay_id.to_hex_string()); + } + + let in_broadcast_counter = metrics_receiver.sink().register_counter(&format!("{}_in_broadcasts", metrics_prefix).into()); + let in_query_counter = metrics_receiver.sink().register_counter(&format!("{}_in_queries", metrics_prefix).into()); + let in_block_candidate_counter = metrics_receiver.sink().register_counter(&format!("{}_in_block_candidates", metrics_prefix).into()); + let block_status_update_counter = metrics_receiver.sink().register_counter(&format!("{}_block_status_updates", metrics_prefix).into()); + let out_message_counter = metrics_receiver.sink().register_counter(&format!("{}_out_messages", metrics_prefix).into()); + let in_message_counter = metrics_receiver.sink().register_counter(&format!("{}_in_messages", metrics_prefix).into()); + let out_query_counter = ResultStatusCounter::new(&metrics_receiver, &format!("{}_out_queries", metrics_prefix)); + + let listener = Arc::new(WorkchainListener { + workchain_id, + listener, + runtime_handle: runtime.clone(), + node_debug_id: node_debug_id.clone(), + in_broadcast_counter: in_broadcast_counter.clone(), + in_query_counter: in_query_counter.clone(), + in_message_counter: in_message_counter.clone(), + in_block_candidate_counter: in_block_candidate_counter.clone(), + block_status_update_counter: block_status_update_counter.clone(), + is_master_chain_overlay, + }); + let overlay_listener: Arc = listener.clone(); + let replay_listener: Arc = + listener.clone(); + + let overlay = network.create_catchain_client( + overlay_id.clone(), + &overlay_short_id, + &nodes, + Arc::downgrade(&overlay_listener), + Arc::downgrade(&replay_listener), + )?; + + let active_validators_adnl_ids: Vec = nodes[0..active_validators_count].iter().map(|node| node.adnl_id.clone()).collect(); + + let overlay = Self { + node_debug_id, + workchain_id, + overlay, + overlay_short_id, + network: Arc::downgrade(&network.clone()), + neighbours_adnl_ids: SpinMutex::new(Vec::new()), + active_validators_adnl_ids, + overlay_id, + local_adnl_id, + runtime_handle: runtime.clone(), + engine: engine.clone(), + listener, + _instance_counter: (*overlays_instance_counter).clone(), + send_message_to_neighbours_counter: metrics_receiver.sink().register_counter(&format!("{}_send_message_to_neighbours_calls", metrics_prefix).into()), + send_all_counter: metrics_receiver.sink().register_counter(&format!("{}_send_all_calls", metrics_prefix).into()), + out_broadcast_counter: metrics_receiver.sink().register_counter(&format!("{}_out_broadcasts", metrics_prefix).into()), + out_query_counter, + out_message_counter, + _in_message_counter: in_message_counter, + block_status_update_counter, + is_master_chain_overlay, + }; + let overlay = Arc::new(overlay); + + log::debug!(target: "verificator", "Verification workchain's #{} overlay {} has been started", workchain_id, overlay.node_debug_id); + + Self::rotate_neighbours(Arc::downgrade(&overlay)); + + Ok(overlay) + } + + /* + Sending API + */ + + /// Send message to neighbours + pub fn send_message_to_private_neighbours(&self, data: BlockPayloadPtr) { + let neighbours_adnl_ids = self.neighbours_adnl_ids.lock().clone(); + + log::trace!(target: "verificator", "Sending multicast to {} neighbours (overlay={})", neighbours_adnl_ids.len(), self.node_debug_id); + + self.send_message_to_neighbours_counter.increment(1); + + self.send_message_multicast(&neighbours_adnl_ids, data); + } + + /// Send message to all nodes + pub fn send_all(&self, data: BlockPayloadPtr) { + log::trace!(target: "verificator", "Sending multicast to all {} active nodes (overlay={})", self.active_validators_adnl_ids.len(), self.node_debug_id); + + self.send_all_counter.increment(1); + + self.send_message_multicast(&self.active_validators_adnl_ids, data); + } + + /// Send message to other nodes + fn send_message_multicast(&self, nodes: &Vec, data: BlockPayloadPtr) { + log::trace!(target: "verificator", "Sending multicast {} bytes to {} nodes (overlay={})", data.data().len(), nodes.len(), self.node_debug_id); + + let received_from_workchain = !self.is_master_chain_overlay; + + for adnl_id_ref in nodes { + if USE_QUERIES_FOR_BLOCK_STATUS { + static BLOCK_STATUS_QUERY_TIMEOUT: Duration = Duration::from_millis(5000); + + let workchain_id = self.workchain_id; + let adnl_id = adnl_id_ref.clone(); + let node_debug_id = self.node_debug_id.clone(); + let listener = self.listener.listener.clone(); + let out_query_counter = self.out_query_counter.clone(); + let block_status_update_counter = self.block_status_update_counter.clone(); + + let response_callback = Box::new(move |result: Result| { + check_execution_time!(50_000); + + log::trace!(target: "verificator", "Block status query response received from {} (overlay={})", &adnl_id, node_debug_id); + + match result { + Ok(responsed_data) => { + out_query_counter.success(); + + if let Some(listener) = listener.upgrade() { + match WorkchainListener::process_serialized_block_status(workchain_id, &adnl_id, &responsed_data, &*listener, block_status_update_counter, received_from_workchain) { + Ok(_block) => { + /* do nothing */ + } + Err(err) => { + warn!(target: "verificator", "Block status response processing error (overlay={}): {:?}", node_debug_id, err); + } + } + } + } + Err(err) => { + out_query_counter.failure(); + warn!(target: "verificator", "Invalid block status response (overlay={}): {:?}", node_debug_id, err); + } + } + }); + + self.out_query_counter.total_increment(); + + self.overlay.send_query( + adnl_id_ref, + &self.local_adnl_id, + "block status", + BLOCK_STATUS_QUERY_TIMEOUT, + &data, + response_callback, + ); + } else { + self.out_message_counter.increment(1); + + self.overlay.send_message(adnl_id_ref, &self.local_adnl_id, &data); + } + } + } + + /// Send broadcast + pub fn send_broadcast( + &self, + sender_id: &PublicKeyHash, + send_as: &PublicKeyHash, + payload: BlockPayloadPtr, + ) { + self.overlay + .send_broadcast_fec_ex(sender_id, send_as, payload); + + self.out_broadcast_counter.increment(1); + } + + /* + Neighbours management + */ + + /// Rotate neighbours + fn rotate_neighbours(overlay_weak: Weak) { + let overlay = { + if let Some(overlay) = overlay_weak.upgrade() { + overlay + } else { + return; + } + }; + + trace!(target: "verificator", "Rotate neighbours (overlay={})", overlay.node_debug_id); + + let mut rng = rand::thread_rng(); + + //initiate next neighbours rotation + + let delay = Duration::from_millis(rng.gen_range( + NEIGHBOURS_ROTATE_MIN_PERIOD_MS, + NEIGHBOURS_ROTATE_MAX_PERIOD_MS + 1, + )); + let next_neighbours_rotate_time = SystemTime::now() + delay; + let workchain_id = overlay.workchain_id; + let node_debug_id = overlay.node_debug_id.clone(); + + overlay.runtime_handle.spawn(async move { + if let Ok(timeout) = next_neighbours_rotate_time.duration_since(SystemTime::now()) { + trace!( + target: "verificator", + "Next neighbours rotation for workchain's #{} private overlay is scheduled at {} (in {:.3}s from now; overlay={})", + workchain_id, + catchain::utils::time_to_string(&next_neighbours_rotate_time), + timeout.as_secs_f64(), + node_debug_id); + + sleep(timeout).await; + } + + //rotate neighbours + + Self::rotate_neighbours(overlay_weak); + }); + + //randomly choose max neighbours from sources + + let sources_count = overlay.active_validators_adnl_ids.len(); + //let mut items_count = MAX_NEIGHBOURS_COUNT; + let mut items_count = ((sources_count as f64).sqrt() as usize).clamp(MIN_NEIGHBOURS_COUNT, MAX_NEIGHBOURS_COUNT); + let mut new_neighbours = Vec::with_capacity(items_count); + + trace!( + target: "verificator", + "...choose {} neighbours from {} sources (overlay={})", + items_count, + sources_count, + overlay.node_debug_id, + ); + + if items_count > sources_count { + items_count = sources_count; + } + + let mut neighbours_string = "".to_string(); + let local_adnl_id = overlay.local_adnl_id.clone(); + + for i in 0..sources_count { + let random_value = rng.gen_range(0, sources_count - i); + if random_value >= items_count { + continue; + } + + let adnl_id = overlay.active_validators_adnl_ids[i].clone(); + + if adnl_id == local_adnl_id { + continue; + } + + new_neighbours.push(adnl_id.clone()); + items_count -= 1; + + if neighbours_string.len() > 0 { + neighbours_string = format!("{}, ", neighbours_string); + } + + neighbours_string = format!("{}{}", neighbours_string, adnl_id); + } + + trace!(target: "verificator", "...new workchain's #{} private overlay {} neighbours are: [{:?}]", overlay.workchain_id, overlay.node_debug_id, neighbours_string); + + *overlay.neighbours_adnl_ids.lock() = new_neighbours; + } +} + +impl Drop for WorkchainOverlay { + fn drop(&mut self) { + info!(target: "verificator", "Dropping verification workchain's #{} private overlay (overlay={})", self.workchain_id, self.node_debug_id); + + if let Some(network) = self.network.upgrade() { + debug!(target: "verificator", "Stopping verification workchain's #{} overlay {}", self.workchain_id, self.node_debug_id); + network.stop_catchain_client(&self.overlay_short_id); + } + + let _result = self.engine.remove_validator_list(self.overlay_id.clone()); + } +} + +struct WorkchainListener { + workchain_id: i32, //workchain identifier + node_debug_id: Arc, //node debug ID + listener: Weak, //listener for workchain overlay events + runtime_handle: tokio::runtime::Handle, //runtime handle for further interaction between threads + in_broadcast_counter: metrics::Counter, //number of incoming broadcasts + in_query_counter: metrics::Counter, //number of incoming queries + in_message_counter: metrics::Counter, //number of incoming messages + in_block_candidate_counter: metrics::Counter, //number of incoming block candidates + block_status_update_counter: metrics::Counter, //number of block status updates + is_master_chain_overlay: bool, //is this overlay for communication with MC +} + +impl Drop for WorkchainListener { + fn drop(&mut self) { + trace!(target: "verificator", "Dropping verification workchain's #{} private overlay listener (overlay={})", self.workchain_id, self.node_debug_id); + } +} + +impl CatchainOverlayLogReplayListener for WorkchainListener { + fn on_time_changed(&self, _timestamp: std::time::SystemTime) { /* do nothing */ + } +} + +impl CatchainOverlayListener for WorkchainListener { + fn on_message(&self, adnl_id: PublicKeyHash, data: &BlockPayloadPtr) { + check_execution_time!(20_000); + + trace!(target: "verificator", "WorkchainListener::on_message (overlay={})", self.node_debug_id); + + let _hang_checker = HangCheck::new(self.runtime_handle.clone(), format!("WorkchainListener::on_message: for workchain overlay {}", self.node_debug_id), Duration::from_millis(1000)); + + self.in_message_counter.increment(1); + + let data = data.clone(); + let workchain_id = self.workchain_id; + let node_debug_id = self.node_debug_id.clone(); + let listener = self.listener.clone(); + let block_status_update_counter = self.block_status_update_counter.clone(); + let received_from_workchain = !self.is_master_chain_overlay; + + self.runtime_handle.spawn(async move { + if let Some(listener) = listener.upgrade() { + log::trace!(target: "verificator", "WorkchainListener::on_message from {} (overlay={})", adnl_id, node_debug_id); + + match Self::process_serialized_block_status(workchain_id, &adnl_id, &data, &*listener, block_status_update_counter, received_from_workchain) { + Ok(_block) => { + //ignore reverse block candidate status message + } + Err(err) => { + let message = format!("WorkchainListener::on_message error (overlay={}): {:?}", node_debug_id, err); + warn!(target: "verificator", "{}", message); + } + } + } else { + warn!(target: "verificator", "Query listener is not bound"); + } + }); + } + + fn on_broadcast(&self, source_key_hash: PublicKeyHash, data: &BlockPayloadPtr) { + trace!(target: "verificator", "WorkchainListener::on_broadcast from node with pubkeyhash={} (overlay={})", source_key_hash, self.node_debug_id); + + let _hang_checker = HangCheck::new(self.runtime_handle.clone(), format!("WorkchainListener::on_broadcast: for workchain overlay {}", self.node_debug_id), Duration::from_millis(1000)); + + self.in_broadcast_counter.increment(1); + + let data = data.clone(); + let workchain_id = self.workchain_id; + let node_debug_id = self.node_debug_id.clone(); + let in_block_candidate_counter = self.in_block_candidate_counter.clone(); + + if let Some(listener) = self.listener.upgrade() { + self.runtime_handle.spawn(async move { + match ton_api::Deserializer::new(&mut &data.data()[..]).read_boxed::() { + Ok(broadcast) => { + let broadcast = match broadcast.downcast::() { + Ok(broadcast) => { + match broadcast { + Broadcast::TonNode_BlockCandidateBroadcast(broadcast) => { + let block_candidate = Arc::new(BlockCandidateBody::new(broadcast)); + + in_block_candidate_counter.increment(1); + + listener.on_workchain_block_candidate(block_candidate); + return; + } + broadcast => { + warn!(target: "verificator", "Unexpected broadcast subtype received in verification workchain's #{} private overlay {}: {:?}", workchain_id, node_debug_id, broadcast); + } + } + + return; + } + Err(broadcast) => broadcast + }; + + warn!(target: "verificator", "Unexpected broadcast received in verification workchain's #{} private overlay {}: {:?}", workchain_id, node_debug_id, broadcast); + } + Err(err) => { + warn!(target: "verificator", "Can't parse broadcast received from {} in verification workchain's #{} private overlay {}: {:?}: {:?}", source_key_hash, workchain_id, node_debug_id, data.data(), err); + } + } + }); + } + } + + fn on_query( + &self, + adnl_id: PublicKeyHash, + data: &BlockPayloadPtr, + response_callback: ExternalQueryResponseCallback, + ) { + check_execution_time!(20_000); + + trace!(target: "verificator", "WorkchainListener::on_query (overlay={})", self.node_debug_id); + + let _hang_checker = HangCheck::new(self.runtime_handle.clone(), format!("WorkchainListener::on_query: for workchain overlay {}", self.node_debug_id), Duration::from_millis(1000)); + + self.in_query_counter.increment(1); + + let data = data.clone(); + let workchain_id = self.workchain_id; + let node_debug_id = self.node_debug_id.clone(); + let listener = self.listener.clone(); + let block_status_update_counter = self.block_status_update_counter.clone(); + let received_from_workchain = !self.is_master_chain_overlay; + + self.runtime_handle.spawn(async move { + if let Some(listener) = listener.upgrade() { + log::trace!(target: "verificator", "WorkchainListener::on_query from {} (overlay={})", adnl_id, node_debug_id); + + match Self::process_serialized_block_status(workchain_id, &adnl_id, &data, &*listener, block_status_update_counter, received_from_workchain) { + Ok(block) => { + let serialized_response = block.lock().serialize(); + response_callback(Ok(serialized_response)); + } + Err(err) => { + let message = format!("WorkchainListener::on_query error (overlay={}): {:?}", node_debug_id, err); + warn!(target: "verificator", "{}", message); + response_callback(Err(failure::format_err!("{}", message))); + } + } + } else { + warn!(target: "verificator", "Query listener is not bound"); + response_callback(Err(failure::format_err!("Query listener is not bound"))); + } + }); + } +} + +impl WorkchainListener { + fn process_serialized_block_status( + workchain_id: i32, + adnl_id: &PublicKeyHash, + data: &BlockPayloadPtr, + listener: &dyn WorkchainOverlayListener, + block_status_update_counter: metrics::Counter, + received_from_workchain: bool, + ) -> Result { + check_execution_time!(50_000); + + match ton_api::Deserializer::new(&mut &data.data()[..]).read_boxed::() { + Ok(message) => { + use ton_api::ton::ton_node::BlockCandidateStatus; + + let message = match message.downcast::() { + Ok(block_status) => { + let block_status = block_status.only(); + + block_status_update_counter.increment(1); + + match listener.on_workchain_block_status_updated(block_status, received_from_workchain) { + Ok(block) => { + return Ok(block); + } + Err(err) => { + failure::bail!("Can't process message received from {} in verification workchain's #{} private overlay: {:?}", adnl_id, workchain_id, err); + } + } + } + Err(message) => message + }; + + failure::bail!("Unexpected message received in verification workchain's #{} private overlay: {:?}", workchain_id, message); + } + Err(err) => { + failure::bail!("Can't parse message received from {} in verification workchain's #{} private overlay: {:?}: {:?}", adnl_id, workchain_id, data.data(), err); + } + } + } +} diff --git a/tests/test_run_net/log_cfg.yml b/tests/test_run_net/log_cfg.yml index 9ef3ad15..b94b4edd 100644 --- a/tests/test_run_net/log_cfg.yml +++ b/tests/test_run_net/log_cfg.yml @@ -16,7 +16,7 @@ appenders: rolling_logfile: kind: rolling_file encoder: - pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {l} [{h({t})}] {I}: {m}{n}" + pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {l} [{h({t})}@{file}(({line}))] {I}: {m}{n}" path: '/shared/output_NODE_NUM.log' policy: kind: compound @@ -82,6 +82,9 @@ loggers: remp: level: trace + verificator: + level: trace + catchain: level: error diff --git a/validator-session/src/lib.rs b/validator-session/src/lib.rs index 073c8ff1..414e8383 100644 --- a/validator-session/src/lib.rs +++ b/validator-session/src/lib.rs @@ -33,14 +33,10 @@ mod vector_bool; mod vote_candidate; pub use cache::*; -use catchain::{function, CatchainPtr}; +use catchain::CatchainPtr; use task_queue::{CallbackTaskQueuePtr, CompletionHandlerProcessor, TaskQueuePtr}; use std::{any::Any, cell::RefCell, fmt, rc::Rc, sync::{Arc, Weak}}; -mod profiling { - pub use catchain::profiling::*; -} - pub mod ton { pub use ton_api::ton::int; pub use ton_api::ton::rpc::validator_session::*;