From c2cfd34de6aa08a6fe4ec4066128e0707906e220 Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Wed, 5 Jul 2023 14:12:51 +0200 Subject: [PATCH 1/6] feat: store total consensus stake; garbage collect validator sets --- .../lib/node/ledger/shell/finalize_block.rs | 10 +- proof_of_stake/src/epoched.rs | 25 ++++ proof_of_stake/src/lib.rs | 125 +++++++++++++----- proof_of_stake/src/storage.rs | 16 +++ proof_of_stake/src/tests.rs | 16 ++- proof_of_stake/src/types.rs | 4 + 6 files changed, 157 insertions(+), 39 deletions(-) diff --git a/apps/src/lib/node/ledger/shell/finalize_block.rs b/apps/src/lib/node/ledger/shell/finalize_block.rs index 3af8ab9e3f..e8e7f1bf56 100644 --- a/apps/src/lib/node/ledger/shell/finalize_block.rs +++ b/apps/src/lib/node/ledger/shell/finalize_block.rs @@ -105,8 +105,14 @@ where &mut self.wl_storage, current_epoch, current_epoch + pos_params.pipeline_len, - &namada_proof_of_stake::consensus_validator_set_handle(), - &namada_proof_of_stake::below_capacity_validator_set_handle(), + )?; + namada_proof_of_stake::store_total_consensus_stake( + &mut self.wl_storage, + current_epoch, + )?; + namada_proof_of_stake::purge_validator_sets_for_old_epoch( + &mut self.wl_storage, + current_epoch, )?; } diff --git a/proof_of_stake/src/epoched.rs b/proof_of_stake/src/epoched.rs index 4899ae1e1d..40c8c8f50d 100644 --- a/proof_of_stake/src/epoched.rs +++ b/proof_of_stake/src/epoched.rs @@ -659,6 +659,29 @@ where } } +/// Zero offset +#[derive( + Debug, + Clone, + BorshDeserialize, + BorshSerialize, + BorshSchema, + PartialEq, + Eq, + PartialOrd, + Ord, +)] +pub struct OffsetZero; +impl EpochOffset for OffsetZero { + fn value(_paras: &PosParams) -> u64 { + 0 + } + + fn dyn_offset() -> DynEpochOffset { + DynEpochOffset::Zero + } +} + /// Offset at pipeline length. #[derive( Debug, @@ -731,6 +754,8 @@ impl EpochOffset for OffsetPipelinePlusUnbondingLen { /// Offset length dynamic choice. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum DynEpochOffset { + /// Zero offset + Zero, /// Offset at pipeline length - 1 PipelineLenMinusOne, /// Offset at pipeline length. diff --git a/proof_of_stake/src/lib.rs b/proof_of_stake/src/lib.rs index 7f7265be91..d8de70ad0c 100644 --- a/proof_of_stake/src/lib.rs +++ b/proof_of_stake/src/lib.rs @@ -56,7 +56,7 @@ use storage::{ validator_address_raw_hash_key, validator_last_slash_key, validator_max_commission_rate_change_key, BondDetails, BondsAndUnbondsDetail, BondsAndUnbondsDetails, EpochedSlashes, - ReverseOrdTokenAmount, RewardsAccumulator, SlashedAmount, UnbondDetails, + ReverseOrdTokenAmount, RewardsAccumulator, SlashedAmount, TotalConsensusStakes, UnbondDetails, ValidatorAddresses, ValidatorUnbondRecords, }; use thiserror::Error; @@ -84,6 +84,10 @@ pub fn staking_token_address(storage: &impl StorageRead) -> Address { .expect("Must be able to read native token address") } +/// Number of epochs below the current epoch for which full validator sets are +/// stored +const STORE_VALIDATOR_SETS_LEN: u64 = 2; + #[allow(missing_docs)] #[derive(Error, Debug)] pub enum GenesisError { @@ -274,6 +278,12 @@ pub fn validator_eth_cold_key_handle( ValidatorEthColdKeys::open(key) } +/// Get the storage handle to the total consensus validator stake +pub fn total_consensus_stake_key_handle() -> TotalConsensusStakes { + let key = storage::total_consensus_stake_key(); + TotalConsensusStakes::open(key) +} + /// Get the storage handle to a PoS validator's state pub fn validator_state_handle(validator: &Address) -> ValidatorStates { let key = storage::validator_state_key(validator); @@ -476,6 +486,9 @@ where )?; } + // Store the total consensus validator stake to storage + store_total_consensus_stake(storage, current_epoch)?; + // Write total deltas to storage total_deltas_handle().init_at_genesis( storage, @@ -488,13 +501,7 @@ where credit_tokens(storage, &staking_token, &ADDRESS, total_bonded)?; // Copy the genesis validator set into the pipeline epoch as well for epoch in (current_epoch.next()).iter_range(params.pipeline_len) { - copy_validator_sets_and_positions( - storage, - current_epoch, - epoch, - &consensus_validator_set_handle(), - &below_capacity_validator_set_handle(), - )?; + copy_validator_sets_and_positions(storage, current_epoch, epoch)?; } tracing::debug!("Genesis initialized"); @@ -1528,14 +1535,15 @@ pub fn copy_validator_sets_and_positions( storage: &mut S, current_epoch: Epoch, target_epoch: Epoch, - consensus_validator_set: &ConsensusValidatorSets, - below_capacity_validator_set: &BelowCapacityValidatorSets, ) -> storage_api::Result<()> where S: StorageRead + StorageWrite, { let prev_epoch = target_epoch.prev(); + let consensus_validator_set = consensus_validator_set_handle(); + let below_capacity_validator_set = below_capacity_validator_set_handle(); + let (consensus, below_capacity) = ( consensus_validator_set.at(&prev_epoch), below_capacity_validator_set.at(&prev_epoch), @@ -1597,28 +1605,31 @@ where // Copy validator positions let mut positions = HashMap::::default(); - let positions_handle = validator_set_positions_handle().at(&prev_epoch); + let validator_set_positions_handle = validator_set_positions_handle(); + let positions_handle = validator_set_positions_handle.at(&prev_epoch); + for result in positions_handle.iter(storage)? { let (validator, position) = result?; positions.insert(validator, position); } - let new_positions_handle = - validator_set_positions_handle().at(&target_epoch); + + let new_positions_handle = validator_set_positions_handle.at(&target_epoch); for (validator, position) in positions { let prev = new_positions_handle.insert(storage, validator, position)?; debug_assert!(prev.is_none()); } - validator_set_positions_handle().set_last_update(storage, current_epoch)?; + validator_set_positions_handle.set_last_update(storage, current_epoch)?; // Copy set of all validator addresses let mut all_validators = HashSet::
::default(); - let all_validators_handle = validator_addresses_handle().at(&prev_epoch); + let validator_addresses_handle = validator_addresses_handle(); + let all_validators_handle = validator_addresses_handle.at(&prev_epoch); for result in all_validators_handle.iter(storage)? { let validator = result?; all_validators.insert(validator); } let new_all_validators_handle = - validator_addresses_handle().at(&target_epoch); + validator_addresses_handle.at(&target_epoch); for validator in all_validators { let was_in = new_all_validators_handle.insert(storage, validator)?; debug_assert!(!was_in); @@ -1627,6 +1638,68 @@ where Ok(()) } +/// Compute total validator stake for the current epoch +fn compute_total_consensus_stake( + storage: &S, + epoch: Epoch, +) -> storage_api::Result +where + S: StorageRead, +{ + consensus_validator_set_handle() + .at(&epoch) + .iter(storage)? + .fold(Ok(token::Amount::zero()), |acc, entry| { + let acc = acc?; + let ( + NestedSubKey::Data { + key: amount, + nested_sub_key: _, + }, + _validator, + ) = entry?; + Ok(acc + amount) + }) +} + +/// Store total consensus stake +pub fn store_total_consensus_stake( + storage: &mut S, + epoch: Epoch, +) -> storage_api::Result<()> +where + S: StorageRead + StorageWrite, +{ + let total = compute_total_consensus_stake(storage, epoch)?; + tracing::debug!( + "Computed total consensus stake for epoch {}: {}", + epoch, + total.to_string_native() + ); + total_consensus_stake_key_handle().set(storage, total, epoch, 0) +} + +/// Purge the validator sets from the epochs older than the current epoch minus +/// `STORE_VALIDATOR_SETS_LEN` +pub fn purge_validator_sets_for_old_epoch( + storage: &mut S, + epoch: Epoch, +) -> storage_api::Result<()> +where + S: StorageRead + StorageWrite, +{ + if Epoch(STORE_VALIDATOR_SETS_LEN) < epoch { + let old_epoch = epoch - STORE_VALIDATOR_SETS_LEN - 1; + consensus_validator_set_handle() + .get_data_handler() + .remove_all(storage, &old_epoch)?; + below_capacity_validator_set_handle() + .get_data_handler() + .remove_all(storage, &old_epoch)?; + } + Ok(()) +} + /// Read the position of the validator in the subset of validators that have the /// same bonded stake. This information is held in its own epoched structure in /// addition to being inside the validator sets. @@ -3252,9 +3325,9 @@ where for epoch in Epoch::iter_bounds_inclusive(start_epoch, end_epoch) { let consensus_stake = - Dec::from(get_total_consensus_stake(storage, epoch)?); + Dec::from(get_total_consensus_stake(storage, epoch, params)?); tracing::debug!( - "Consensus stake in epoch {}: {}", + "Total consensus stake in epoch {}: {}", epoch, consensus_stake ); @@ -3856,22 +3929,14 @@ where fn get_total_consensus_stake( storage: &S, epoch: Epoch, + params: &PosParams, ) -> storage_api::Result where S: StorageRead, { - let mut total = token::Amount::default(); - for res in consensus_validator_set_handle().at(&epoch).iter(storage)? { - let ( - NestedSubKey::Data { - key: bonded_stake, - nested_sub_key: _, - }, - _validator, - ) = res?; - total += bonded_stake; - } - Ok(total) + total_consensus_stake_key_handle() + .get(storage, epoch, params) + .map(|o| o.expect("Total consensus stake could not be retrieved.")) } /// Find slashes applicable to a validator with inclusive `start` and exclusive diff --git a/proof_of_stake/src/storage.rs b/proof_of_stake/src/storage.rs index 79124fe3bb..54bd7cfe6b 100644 --- a/proof_of_stake/src/storage.rs +++ b/proof_of_stake/src/storage.rs @@ -36,6 +36,7 @@ const VALIDATOR_TOTAL_UNBONDED_STORAGE_KEY: &str = "total_unbonded"; const VALIDATOR_SETS_STORAGE_PREFIX: &str = "validator_sets"; const CONSENSUS_VALIDATOR_SET_STORAGE_KEY: &str = "consensus"; const BELOW_CAPACITY_VALIDATOR_SET_STORAGE_KEY: &str = "below_capacity"; +const TOTAL_CONSENSUS_STAKE_STORAGE_KEY: &str = "total_consensus_stake"; const TOTAL_DELTAS_STORAGE_KEY: &str = "total_deltas"; const VALIDATOR_SET_POSITIONS_KEY: &str = "validator_set_positions"; const CONSENSUS_KEYS: &str = "consensus_keys"; @@ -584,6 +585,21 @@ pub fn is_below_capacity_validator_set_key(key: &Key) -> bool { matches!(&key.segments[..], [DbKeySeg::AddressSeg(addr), DbKeySeg::StringSeg(key), DbKeySeg::StringSeg(set_type), DbKeySeg::StringSeg(lazy_map), DbKeySeg::StringSeg(data), DbKeySeg::StringSeg(_epoch), DbKeySeg::StringSeg(_), DbKeySeg::StringSeg(_amount), DbKeySeg::StringSeg(_), DbKeySeg::StringSeg(_position)] if addr == &ADDRESS && key == VALIDATOR_SETS_STORAGE_PREFIX && set_type == BELOW_CAPACITY_VALIDATOR_SET_STORAGE_KEY && lazy_map == LAZY_MAP_SUB_KEY && data == lazy_map::DATA_SUBKEY) } +/// Storage key for total consensus stake +pub fn total_consensus_stake_key() -> Key { + Key::from(ADDRESS.to_db_key()) + .push(&TOTAL_CONSENSUS_STAKE_STORAGE_KEY.to_owned()) + .expect("Cannot obtain a total consensus stake key") +} + +/// Is storage key for the total consensus stake? +pub fn is_total_consensus_stake_key(key: &Key) -> bool { + matches!(&key.segments[..], [ + DbKeySeg::AddressSeg(addr), + DbKeySeg::StringSeg(key) + ] if addr == &ADDRESS && key == TOTAL_CONSENSUS_STAKE_STORAGE_KEY) +} + /// Storage key for total deltas of all validators. pub fn total_deltas_key() -> Key { Key::from(ADDRESS.to_db_key()) diff --git a/proof_of_stake/src/tests.rs b/proof_of_stake/src/tests.rs index 451a6880fe..6136b7a94b 100644 --- a/proof_of_stake/src/tests.rs +++ b/proof_of_stake/src/tests.rs @@ -43,15 +43,17 @@ use crate::{ copy_validator_sets_and_positions, find_validator_by_raw_hash, get_num_consensus_validators, init_genesis, insert_validator_into_validator_set, is_validator, process_slashes, + purge_validator_sets_for_old_epoch, read_below_capacity_validator_set_addresses_with_stake, read_below_threshold_validator_set_addresses, read_consensus_validator_set_addresses_with_stake, read_total_stake, read_validator_delta_value, read_validator_stake, slash, - staking_token_address, total_deltas_handle, unbond_handle, unbond_tokens, - unjail_validator, update_validator_deltas, update_validator_set, - validator_consensus_key_handle, validator_set_update_tendermint, - validator_slashes_handle, validator_state_handle, withdraw_tokens, - write_validator_address_raw_hash, BecomeValidator, + staking_token_address, store_total_consensus_stake, total_deltas_handle, + unbond_handle, unbond_tokens, unjail_validator, update_validator_deltas, + update_validator_set, validator_consensus_key_handle, + validator_set_update_tendermint, validator_slashes_handle, + validator_state_handle, withdraw_tokens, write_validator_address_raw_hash, + BecomeValidator, }; proptest! { @@ -2003,14 +2005,14 @@ fn get_tendermint_set_updates( fn advance_epoch(s: &mut TestWlStorage, params: &PosParams) -> Epoch { s.storage.block.epoch = s.storage.block.epoch.next(); let current_epoch = s.storage.block.epoch; + store_total_consensus_stake(s, current_epoch).unwrap(); copy_validator_sets_and_positions( s, current_epoch, current_epoch + params.pipeline_len, - &consensus_validator_set_handle(), - &below_capacity_validator_set_handle(), ) .unwrap(); + purge_validator_sets_for_old_epoch(s, current_epoch).unwrap(); // process_slashes(s, current_epoch).unwrap(); // dbg!(current_epoch); current_epoch diff --git a/proof_of_stake/src/types.rs b/proof_of_stake/src/types.rs index caf705fce3..53f09a5d13 100644 --- a/proof_of_stake/src/types.rs +++ b/proof_of_stake/src/types.rs @@ -121,6 +121,10 @@ pub type BelowCapacityValidatorSets = crate::epoched::NestedEpoched< crate::epoched::OffsetPipelineLen, >; +/// Epoched total consensus validator stake +pub type TotalConsensusStakes = + crate::epoched::Epoched; + /// Epoched validator's deltas. pub type ValidatorDeltas = crate::epoched::EpochedDelta< token::Change, From 363aa470eee38ae1e634e465bb02452583a87270 Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Sun, 9 Jul 2023 21:14:32 +0200 Subject: [PATCH 2/6] add test case --- proof_of_stake/src/tests.rs | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/proof_of_stake/src/tests.rs b/proof_of_stake/src/tests.rs index 6136b7a94b..6476827417 100644 --- a/proof_of_stake/src/tests.rs +++ b/proof_of_stake/src/tests.rs @@ -53,7 +53,7 @@ use crate::{ update_validator_set, validator_consensus_key_handle, validator_set_update_tendermint, validator_slashes_handle, validator_state_handle, withdraw_tokens, write_validator_address_raw_hash, - BecomeValidator, + BecomeValidator, STORE_VALIDATOR_SETS_LEN, }; proptest! { @@ -1161,8 +1161,7 @@ fn test_validator_sets() { .unwrap(); }; - // Start with two genesis validators with 1 NAM stake - let epoch = Epoch::default(); + // Create genesis validators let ((val1, pk1), stake1) = (gen_validator(), token::Amount::native_whole(1)); let ((val2, pk2), stake2) = @@ -1185,6 +1184,9 @@ fn test_validator_sets() { println!("val6: {val6}, {pk6}, {}", stake6.to_string_native()); println!("val7: {val7}, {pk7}, {}", stake7.to_string_native()); + let start_epoch = Epoch::default(); + let epoch = start_epoch; + init_genesis( &mut s, ¶ms, @@ -1751,6 +1753,28 @@ fn test_validator_sets() { }) ); assert_eq!(tm_updates[1], ValidatorSetUpdate::Deactivated(pk4)); + + // Check that the validator sets were purged for the old epochs + let last_epoch = epoch; + for e in Epoch::iter_bounds_inclusive( + start_epoch, + last_epoch + .sub_or_default(Epoch(STORE_VALIDATOR_SETS_LEN)) + .sub_or_default(Epoch(1)), + ) { + assert!( + consensus_validator_set_handle() + .at(&e) + .is_empty(&s) + .unwrap() + ); + assert!( + below_capacity_validator_set_handle() + .at(&e) + .is_empty(&s) + .unwrap() + ); + } } /// When a consensus set validator with 0 voting power adds a bond in the same From c7dab48e7b917591889fe054ad852331757cc958 Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Sun, 9 Jul 2023 21:16:02 +0200 Subject: [PATCH 3/6] refactor: introduce name for magic constant --- proof_of_stake/src/types.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/proof_of_stake/src/types.rs b/proof_of_stake/src/types.rs index 53f09a5d13..736ffe7a46 100644 --- a/proof_of_stake/src/types.rs +++ b/proof_of_stake/src/types.rs @@ -29,6 +29,10 @@ use crate::parameters::PosParams; // core::types::token::NATIVE_MAX_DECIMAL_PLACES?? const U64_MAX: u64 = u64::MAX; +/// Number of epochs below the current epoch for which validator deltas and +/// slashes are stored +const VALIDATOR_DELTAS_SLASHES_LEN: u64 = 23; + // TODO: add this to the spec /// Stored positions of validators in validator sets pub type ValidatorSetPositions = crate::epoched::NestedEpoched< @@ -129,14 +133,14 @@ pub type TotalConsensusStakes = pub type ValidatorDeltas = crate::epoched::EpochedDelta< token::Change, crate::epoched::OffsetUnbondingLen, - 23, + VALIDATOR_DELTAS_SLASHES_LEN, >; /// Epoched total deltas. pub type TotalDeltas = crate::epoched::EpochedDelta< token::Change, crate::epoched::OffsetUnbondingLen, - 23, + VALIDATOR_DELTAS_SLASHES_LEN, >; /// Epoched validator commission rate @@ -168,7 +172,7 @@ pub type ValidatorSlashes = NestedMap; pub type EpochedSlashes = crate::epoched::NestedEpoched< ValidatorSlashes, crate::epoched::OffsetUnbondingLen, - 23, + VALIDATOR_DELTAS_SLASHES_LEN, >; /// Epoched validator's unbonds From e43d5f7e086976d16e3c91995dc8f50dc336ba4c Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Sun, 9 Jul 2023 21:27:21 +0200 Subject: [PATCH 4/6] refactor: simplify code --- proof_of_stake/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/proof_of_stake/src/lib.rs b/proof_of_stake/src/lib.rs index d8de70ad0c..204df78176 100644 --- a/proof_of_stake/src/lib.rs +++ b/proof_of_stake/src/lib.rs @@ -3336,6 +3336,7 @@ where let infracting_stake = slashes.iter(storage)?.fold( Ok(Dec::zero()), |acc: storage_api::Result, res| { + let acc = acc?; let ( NestedSubKey::Data { key: validator, @@ -3349,11 +3350,7 @@ where .unwrap_or_default(); // println!("Val {} stake: {}", &validator, validator_stake); - if let Ok(inner) = acc { - Ok(inner + Dec::from(validator_stake)) - } else { - acc - } + Ok(acc + Dec::from(validator_stake)) // TODO: does something more complex need to be done // here in the event some of these slashes correspond to // the same validator? From 9dc5fff4d72d3f9290114df5c3806a5d87a62237 Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Mon, 10 Jul 2023 16:12:09 +0200 Subject: [PATCH 5/6] add changelog --- .../1129-clear-out-validator-sets-for-old-epochs.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changelog/unreleased/improvements/1129-clear-out-validator-sets-for-old-epochs.md diff --git a/.changelog/unreleased/improvements/1129-clear-out-validator-sets-for-old-epochs.md b/.changelog/unreleased/improvements/1129-clear-out-validator-sets-for-old-epochs.md new file mode 100644 index 0000000000..e7fb6e9063 --- /dev/null +++ b/.changelog/unreleased/improvements/1129-clear-out-validator-sets-for-old-epochs.md @@ -0,0 +1,2 @@ +- PoS: purge validator sets for old epochs from the storage; store total + validator stake ([\#1129](https://github.com/anoma/namada/issues/1129)) \ No newline at end of file From 4abcc9b9c713de10f4343eac6e9559fbee4fbf0c Mon Sep 17 00:00:00 2001 From: Aleksandr Karbyshev Date: Mon, 10 Jul 2023 16:25:53 +0200 Subject: [PATCH 6/6] refactor: fix formatting --- proof_of_stake/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proof_of_stake/src/lib.rs b/proof_of_stake/src/lib.rs index 204df78176..805d79443c 100644 --- a/proof_of_stake/src/lib.rs +++ b/proof_of_stake/src/lib.rs @@ -56,8 +56,9 @@ use storage::{ validator_address_raw_hash_key, validator_last_slash_key, validator_max_commission_rate_change_key, BondDetails, BondsAndUnbondsDetail, BondsAndUnbondsDetails, EpochedSlashes, - ReverseOrdTokenAmount, RewardsAccumulator, SlashedAmount, TotalConsensusStakes, UnbondDetails, - ValidatorAddresses, ValidatorUnbondRecords, + ReverseOrdTokenAmount, RewardsAccumulator, SlashedAmount, + TotalConsensusStakes, UnbondDetails, ValidatorAddresses, + ValidatorUnbondRecords, }; use thiserror::Error; use types::{