Skip to content

Commit

Permalink
Simplify
Browse files Browse the repository at this point in the history
  • Loading branch information
@pwojcikdev
pwojcikdev committed Aug 27, 2024
1 parent 9762eeb commit e8a8bb8
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 66 deletions.
63 changes: 30 additions & 33 deletions nano/store/lmdb/lmdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,53 +261,50 @@ void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction

release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");

const size_t batch_size = 1000 * 10;

nano::account next = 0;
size_t processed_accounts = 0;
while (true)
{
transaction.refresh ();
auto iterate_accounts = [this] (auto && func) {
auto transaction = tx_begin_read ();

// Manually create v22 compatible iterator to read accounts
auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts);
auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);

if (it == end)
{
break;
}

for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
for (; it != end; ++it)
{
auto const & account = it->first;
auto const & account_info = it->second;

if (!account_info.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::lmdb::db_val value;
auto status = get (transaction, tables::rep_weights, account_info.representative, value);
if (success (status))
{
total = nano::amount{ value }.number ();
}
total += account_info.balance.number ();
status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
release_assert_success (status);
}
func (account, account_info);
}
};

// TODO: Make this smaller in dev builds
const size_t batch_size = 250000;

processed_accounts++;
if (processed_accounts % 250000 == 0)
size_t processed = 0;
iterate_accounts ([this, &transaction, &processed] (nano::account const & account, nano::account_info_v22 const & account_info) {
if (!account_info.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::lmdb::db_val value;
auto status = get (transaction, tables::rep_weights, account_info.representative, value);
if (success (status))
{
logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
total = nano::amount{ value }.number ();
}
total += account_info.balance.number ();
status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
release_assert_success (status);
}

next = account.number () + 1;
processed++;
if (processed % batch_size == 0)
{
logger.info (nano::log::type::lmdb, "Processed {} accounts", processed);
transaction.refresh (); // Refresh to prevent excessive memory usage
}
}
});

logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
logger.info (nano::log::type::lmdb, "Done processing {} accounts", processed);
version.put (transaction, 23);

logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed");
Expand Down
63 changes: 30 additions & 33 deletions nano/store/rocksdb/rocksdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,53 +307,50 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti

release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");

const size_t batch_size = 1000 * 10;

nano::account next = 0;
size_t processed_accounts = 0;
while (true)
{
transaction.refresh ();
auto iterate_accounts = [this] (auto && func) {
auto transaction = tx_begin_read ();

// Manually create v22 compatible iterator to read accounts
auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts);
auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);

if (it == end)
{
break;
}

for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
for (; it != end; ++it)
{
auto const & account = it->first;
auto const & account_info = it->second;

if (!account_info.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::rocksdb::db_val value;
auto status = get (transaction, tables::rep_weights, account_info.representative, value);
if (success (status))
{
total = nano::amount{ value }.number ();
}
total += account_info.balance.number ();
status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
release_assert_success (status);
}
func (account, account_info);
}
};

processed_accounts++;
if (processed_accounts % 250000 == 0)
// TODO: Make this smaller in dev builds
const size_t batch_size = 250000;

size_t processed = 0;
iterate_accounts ([this, &transaction, &processed] (nano::account const & account, nano::account_info_v22 const & account_info) {
if (!account_info.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::rocksdb::db_val value;
auto status = get (transaction, tables::rep_weights, account_info.representative, value);
if (success (status))
{
logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
total = nano::amount{ value }.number ();
}
total += account_info.balance.number ();
status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
release_assert_success (status);
}

next = account.number () + 1;
processed++;
if (processed % batch_size == 0)
{
logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed);
transaction.refresh (); // Refresh to prevent excessive memory usage
}
}
});

logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
logger.info (nano::log::type::rocksdb, "Done processing {} accounts", processed);
version.put (transaction, 23);

logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed");
Expand Down

0 comments on commit e8a8bb8

Please sign in to comment.