From a03e245747a9a450f777ed1293ebb8fad821cc08 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Thu, 11 Jul 2024 16:49:37 +0200 Subject: [PATCH 01/40] Intermediate commit: serialization, JNI, DB API. --- .../radixdlt/db/RocksDbMigrationStore.java | 55 ++ .../main/java/com/radixdlt/db/StoreId.java | 21 + .../com/radixdlt/environment/NodeConfig.java | 76 +++ .../environment/NodeRustEnvironment.java | 9 +- .../com/radixdlt/p2p/AddressBookEntryDTO.java | 15 + .../main/java/com/radixdlt/p2p/NodeIdDTO.java | 13 + .../radixdlt/p2p/RocksDbAddressBookStore.java | 66 +++ .../p2p/RocksDbHighPriorityPeersStore.java | 45 ++ .../com/radixdlt/safety/BFTHeaderDTO.java | 12 + .../radixdlt/safety/BFTValidatorIdDTO.java | 16 + .../java/com/radixdlt/safety/HighQCDTO.java | 14 + .../radixdlt/safety/QuorumCertificateDTO.java | 12 + .../radixdlt/safety/RocksDbSafetyStore.java | 47 ++ .../java/com/radixdlt/safety/RoundDTO.java | 11 + .../com/radixdlt/safety/SafetyStateDTO.java | 13 + .../safety/TimeoutCertificateDTO.java | 13 + .../safety/TimestampedECDSASignatureDTO.java | 13 + .../safety/TimestampedECDSASignaturesDTO.java | 14 + .../java/com/radixdlt/safety/VertexIdDTO.java | 40 ++ .../java/com/radixdlt/safety/VoteDTO.java | 19 + .../java/com/radixdlt/safety/VoteDataDTO.java | 11 + .../com/radixdlt/sbor/NodeSborCodecs.java | 22 + .../safety/RocksDbSafetyStoreTest.java | 60 ++ .../state-manager/src/jni/address_book.rs | 135 +++++ .../src/jni/high_priority_peers.rs | 101 ++++ .../state-manager/src/jni/migration_store.rs | 101 ++++ core-rust/state-manager/src/jni/mod.rs | 4 + .../src/jni/node_rust_environment.rs | 28 +- .../state-manager/src/jni/safety_store.rs | 101 ++++ core-rust/state-manager/src/lib.rs | 1 + core-rust/state-manager/src/node.rs | 81 +++ .../src/store/address_book_components.rs | 26 + core-rust/state-manager/src/store/codecs.rs | 2 +- .../src/store/column_families.rs | 471 +++++++++++++++ .../state-manager/src/store/migration.rs | 16 + core-rust/state-manager/src/store/mod.rs | 5 +- core-rust/state-manager/src/store/rocks_db.rs | 536 ++++-------------- core-rust/state-manager/src/store/traits.rs | 29 + .../state-manager/src/store/typed_cf_api.rs | 56 ++ .../java/com/radixdlt/RadixNodeModule.java | 4 +- .../radixdlt/api/system/SystemApiModule.java | 4 +- .../bootstrap/RadixNodeBootstrapper.java | 4 +- .../RadixNodeBootstrapperModule.java | 8 +- .../safety/BerkeleySafetyStateStore.java | 4 +- .../consensus/safety/SafetyState.java | 11 + .../main/java/com/radixdlt/p2p/P2PModule.java | 4 +- .../addressbook/BerkeleyAddressBookStore.java | 4 +- .../modules/BerkeleySafetyStoreModule.java | 4 +- .../rev2/modules/REv2StateManagerModule.java | 22 +- .../store/StateManagerStorageLocation.java | 80 +++ ... StorageLocationFromPropertiesModule.java} | 11 +- .../PrefixedNodeStorageLocationModule.java | 4 +- .../radixdlt/rev2/REv2StateComputerTest.java | 4 +- .../com/radixdlt/rev2/RustMempoolTest.java | 151 ++--- 54 files changed, 2086 insertions(+), 543 deletions(-) create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java create mode 100644 core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java create mode 100644 core-rust/state-manager/src/jni/address_book.rs create mode 100644 core-rust/state-manager/src/jni/high_priority_peers.rs create mode 100644 core-rust/state-manager/src/jni/migration_store.rs create mode 100644 core-rust/state-manager/src/jni/safety_store.rs create mode 100644 core-rust/state-manager/src/node.rs create mode 100644 core-rust/state-manager/src/store/address_book_components.rs create mode 100644 core-rust/state-manager/src/store/column_families.rs create mode 100644 core-rust/state-manager/src/store/migration.rs create mode 100644 core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java rename core/src/main/java/com/radixdlt/store/{NodeStorageLocationFromPropertiesModule.java => StorageLocationFromPropertiesModule.java} (93%) diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java new file mode 100644 index 0000000000..7731d62bc3 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java @@ -0,0 +1,55 @@ +package com.radixdlt.db; + +import com.google.common.reflect.TypeToken; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.sbor.Natives; + +public class RocksDbMigrationStore { + static { + System.loadLibrary("corerust"); + } + + private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] migrationDone( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + /** + * Stores a pointer to the rust core api server across JNI calls. In the JNI model, this is + * equivalent to the CoreApiServer "owning" the rust core api server memory. On each call into + * Rust, we map the rustCoreApiServerPointer onto a concrete implementation in Rust land, and it + * uses that to access all state and make calls. + */ + @SuppressWarnings("unused") + private final long rustCoreApiServerPointer = 0; + + public static RocksDbMigrationStore create( + Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbMigrationStore(metrics, nodeRustEnvironment); + } + + private RocksDbMigrationStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + isMigratedFunc = + Natives.builder(nodeRustEnvironment, RocksDbMigrationStore::isMigrated) + .measure(timer.label(new Metrics.MethodId(RocksDbMigrationStore.class, "isMigrated"))) + .build(new TypeToken<>() {}); + migrationDoneFunc = + Natives.builder(nodeRustEnvironment, RocksDbMigrationStore::migrationDone) + .measure( + timer.label(new Metrics.MethodId(RocksDbMigrationStore.class, "migrationDone"))) + .build(new TypeToken<>() {}); + } + + public boolean isMigrated(StoreId storeId) { + return this.isMigratedFunc.call(storeId); + } + + public void migrationDone(StoreId storeId) { + this.migrationDoneFunc.call(storeId); + } + + private final Natives.Call1 isMigratedFunc; + private final Natives.Call1 migrationDoneFunc; +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java b/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java new file mode 100644 index 0000000000..9a8d18059c --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java @@ -0,0 +1,21 @@ +package com.radixdlt.db; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.EnumCodec; + +/** + * Java side representation of the StoreId enum in Rust + */ +public sealed interface StoreId { + record AddressBookStoreId() implements StoreId {} + + record SafetyStoreId() implements StoreId {} + + AddressBookStoreId ADDRESS_BOOK = new AddressBookStoreId(); + SafetyStoreId SAFETY_STORE = new SafetyStoreId(); + + static void registerCodec(CodecMap codecMap) { + codecMap.register( + StoreId.class, codecs -> EnumCodec.fromPermittedRecordSubclasses(StoreId.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java new file mode 100644 index 0000000000..d06a0d378c --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java @@ -0,0 +1,76 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.environment; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record NodeConfig(DatabaseBackendConfig databaseBackendConfig) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + NodeConfig.class, + codecs -> StructCodec.fromRecordComponents(NodeConfig.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java index 4e394b651e..1ca32dadf0 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java @@ -90,12 +90,15 @@ public final class NodeRustEnvironment implements AutoCloseable { public NodeRustEnvironment( MempoolRelayDispatcher mempoolRelayDispatcher, FatalPanicHandler fatalPanicHandler, - StateManagerConfig config) { + StateManagerConfig config, + NodeConfig nodeConfig) { this.mempoolRelayDispatcher = mempoolRelayDispatcher; this.fatalPanicHandler = fatalPanicHandler; final var encodedConfig = NodeSborCodecs.encode(config, NodeSborCodecs.resolveCodec(new TypeToken<>() {})); - init(this, encodedConfig); + final var encodedNodeConfig = + NodeSborCodecs.encode(nodeConfig, NodeSborCodecs.resolveCodec(new TypeToken<>() {})); + init(this, encodedConfig, encodedNodeConfig); } @Override @@ -128,7 +131,7 @@ public void handleFatalPanic() { this.fatalPanicHandler.handleFatalPanic(); } - private static native void init(NodeRustEnvironment nodeRustEnvironment, byte[] config); + private static native void init(NodeRustEnvironment nodeRustEnvironment, byte[] config, byte[] nodeConfig); private static native void cleanup(NodeRustEnvironment nodeRustEnvironment); } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java new file mode 100644 index 0000000000..e19a9af307 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java @@ -0,0 +1,15 @@ +package com.radixdlt.p2p; + +import com.radixdlt.lang.Option; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +import java.util.Set; + +public record AddressBookEntryDTO(NodeIdDTO nodeId, Option bannedUntil, Set knownAddresses) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + AddressBookEntryDTO.class, + codecs -> StructCodec.fromRecordComponents(AddressBookEntryDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java new file mode 100644 index 0000000000..70534af617 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java @@ -0,0 +1,13 @@ +package com.radixdlt.p2p; + +import com.radixdlt.crypto.ECDSASecp256k1PublicKey; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record NodeIdDTO(ECDSASecp256k1PublicKey publicKey) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + NodeIdDTO.class, + codecs -> StructCodec.fromRecordComponents(NodeIdDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java new file mode 100644 index 0000000000..171db45666 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java @@ -0,0 +1,66 @@ +package com.radixdlt.p2p; + +import static com.radixdlt.lang.Tuple.*; + +import com.google.common.collect.ImmutableList; +import com.google.common.reflect.TypeToken; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.sbor.Natives; +import java.util.List; + +public class RocksDbAddressBookStore { + static { + System.loadLibrary("corerust"); + } + + private static native byte[] removeOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] upsertOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] reset(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] getAll(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + public static RocksDbAddressBookStore create(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbAddressBookStore(metrics, nodeRustEnvironment); + } + + private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + removeOneFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::removeOne) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "removeOne"))) + .build(new TypeToken<>() {}); + upsertOneFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::upsertOne) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "upsertOne"))) + .build(new TypeToken<>() {}); + resetFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::reset) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "reset"))) + .build(new TypeToken<>() {}); + getAllFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::getAll) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "getAll"))) + .build(new TypeToken<>() {}); + } + + boolean upsertEntry(AddressBookEntryDTO entry) { + return this.upsertOneFunc.call(entry); + } + + boolean removeEntry(NodeIdDTO nodeId) { + return this.removeOneFunc.call(nodeId); + } + + void reset() { + this.resetFunc.call(Tuple0.of()); + } + + ImmutableList getAllEntries() { + return ImmutableList.copyOf(this.getAllFunc.call(Tuple0.of())); + } + + private final Natives.Call1 removeOneFunc; + private final Natives.Call1 upsertOneFunc; + private final Natives.Call1 resetFunc; + private final Natives.Call1> getAllFunc; +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java new file mode 100644 index 0000000000..cad0871806 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java @@ -0,0 +1,45 @@ +package com.radixdlt.p2p; + +import static com.radixdlt.lang.Tuple.*; + +import com.google.common.reflect.TypeToken; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.sbor.Natives; +import java.util.List; + +public class RocksDbHighPriorityPeersStore { + static { + System.loadLibrary("corerust"); + } + + private static native byte[] upsertAllHighPriorityPeers(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] getAllHighPriorityPeers(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + public static RocksDbHighPriorityPeersStore create(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbHighPriorityPeersStore(metrics, nodeRustEnvironment); + } + + private RocksDbHighPriorityPeersStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + upsertAllHighPriorityPeersFunc = + Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::upsertAllHighPriorityPeers) + .measure(timer.label(new Metrics.MethodId(RocksDbHighPriorityPeersStore.class, "upsertAllHighPriorityPeers"))) + .build(new TypeToken<>() {}); + getAllHighPriorityPeersFunc = + Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::getAllHighPriorityPeers) + .measure(timer.label(new Metrics.MethodId(RocksDbHighPriorityPeersStore.class, "getAllHighPriorityPeers"))) + .build(new TypeToken<>() {}); + } + + void storeHighPriorityPeers(List ids) { + this.upsertAllHighPriorityPeersFunc.call(ids); + } + + List getHighPriorityPeers() { + return this.getAllHighPriorityPeersFunc.call(Tuple0.of()); + } + + private final Natives.Call1, Tuple0> upsertAllHighPriorityPeersFunc; + private final Natives.Call1> getAllHighPriorityPeersFunc; +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java new file mode 100644 index 0000000000..ba230b8e35 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java @@ -0,0 +1,12 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; +import com.radixdlt.statecomputer.commit.LedgerHeader; + +public record BFTHeaderDTO(RoundDTO round, VertexIdDTO vertexId, LedgerHeader ledgerHeader) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + BFTHeaderDTO.class, codecs -> StructCodec.fromRecordComponents(BFTHeaderDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java new file mode 100644 index 0000000000..60c23de318 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java @@ -0,0 +1,16 @@ +package com.radixdlt.safety; + +import com.radixdlt.crypto.ECDSASecp256k1PublicKey; +import com.radixdlt.rev2.ComponentAddress; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +/** Representation of the validator suitable for Java-Rust exchange. */ +public record BFTValidatorIdDTO( + ECDSASecp256k1PublicKey key, ComponentAddress validatorAddress, String shortenedName) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + BFTValidatorIdDTO.class, + codecs -> StructCodec.fromRecordComponents(BFTValidatorIdDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java new file mode 100644 index 0000000000..234cf0afa4 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java @@ -0,0 +1,14 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record HighQCDTO( + QuorumCertificateDTO highestQC, + QuorumCertificateDTO highestCommittedQC, + TimeoutCertificateDTO highestTC) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + HighQCDTO.class, codecs -> StructCodec.fromRecordComponents(HighQCDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java new file mode 100644 index 0000000000..c9f480801c --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java @@ -0,0 +1,12 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record QuorumCertificateDTO(TimestampedECDSASignaturesDTO signatures, VoteDataDTO voteData) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + QuorumCertificateDTO.class, + codecs -> StructCodec.fromRecordComponents(QuorumCertificateDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java new file mode 100644 index 0000000000..19d18f2561 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java @@ -0,0 +1,47 @@ +package com.radixdlt.safety; + +import static com.radixdlt.lang.Tuple.*; + +import com.google.common.reflect.TypeToken; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.sbor.Natives; +import java.util.Optional; + +public class RocksDbSafetyStore { + static { + System.loadLibrary("corerust"); + } + + private static native byte[] upsert(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] get(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + public static RocksDbSafetyStore create( + Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbSafetyStore(metrics, nodeRustEnvironment); + } + + private RocksDbSafetyStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + upsertFunc = + Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::upsert) + .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "upsert"))) + .build(new TypeToken<>() {}); + getFunc = + Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::get) + .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "get"))) + .build(new TypeToken<>() {}); + } + + public void upsert(SafetyStateDTO state) { + this.upsertFunc.call(state); + } + + public Optional get() { + return Optional.ofNullable(this.getFunc.call(Tuple0.of())); + } + + private final Natives.Call1 upsertFunc; + private final Natives.Call1 getFunc; +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java new file mode 100644 index 0000000000..95c3496ac1 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java @@ -0,0 +1,11 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record RoundDTO(long round) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + RoundDTO.class, codecs -> StructCodec.fromRecordComponents(RoundDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java new file mode 100644 index 0000000000..d1e0b40ba8 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java @@ -0,0 +1,13 @@ +package com.radixdlt.safety; + +import com.radixdlt.lang.Option; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record SafetyStateDTO(BFTValidatorIdDTO validatorId, RoundDTO round, Option lastVote) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + SafetyStateDTO.class, + codecs -> StructCodec.fromRecordComponents(SafetyStateDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java new file mode 100644 index 0000000000..d30d384c27 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java @@ -0,0 +1,13 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record TimeoutCertificateDTO( + long epoch, RoundDTO round, TimestampedECDSASignaturesDTO signatures) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + TimeoutCertificateDTO.class, + codecs -> StructCodec.fromRecordComponents(TimeoutCertificateDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java new file mode 100644 index 0000000000..987715be15 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java @@ -0,0 +1,13 @@ +package com.radixdlt.safety; + +import com.radixdlt.crypto.ECDSASecp256k1Signature; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record TimestampedECDSASignatureDTO(long timestamp, ECDSASecp256k1Signature signature) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + TimestampedECDSASignatureDTO.class, + codecs -> StructCodec.fromRecordComponents(TimestampedECDSASignatureDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java new file mode 100644 index 0000000000..5a3c78a42a --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java @@ -0,0 +1,14 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; +import java.util.Map; + +public record TimestampedECDSASignaturesDTO( + Map nodeToTimestampedSignature) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + TimestampedECDSASignaturesDTO.class, + codecs -> StructCodec.fromRecordComponents(TimestampedECDSASignaturesDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java new file mode 100644 index 0000000000..7ccb666b63 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java @@ -0,0 +1,40 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; +import java.util.Arrays; + +public record VertexIdDTO(byte[] idBytes) { + public VertexIdDTO { + if (idBytes == null) { + throw new IllegalArgumentException("idBytes is null"); + } + + if (idBytes.length != 32) { + throw new IllegalArgumentException("idBytes has invalid length: " + idBytes.length); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o instanceof VertexIdDTO that && Arrays.equals(idBytes, that.idBytes); + } + + @Override + public int hashCode() { + return Arrays.hashCode(idBytes); + } + + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + VertexIdDTO.class, + codecs -> + StructCodec.with( + VertexIdDTO::new, + codecs.of(byte[].class), + (t, encoder) -> encoder.encode(t.idBytes()))); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java new file mode 100644 index 0000000000..57877693c4 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java @@ -0,0 +1,19 @@ +package com.radixdlt.safety; + +import com.radixdlt.crypto.ECDSASecp256k1Signature; +import com.radixdlt.lang.Option; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record VoteDTO( + BFTValidatorIdDTO author, + HighQCDTO highQC, + VoteDataDTO voteData, + long timestamp, + ECDSASecp256k1Signature signature, + Option timeoutSignature) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + VoteDTO.class, codecs -> StructCodec.fromRecordComponents(VoteDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java new file mode 100644 index 0000000000..9821b91fdf --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java @@ -0,0 +1,11 @@ +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +public record VoteDataDTO(BFTHeaderDTO proposed, BFTHeaderDTO parent, BFTHeaderDTO committed) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + VoteDataDTO.class, codecs -> StructCodec.fromRecordComponents(VoteDataDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index b5228878a9..4ea7afaed3 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -67,6 +67,7 @@ import com.google.common.hash.HashCode; import com.google.common.reflect.TypeToken; import com.radixdlt.crypto.*; +import com.radixdlt.db.StoreId; import com.radixdlt.environment.*; import com.radixdlt.exceptions.StateManagerRuntimeError; import com.radixdlt.genesis.*; @@ -75,10 +76,13 @@ import com.radixdlt.mempool.ProposalTransactionsRequest; import com.radixdlt.mempool.RustMempoolConfig; import com.radixdlt.message.*; +import com.radixdlt.p2p.AddressBookEntryDTO; +import com.radixdlt.p2p.NodeIdDTO; import com.radixdlt.prometheus.LedgerStatus; import com.radixdlt.prometheus.RecentSelfProposalMissStatistic; import com.radixdlt.protocol.*; import com.radixdlt.rev2.*; +import com.radixdlt.safety.*; import com.radixdlt.sbor.codec.Codec; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; @@ -213,6 +217,24 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { StateTreeGcConfig.registerCodec(codecMap); LedgerProofsGcConfig.registerCodec(codecMap); LedgerSyncLimitsConfig.registerCodec(codecMap); + NodeConfig.registerCodec(codecMap); + + BFTHeaderDTO.registerCodec(codecMap); + BFTValidatorIdDTO.registerCodec(codecMap); + HighQCDTO.registerCodec(codecMap); + QuorumCertificateDTO.registerCodec(codecMap); + RoundDTO.registerCodec(codecMap); + SafetyStateDTO.registerCodec(codecMap); + TimeoutCertificateDTO.registerCodec(codecMap); + TimestampedECDSASignatureDTO.registerCodec(codecMap); + TimestampedECDSASignaturesDTO.registerCodec(codecMap); + VertexIdDTO.registerCodec(codecMap); + VoteDataDTO.registerCodec(codecMap); + VoteDTO.registerCodec(codecMap); + + AddressBookEntryDTO.registerCodec(codecMap); + NodeIdDTO.registerCodec(codecMap); + StoreId.registerCodec(codecMap); } public static void registerCodecsForExistingTypes(CodecMap codecMap) { diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java new file mode 100644 index 0000000000..ff34477375 --- /dev/null +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -0,0 +1,60 @@ +package com.radixdlt.safety; + +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; + +import static org.junit.Assert.*; + +public class RocksDbSafetyStoreTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test_safety_state_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var safetyStore = RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + var originalSafetyState = new SafetyStateDTO(new byte[] {1, 2, 3}); + } + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } +} diff --git a/core-rust/state-manager/src/jni/address_book.rs b/core-rust/state-manager/src/jni/address_book.rs new file mode 100644 index 0000000000..98cd0b5bd9 --- /dev/null +++ b/core-rust/state-manager/src/jni/address_book.rs @@ -0,0 +1,135 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; +use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use crate::traits::node::AddressBookStore; +use jni::objects::{JClass, JObject}; +use jni::sys::jbyteArray; +use jni::JNIEnv; +use node_common::java::*; +use crate::address_book_components::{AddressBookEntry, AddressBookNodeId}; + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |request: AddressBookNodeId| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .remove_one(&request); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_upsertOne( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_data: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_data, |request: AddressBookEntry| { + // It should be safe to unwrap here, as this operation was already successful inside + // jni_sbor_coded_call. + let entity = jni_jbytearray_to_vector(&env, node_data).unwrap(); + + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .upsert_one(&request.node_id, &entity); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_reset( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .reset(); + vec![0] + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .get_all() + }) +} + +pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/high_priority_peers.rs b/core-rust/state-manager/src/jni/high_priority_peers.rs new file mode 100644 index 0000000000..cd7b2a898f --- /dev/null +++ b/core-rust/state-manager/src/jni/high_priority_peers.rs @@ -0,0 +1,101 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; +use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use crate::traits::node::HighPriorityPeersStore; +use jni::objects::{JClass, JObject}; +use jni::sys::jbyteArray; +use jni::JNIEnv; +use node_common::java::*; + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |request: Vec| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .upsert_all_peers(&request); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHighPriorityPeers( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .get_all_peers() + }) +} + +pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/migration_store.rs b/core-rust/state-manager/src/jni/migration_store.rs new file mode 100644 index 0000000000..c04de6b7bd --- /dev/null +++ b/core-rust/state-manager/src/jni/migration_store.rs @@ -0,0 +1,101 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use crate::traits::node::MigrationStore; +use jni::objects::{JClass, JObject}; +use jni::sys::jbyteArray; +use jni::JNIEnv; +use node_common::java::*; +use crate::migration::MigrationId; + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbMigrationStore_migrationDone( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + migration_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .migration_done(migration_store_id); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbMigrationStore_isMigrated( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + migration_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .is_migration_required(migration_store_id) + }) +} + +pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/mod.rs b/core-rust/state-manager/src/jni/mod.rs index 8a2af6ec09..1b95eae905 100644 --- a/core-rust/state-manager/src/jni/mod.rs +++ b/core-rust/state-manager/src/jni/mod.rs @@ -62,12 +62,16 @@ * permissions under this License. */ +pub mod address_book; pub mod db_checkpoints; pub mod fatal_panic_handler; +pub mod high_priority_peers; pub mod mempool; +pub mod migration_store; pub mod node_rust_environment; pub mod prometheus; pub mod protocol_update; +pub mod safety_store; pub mod state_computer; pub mod state_reader; pub mod test_state_reader; diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index d91170b0d1..b83c2bba90 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -63,6 +63,7 @@ */ use std::ops::Deref; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; @@ -71,7 +72,7 @@ use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::environment::setup_tracing; -use node_common::java::{jni_call, jni_jbytearray_to_vector}; +use node_common::java::{jni_call, jni_jbytearray_to_vector, StructFromJava}; use node_common::locks::*; use prometheus::Registry; @@ -86,10 +87,11 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::rocks_db::{ActualNodeDatabase, ActualStateManagerDatabase}; use crate::transaction::Preparator; use crate::{Committer, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; +use crate::node::NodeConfig; const POINTER_JNI_FIELD_NAME: &str = "rustNodeRustEnvironmentPointer"; @@ -99,9 +101,10 @@ extern "system" fn Java_com_radixdlt_environment_NodeRustEnvironment_init( _class: JClass, j_node_rust_env: JObject, j_config: jbyteArray, + j_node_config: jbyteArray, ) { jni_call(&env, || { - JNINodeRustEnvironment::init(&env, j_node_rust_env, j_config) + JNINodeRustEnvironment::init(&env, j_node_rust_env, j_config, j_node_config) }); } @@ -122,13 +125,16 @@ pub struct JNINodeRustEnvironment { pub state_manager: StateManager, pub metric_registry: Arc, pub running_task_tracker: UntilDropTracker, + pub node_store: Arc>, } impl JNINodeRustEnvironment { - pub fn init(env: &JNIEnv, j_node_rust_env: JObject, j_config: jbyteArray) { + pub fn init(env: &JNIEnv, j_node_rust_env: JObject, j_config: jbyteArray, j_node_config: jbyteArray) { let config_bytes: Vec = jni_jbytearray_to_vector(env, j_config).unwrap(); let config = StateManagerConfig::valid_from_java(&config_bytes).unwrap(); + let db_config_bytes: Vec = jni_jbytearray_to_vector(env, j_node_config).unwrap(); + let node_config = NodeConfig::from_java(&db_config_bytes).unwrap(); let network = config.network_definition.clone(); let runtime = Arc::new(Runtime::new().unwrap()); @@ -162,6 +168,10 @@ impl JNINodeRustEnvironment { ); let running_task_tracker = scheduler.into_task_tracker(); + + let node_db_path = PathBuf::from(node_config.database_backend_config.rocks_db_path); + let raw_node_db = ActualNodeDatabase::new(node_db_path); + let node_store = Arc::new(lock_factory.named("node_database").new_db_lock(raw_node_db)); let jni_node_rust_env = JNINodeRustEnvironment { runtime, @@ -169,6 +179,7 @@ impl JNINodeRustEnvironment { state_manager, metric_registry, running_task_tracker, + node_store, }; env.set_rust_field(j_node_rust_env, POINTER_JNI_FIELD_NAME, jni_node_rust_env) @@ -207,6 +218,15 @@ impl JNINodeRustEnvironment { .database .clone() } + + pub fn get_node_database( + env: &JNIEnv, + j_node_rust_env: JObject, + ) -> Arc> { + Self::get(env, j_node_rust_env) + .node_store + .clone() + } pub fn get_mempool(env: &JNIEnv, j_node_rust_env: JObject) -> Arc> { Self::get(env, j_node_rust_env) diff --git a/core-rust/state-manager/src/jni/safety_store.rs b/core-rust/state-manager/src/jni/safety_store.rs new file mode 100644 index 0000000000..67657b564b --- /dev/null +++ b/core-rust/state-manager/src/jni/safety_store.rs @@ -0,0 +1,101 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; +use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use crate::traits::node::SafetyStateStore; +use jni::objects::{JClass, JObject}; +use jni::sys::jbyteArray; +use jni::JNIEnv; +use node_common::java::*; + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbSafetyStore_upsert( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |request: Vec| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .upsert_safety_state(&request); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbSafetyStore_get( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .get_safety_state() + }) +} + +pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/lib.rs b/core-rust/state-manager/src/lib.rs index 694c01fdb5..c9bc8dc4a0 100644 --- a/core-rust/state-manager/src/lib.rs +++ b/core-rust/state-manager/src/lib.rs @@ -73,6 +73,7 @@ pub mod mempool; pub mod metrics; pub mod protocol; pub mod query; +mod node; mod receipt; mod staging; mod state_manager; diff --git a/core-rust/state-manager/src/node.rs b/core-rust/state-manager/src/node.rs new file mode 100644 index 0000000000..fac86c13c9 --- /dev/null +++ b/core-rust/state-manager/src/node.rs @@ -0,0 +1,81 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; +use crate::store::DatabaseBackendConfig; + +#[derive(Clone, Debug, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct NodeConfig { + pub database_backend_config: DatabaseBackendConfig, +} + +impl NodeConfig { + pub fn new_for_testing(rocks_db_path: impl Into) -> Self { + NodeConfig { + database_backend_config: DatabaseBackendConfig { + rocks_db_path: rocks_db_path.into(), + }, + } + } +} diff --git a/core-rust/state-manager/src/store/address_book_components.rs b/core-rust/state-manager/src/store/address_book_components.rs new file mode 100644 index 0000000000..f2c7915447 --- /dev/null +++ b/core-rust/state-manager/src/store/address_book_components.rs @@ -0,0 +1,26 @@ +use crate::engine_prelude::Sbor; + +/// Id of the node stored in the Address book (Secp256k1 public key) +#[derive(Clone, Copy, Sbor)] +#[sbor(transparent)] +pub struct AddressBookNodeId(pub [u8; AddressBookNodeId::LENGTH]); + +impl AddressBookNodeId { + pub const LENGTH: usize = 33; + + pub fn new(id: [u8; 33]) -> Self { + Self(id) + } + + pub fn as_bytes(&self) -> &[u8; AddressBookNodeId::LENGTH] { + &self.0 + } +} + +/// Address book entry +#[derive(Clone, Sbor)] +pub struct AddressBookEntry { + pub node_id: AddressBookNodeId, + pub banned_until: Option, + pub known_addresses: Vec, +} diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index 73ae02b7b8..f1cd8d2aa9 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -260,7 +260,7 @@ impl DbCodec for StoredTreeNodeKeyDbCodec { impl BoundedDbCodec for StoredTreeNodeKeyDbCodec { fn upper_bound_encoding(&self) -> Vec { // Note: here we use knowledge of `encode_key()`'s internals: it puts the state version - // first. Additionally we need to assume that maximum state version is never reached. + // first. Additionally, we need to assume that maximum state version is never reached. encode_key(&StoredTreeNodeKey::new( Version::MAX, NibblePath::new_even(vec![]), diff --git a/core-rust/state-manager/src/store/column_families.rs b/core-rust/state-manager/src/store/column_families.rs new file mode 100644 index 0000000000..dd1ea26152 --- /dev/null +++ b/core-rust/state-manager/src/store/column_families.rs @@ -0,0 +1,471 @@ +use crate::address_book_components::AddressBookNodeId; +use crate::migration::{MigrationId, MigrationStatus}; +use crate::store::codecs::{ + BlueprintAndCreationIndexKeyDbCodec, EpochDbCodec, HashDbCodec, NodeIdDbCodec, + PrefixGlobalAddressDbCodec, RawLedgerTransactionDbCodec, ScenarioSequenceNumberDbCodec, + StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, + TypeAndCreationIndexKeyDbCodec, +}; +use crate::store::typed_cf_api::{ + AddressBookNodeIdDbCodec, DefaultCf, DirectDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec, + PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, +}; +use crate::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; +use crate::traits::indices::{ + CreationId, EntityBlueprintId, ObjectBlueprintName, VersionedEntityBlueprintId, + VersionedObjectBlueprintName, +}; +use crate::traits::scenario::{ + ExecutedScenario, ScenarioSequenceNumber, VersionedExecutedScenario, +}; +use crate::traits::{ + ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, + VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, + VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, +}; +use crate::transaction::{LedgerTransactionHash, RawLedgerTransaction}; +use crate::{ + CommittedTransactionIdentifiers, LedgerProof, LedgerTransactionReceipt, + LocalTransactionExecution, StateVersion, VersionedCommittedTransactionIdentifiers, + VersionedLedgerProof, VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, +}; +use radix_common::crypto::Hash; +use radix_common::prelude::{EntityType, Epoch, GlobalAddress, NodeId, PackageAddress}; +use radix_substate_store_impls::state_tree::tree_store::{ + StoredTreeNodeKey, TreeNode, VersionedTreeNode, +}; +use radix_substate_store_interface::interface::{DbSubstateKey, DbSubstateValue}; +use radix_transactions::model::{IntentHash, NotarizedTransactionHash}; +use std::fmt; + +/// Committed transactions. +/// Schema: `StateVersion.to_bytes()` -> `RawLedgerTransaction.as_ref::<[u8]>()` +/// Note: This table does not use explicit versioning wrapper, since the serialized content of +/// [`RawLedgerTransaction`] is itself versioned. +pub struct RawLedgerTransactionsCf; +impl DefaultCf for RawLedgerTransactionsCf { + type Key = StateVersion; + type Value = RawLedgerTransaction; + + const DEFAULT_NAME: &'static str = "raw_ledger_transactions"; + type KeyCodec = StateVersionDbCodec; + type ValueCodec = RawLedgerTransactionDbCodec; +} + +/// Identifiers of committed transactions. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedCommittedTransactionIdentifiers)` +pub struct CommittedTransactionIdentifiersCf; +impl VersionedCf for CommittedTransactionIdentifiersCf { + type Key = StateVersion; + type Value = CommittedTransactionIdentifiers; + + const VERSIONED_NAME: &'static str = "committed_transaction_identifiers"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedCommittedTransactionIdentifiers; +} + +/// Ledger receipts of committed transactions. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerTransactionReceipt)` +pub struct TransactionReceiptsCf; +impl VersionedCf for TransactionReceiptsCf { + type Key = StateVersion; + type Value = LedgerTransactionReceipt; + + const VERSIONED_NAME: &'static str = "transaction_receipts"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedLedgerTransactionReceipt; +} + +/// Off-ledger details of committed transaction results (stored only when configured via +/// `enable_local_transaction_execution_index`). +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLocalTransactionExecution)` +pub struct LocalTransactionExecutionsCf; +impl VersionedCf for LocalTransactionExecutionsCf { + type Key = StateVersion; + type Value = LocalTransactionExecution; + + const VERSIONED_NAME: &'static str = "local_transaction_executions"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedLocalTransactionExecution; +} + +/// Ledger proofs of committed transactions. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` +pub struct LedgerProofsCf; +impl VersionedCf for LedgerProofsCf { + type Key = StateVersion; + type Value = LedgerProof; + + const VERSIONED_NAME: &'static str = "ledger_proofs"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedLedgerProof; +} + +/// Ledger proofs of new epochs - i.e. the proofs which trigger the given `next_epoch`. +/// Schema: `Epoch.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` +/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. +pub struct EpochLedgerProofsCf; +impl VersionedCf for EpochLedgerProofsCf { + type Key = Epoch; + type Value = LedgerProof; + + const VERSIONED_NAME: &'static str = "epoch_ledger_proofs"; + type KeyCodec = EpochDbCodec; + type VersionedValue = VersionedLedgerProof; +} + +/// Ledger proofs that initialize protocol updates, i.e. proofs of Consensus `origin`, +/// with headers containing a non-empty `next_protocol_version`. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` +/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. +pub struct ProtocolUpdateInitLedgerProofsCf; +impl VersionedCf for ProtocolUpdateInitLedgerProofsCf { + type Key = StateVersion; + type Value = LedgerProof; + + const VERSIONED_NAME: &'static str = "protocol_update_init_ledger_proofs"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedLedgerProof; +} + +/// Ledger proofs of ProtocolUpdate `origin`, i.e. proofs created locally +/// while protocol update state modifications were being applied. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` +/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. +pub struct ProtocolUpdateExecutionLedgerProofsCf; +impl VersionedCf for ProtocolUpdateExecutionLedgerProofsCf { + type Key = StateVersion; + type Value = LedgerProof; + + const VERSIONED_NAME: &'static str = "protocol_update_execution_ledger_proofs"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedLedgerProof; +} + +/// DB "index" table for transaction's [`IntentHash`] resolution. +/// Schema: `IntentHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` +/// Note: This table does not use explicit versioning wrapper, since the value represents a DB +/// key of another table (and versioning DB keys is not useful). +pub struct IntentHashesCf; +impl DefaultCf for IntentHashesCf { + type Key = IntentHash; + type Value = StateVersion; + + const DEFAULT_NAME: &'static str = "intent_hashes"; + type KeyCodec = HashDbCodec; + type ValueCodec = StateVersionDbCodec; +} + +/// DB "index" table for transaction's [`NotarizedTransactionHash`] resolution. +/// Schema: `NotarizedTransactionHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` +/// Note: This table does not use explicit versioning wrapper, since the value represents a DB +/// key of another table (and versioning DB keys is not useful). +pub struct NotarizedTransactionHashesCf; +impl DefaultCf for NotarizedTransactionHashesCf { + type Key = NotarizedTransactionHash; + type Value = StateVersion; + + const DEFAULT_NAME: &'static str = "notarized_transaction_hashes"; + type KeyCodec = HashDbCodec; + type ValueCodec = StateVersionDbCodec; +} + +/// DB "index" table for transaction's [`LedgerTransactionHash`] resolution. +/// Schema: `LedgerTransactionHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` +/// Note: This table does not use explicit versioning wrapper, since the value represents a DB +/// key of another table (and versioning DB keys is not useful). +pub struct LedgerTransactionHashesCf; +impl DefaultCf for LedgerTransactionHashesCf { + type Key = LedgerTransactionHash; + type Value = StateVersion; + + const DEFAULT_NAME: &'static str = "ledger_transaction_hashes"; + type KeyCodec = HashDbCodec; + type ValueCodec = StateVersionDbCodec; +} + +/// Radix Engine's runtime Substate database. +/// Schema: `encode_to_rocksdb_bytes(DbPartitionKey, DbSortKey)` -> `Vec` +/// Note: This table does not use explicit versioning wrapper, since each serialized substate +/// value is already versioned. +pub struct SubstatesCf; +impl DefaultCf for SubstatesCf { + type Key = DbSubstateKey; + type Value = DbSubstateValue; + + const DEFAULT_NAME: &'static str = "substates"; + type KeyCodec = SubstateKeyDbCodec; + type ValueCodec = DirectDbCodec; +} + +/// Ancestor information for the RE Nodes of [`Substates`] (which is useful and can be computed, +/// but is not provided by the Engine itself). +/// Schema: `NodeId.0` -> `scrypto_encode(VersionedSubstateNodeAncestryRecord)` +/// Note: we do not persist records of root Nodes (which do not have any ancestor). +pub struct SubstateNodeAncestryRecordsCf; +impl VersionedCf for SubstateNodeAncestryRecordsCf { + type Key = NodeId; + type Value = SubstateNodeAncestryRecord; + + const VERSIONED_NAME: &'static str = "substate_node_ancestry_records"; + type KeyCodec = NodeIdDbCodec; + type VersionedValue = VersionedSubstateNodeAncestryRecord; +} + +/// Vertex store. +/// Schema: `[]` -> `scrypto_encode(VersionedVertexStoreBlob)` +/// Note: This is a single-entry table (i.e. the empty key is the only allowed key). +pub struct VertexStoreCf; +impl VersionedCf for VertexStoreCf { + type Key = (); + type Value = VertexStoreBlob; + + const VERSIONED_NAME: &'static str = "vertex_store"; + type KeyCodec = UnitDbCodec; + type VersionedValue = VersionedVertexStoreBlob; +} + +/// Individual nodes of the Substate database's hash tree. +/// Schema: `encode_key(StoredTreeNodeKey)` -> `scrypto_encode(VersionedTreeNode)`. +pub struct StateTreeNodesCf; +impl VersionedCf for StateTreeNodesCf { + type Key = StoredTreeNodeKey; + type Value = TreeNode; + + // Note: the legacy `state_hash_tree` name lives on here because it already got persisted. + const VERSIONED_NAME: &'static str = "state_hash_tree_nodes"; + type KeyCodec = StoredTreeNodeKeyDbCodec; + type VersionedValue = VersionedTreeNode; +} + +/// Parts of the Substate database's hash tree that became stale at a specific state version. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedStaleTreeParts)`. +pub struct StaleStateTreePartsCf; +impl VersionedCf for StaleStateTreePartsCf { + type Key = StateVersion; + type Value = StaleTreeParts; + + // Note: the legacy `state_hash_tree` name lives on here because it already got persisted. + const VERSIONED_NAME: &'static str = "stale_state_hash_tree_parts"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedStaleTreeParts; +} + +/// Transaction accumulator tree slices added at a specific state version. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedTransactionAccuTreeSlice)`. +pub struct TransactionAccuTreeSlicesCf; +impl VersionedCf for TransactionAccuTreeSlicesCf { + type Key = StateVersion; + type Value = TransactionAccuTreeSlice; + + const VERSIONED_NAME: &'static str = "transaction_accu_tree_slices"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedTransactionAccuTreeSlice; +} + +/// Receipt accumulator tree slices added at a specific state version. +/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedReceiptAccuTreeSlice)`. +pub struct ReceiptAccuTreeSlicesCf; +impl VersionedCf for ReceiptAccuTreeSlicesCf { + type Key = StateVersion; + type Value = ReceiptAccuTreeSlice; + + const VERSIONED_NAME: &'static str = "receipt_accu_tree_slices"; + type KeyCodec = StateVersionDbCodec; + type VersionedValue = VersionedReceiptAccuTreeSlice; +} + +/// An enum key for [`ExtensionsDataCf`]. +#[derive(Eq, PartialEq, Hash, PartialOrd, Ord, Clone, Debug)] +pub enum ExtensionsDataKey { + AccountChangeIndexLastProcessedStateVersion, + AccountChangeIndexEnabled, + LocalTransactionExecutionIndexEnabled, + December2023LostSubstatesRestored, + StateTreeAssociatedValuesStatus, + EntityListingIndicesLastProcessedStateVersion, +} + +// IMPORTANT NOTE: the strings defined below are used as database identifiers. Any change would +// effectively clear the associated extension's state in DB. For this reason, we choose to +// define them manually (rather than using the enum's `Into`, which is refactor-sensitive). +impl fmt::Display for ExtensionsDataKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let str = match self { + Self::AccountChangeIndexLastProcessedStateVersion => { + "account_change_index_last_processed_state_version" + } + Self::AccountChangeIndexEnabled => "account_change_index_enabled", + Self::LocalTransactionExecutionIndexEnabled => { + "local_transaction_execution_index_enabled" + } + Self::December2023LostSubstatesRestored => "december_2023_lost_substates_restored", + Self::StateTreeAssociatedValuesStatus => "state_tree_associated_values_status", + Self::EntityListingIndicesLastProcessedStateVersion => { + "entity_listing_indices_last_processed_state_version" + } + }; + write!(f, "{str}") + } +} + +/// Various data needed by extensions. +/// Schema: `ExtensionsDataKeys.to_string().as_bytes() -> Vec`. +/// Note: This table does not use explicit versioning wrapper, since each extension manages the +/// serialization of their data (of its custom type). +pub struct ExtensionsDataCf; +impl TypedCf for ExtensionsDataCf { + type Key = ExtensionsDataKey; + type Value = Vec; + + type KeyCodec = PredefinedDbCodec; + type ValueCodec = DirectDbCodec; + + const NAME: &'static str = "extensions_data"; + + fn key_codec(&self) -> PredefinedDbCodec { + PredefinedDbCodec::new_from_string_representations(vec![ + ExtensionsDataKey::AccountChangeIndexEnabled, + ExtensionsDataKey::AccountChangeIndexLastProcessedStateVersion, + ExtensionsDataKey::LocalTransactionExecutionIndexEnabled, + ExtensionsDataKey::December2023LostSubstatesRestored, + ExtensionsDataKey::StateTreeAssociatedValuesStatus, + ExtensionsDataKey::EntityListingIndicesLastProcessedStateVersion, + ]) + } + + fn value_codec(&self) -> DirectDbCodec { + DirectDbCodec::default() + } +} + +/// Account addresses and state versions at which they were changed. +/// Schema: `[GlobalAddress.0, StateVersion.to_bytes()].concat() -> []`. +/// Note: This is a key-only table (i.e. the empty value is the only allowed value). Given fast +/// prefix iterator from RocksDB this emulates a `Map>`. +pub struct AccountChangeStateVersionsCf; +impl TypedCf for AccountChangeStateVersionsCf { + type Key = (GlobalAddress, StateVersion); + type Value = (); + + type KeyCodec = PrefixGlobalAddressDbCodec; + type ValueCodec = UnitDbCodec; + + const NAME: &'static str = "account_change_state_versions"; + + fn key_codec(&self) -> PrefixGlobalAddressDbCodec { + PrefixGlobalAddressDbCodec::new(StateVersionDbCodec::default()) + } + + fn value_codec(&self) -> UnitDbCodec { + UnitDbCodec::default() + } +} + +/// Additional details of "Scenarios" (and their transactions) executed as part of Genesis, +/// keyed by their sequence number (i.e. their index in the list of Scenarios to execute). +/// Schema: `ScenarioSequenceNumber.to_be_bytes()` -> `scrypto_encode(VersionedExecutedScenario)` +pub struct ExecutedScenariosCf; +impl VersionedCf for ExecutedScenariosCf { + type Key = ScenarioSequenceNumber; + type Value = ExecutedScenario; + + // Note: a legacy name is still used here, even though we now have scenarios run outside Genesis + const VERSIONED_NAME: &'static str = "executed_genesis_scenarios"; + type KeyCodec = ScenarioSequenceNumberDbCodec; + type VersionedValue = VersionedExecutedScenario; +} + +/// Progress of the GC process pruning the [`LedgerProofsCf`]. +/// Schema: `[]` -> `scrypto_encode(VersionedLedgerProofsGcProgress)` +/// Note: This is a single-entry table (i.e. the empty key is the only allowed key). +pub struct LedgerProofsGcProgressCf; +impl VersionedCf for LedgerProofsGcProgressCf { + type Key = (); + type Value = LedgerProofsGcProgress; + + const VERSIONED_NAME: &'static str = "ledger_proofs_gc_progress"; + type KeyCodec = UnitDbCodec; + type VersionedValue = VersionedLedgerProofsGcProgress; +} + +/// Node IDs and blueprints of all entities, indexed by their type and creation order. +/// Schema: `[EntityType as u8, StateVersion.to_be_bytes(), (index_within_txn as u32).to_be_bytes()].concat()` -> `scrypto_encode(VersionedEntityBlueprintId)` +pub struct TypeAndCreationIndexedEntitiesCf; +impl VersionedCf for TypeAndCreationIndexedEntitiesCf { + type Key = (EntityType, CreationId); + type Value = EntityBlueprintId; + + const VERSIONED_NAME: &'static str = "type_and_creation_indexed_entities"; + type KeyCodec = TypeAndCreationIndexKeyDbCodec; + type VersionedValue = VersionedEntityBlueprintId; +} + +/// Node IDs and blueprints of all objects, indexed by their blueprint ID and creation order. +/// Schema: `[PackageAddress.0, hash(blueprint_name), StateVersion.to_be_bytes(), (index_within_txn as u32).to_be_bytes()].concat()` -> `scrypto_encode(VersionedObjectBlueprintName)` +pub struct BlueprintAndCreationIndexedObjectsCf; +impl VersionedCf for BlueprintAndCreationIndexedObjectsCf { + type Key = (PackageAddress, Hash, CreationId); + type Value = ObjectBlueprintName; + + const VERSIONED_NAME: &'static str = "blueprint_and_creation_indexed_objects"; + type KeyCodec = BlueprintAndCreationIndexKeyDbCodec; + type VersionedValue = VersionedObjectBlueprintName; +} + +/// Substate values associated with leaf nodes of the state hash tree's Substate Tier. +/// Needed for [`LeafSubstateValueStore`]. +/// Note: This table does not use explicit versioning wrapper, since each serialized substate +/// value is already versioned. +pub struct AssociatedStateTreeValuesCf; +impl DefaultCf for AssociatedStateTreeValuesCf { + type Key = StoredTreeNodeKey; + type Value = DbSubstateValue; + + const DEFAULT_NAME: &'static str = "associated_state_tree_values"; + type KeyCodec = StoredTreeNodeKeyDbCodec; + type ValueCodec = DirectDbCodec; +} + +/// Address book and safety state store migration status. Filled once during the migration. +pub struct MigrationStatusCf; +impl DefaultCf for MigrationStatusCf { + type Key = MigrationId; + type Value = MigrationStatus; + + const DEFAULT_NAME: &'static str = "migration_status"; + type KeyCodec = MigrationIdDbCodec; + type ValueCodec = MigrationStatusDbCodec; +} + +/// Address book +pub struct AddressBookCf; +impl DefaultCf for AddressBookCf { + type Key = AddressBookNodeId; + type Value = Vec; + + const DEFAULT_NAME: &'static str = "address_book"; + type KeyCodec = AddressBookNodeIdDbCodec; + type ValueCodec = DirectDbCodec; +} + +/// Safety store +pub struct SafetyStoreCf; +impl DefaultCf for SafetyStoreCf { + type Key = (); + type Value = Vec; + + const DEFAULT_NAME: &'static str = "safety_store"; + type KeyCodec = UnitDbCodec; + type ValueCodec = DirectDbCodec; +} + +pub struct HighPriorityPeersCf; +impl DefaultCf for HighPriorityPeersCf { + type Key = (); + type Value = Vec; + + const DEFAULT_NAME: &'static str = "high_priority_peers"; + type KeyCodec = UnitDbCodec; + type ValueCodec = DirectDbCodec; +} diff --git a/core-rust/state-manager/src/store/migration.rs b/core-rust/state-manager/src/store/migration.rs new file mode 100644 index 0000000000..88b2c40f2d --- /dev/null +++ b/core-rust/state-manager/src/store/migration.rs @@ -0,0 +1,16 @@ +use sbor::Sbor; + +/// Identifiers for the migrated stores +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Sbor)] +pub enum MigrationId { + AddressBook, + SafetyState, +} + +/// Status of the migration +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Sbor)] +pub enum MigrationStatus { + Completed, +} diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index c79bbcf5fd..194ae6f773 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -62,12 +62,15 @@ * permissions under this License. */ -mod codecs; +pub mod address_book_components; +pub mod column_families; pub mod historical_state; pub mod jmt_gc; +pub mod migration; pub mod proofs_gc; pub mod rocks_db; pub mod traits; +mod codecs; mod typed_cf_api; use crate::store::traits::measurement::MeasurableDatabase; diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs index 520ae438c9..3ff08fe04d 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -63,7 +63,6 @@ */ use std::collections::HashSet; -use std::fmt; use crate::engine_prelude::*; use crate::store::traits::*; @@ -71,8 +70,6 @@ use crate::{ BySubstate, CommittedTransactionIdentifiers, LedgerProof, LedgerProofOrigin, LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, ReceiptTreeHash, StateVersion, SubstateChangeAction, TransactionTreeHash, - VersionedCommittedTransactionIdentifiers, VersionedLedgerProof, - VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, }; use node_common::utils::IsAccountExt; use rocksdb::checkpoint::Checkpoint; @@ -86,28 +83,27 @@ use std::path::PathBuf; use node_common::locks::Snapshottable; use tracing::{error, info, warn}; +use super::traits::extensions::*; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; +use crate::address_book_components::AddressBookNodeId; +use crate::column_families::*; +use crate::migration::{MigrationId, MigrationStatus}; use crate::query::TransactionIdentifierLoader; -use crate::store::codecs::*; use crate::store::historical_state::StateTreeBasedSubstateDatabase; -use crate::store::traits::gc::{ - LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore, VersionedLedgerProofsGcProgress, -}; +use crate::store::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; use crate::store::traits::indices::{ - CreationId, EntityBlueprintId, EntityListingIndex, ObjectBlueprintName, ObjectBlueprintNameV1, - VersionedEntityBlueprintId, VersionedObjectBlueprintName, + CreationId, EntityBlueprintId, EntityListingIndex, ObjectBlueprintNameV1, }; use crate::store::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDatabase}; use crate::store::traits::scenario::{ - ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, VersionedExecutedScenario, + ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; use crate::store::typed_cf_api::*; +use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; -use super::traits::extensions::*; - /// A listing of all column family names used by the Node. /// /// This is directly needed to initialize the column families within the DB, but is also a nice @@ -126,7 +122,7 @@ use super::traits::extensions::*; /// The `NAME` constants defined by `*Cf` structs (and referenced below) are used as database column /// family names. Any change would effectively mean a ledger wipe. For this reason, we choose to /// define them manually (rather than using the `Into`, which is refactor-sensitive). -const ALL_COLUMN_FAMILIES: [&str; 25] = [ +const ALL_STATE_MANAGER_COLUMN_FAMILIES: [&str; 25] = [ RawLedgerTransactionsCf::DEFAULT_NAME, CommittedTransactionIdentifiersCf::VERSIONED_NAME, TransactionReceiptsCf::VERSIONED_NAME, @@ -154,394 +150,11 @@ const ALL_COLUMN_FAMILIES: [&str; 25] = [ BlueprintAndCreationIndexedObjectsCf::VERSIONED_NAME, ]; -/// Committed transactions. -/// Schema: `StateVersion.to_bytes()` -> `RawLedgerTransaction.as_ref::<[u8]>()` -/// Note: This table does not use explicit versioning wrapper, since the serialized content of -/// [`RawLedgerTransaction`] is itself versioned. -struct RawLedgerTransactionsCf; -impl DefaultCf for RawLedgerTransactionsCf { - type Key = StateVersion; - type Value = RawLedgerTransaction; - - const DEFAULT_NAME: &'static str = "raw_ledger_transactions"; - type KeyCodec = StateVersionDbCodec; - type ValueCodec = RawLedgerTransactionDbCodec; -} - -/// Identifiers of committed transactions. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedCommittedTransactionIdentifiers)` -struct CommittedTransactionIdentifiersCf; -impl VersionedCf for CommittedTransactionIdentifiersCf { - type Key = StateVersion; - type Value = CommittedTransactionIdentifiers; - - const VERSIONED_NAME: &'static str = "committed_transaction_identifiers"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedCommittedTransactionIdentifiers; -} - -/// Ledger receipts of committed transactions. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerTransactionReceipt)` -struct TransactionReceiptsCf; -impl VersionedCf for TransactionReceiptsCf { - type Key = StateVersion; - type Value = LedgerTransactionReceipt; - - const VERSIONED_NAME: &'static str = "transaction_receipts"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedLedgerTransactionReceipt; -} - -/// Off-ledger details of committed transaction results (stored only when configured via -/// `enable_local_transaction_execution_index`). -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLocalTransactionExecution)` -struct LocalTransactionExecutionsCf; -impl VersionedCf for LocalTransactionExecutionsCf { - type Key = StateVersion; - type Value = LocalTransactionExecution; - - const VERSIONED_NAME: &'static str = "local_transaction_executions"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedLocalTransactionExecution; -} - -/// Ledger proofs of committed transactions. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` -struct LedgerProofsCf; -impl VersionedCf for LedgerProofsCf { - type Key = StateVersion; - type Value = LedgerProof; - - const VERSIONED_NAME: &'static str = "ledger_proofs"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedLedgerProof; -} - -/// Ledger proofs of new epochs - i.e. the proofs which trigger the given `next_epoch`. -/// Schema: `Epoch.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` -/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. -struct EpochLedgerProofsCf; -impl VersionedCf for EpochLedgerProofsCf { - type Key = Epoch; - type Value = LedgerProof; - - const VERSIONED_NAME: &'static str = "epoch_ledger_proofs"; - type KeyCodec = EpochDbCodec; - type VersionedValue = VersionedLedgerProof; -} - -/// Ledger proofs that initialize protocol updates, i.e. proofs of Consensus `origin`, -/// with headers containing a non-empty `next_protocol_version`. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` -/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. -struct ProtocolUpdateInitLedgerProofsCf; -impl VersionedCf for ProtocolUpdateInitLedgerProofsCf { - type Key = StateVersion; - type Value = LedgerProof; - - const VERSIONED_NAME: &'static str = "protocol_update_init_ledger_proofs"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedLedgerProof; -} - -/// Ledger proofs of ProtocolUpdate `origin`, i.e. proofs created locally -/// while protocol update state modifications were being applied. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedLedgerProof)` -/// Note: This duplicates a small subset of [`LedgerProofsCf`]'s values. -struct ProtocolUpdateExecutionLedgerProofsCf; -impl VersionedCf for ProtocolUpdateExecutionLedgerProofsCf { - type Key = StateVersion; - type Value = LedgerProof; - - const VERSIONED_NAME: &'static str = "protocol_update_execution_ledger_proofs"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedLedgerProof; -} - -/// DB "index" table for transaction's [`IntentHash`] resolution. -/// Schema: `IntentHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` -/// Note: This table does not use explicit versioning wrapper, since the value represents a DB -/// key of another table (and versioning DB keys is not useful). -struct IntentHashesCf; -impl DefaultCf for IntentHashesCf { - type Key = IntentHash; - type Value = StateVersion; - - const DEFAULT_NAME: &'static str = "intent_hashes"; - type KeyCodec = HashDbCodec; - type ValueCodec = StateVersionDbCodec; -} - -/// DB "index" table for transaction's [`NotarizedTransactionHash`] resolution. -/// Schema: `NotarizedTransactionHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` -/// Note: This table does not use explicit versioning wrapper, since the value represents a DB -/// key of another table (and versioning DB keys is not useful). -struct NotarizedTransactionHashesCf; -impl DefaultCf for NotarizedTransactionHashesCf { - type Key = NotarizedTransactionHash; - type Value = StateVersion; - - const DEFAULT_NAME: &'static str = "notarized_transaction_hashes"; - type KeyCodec = HashDbCodec; - type ValueCodec = StateVersionDbCodec; -} - -/// DB "index" table for transaction's [`LedgerTransactionHash`] resolution. -/// Schema: `LedgerTransactionHash.as_ref::<[u8]>()` -> `StateVersion.to_bytes()` -/// Note: This table does not use explicit versioning wrapper, since the value represents a DB -/// key of another table (and versioning DB keys is not useful). -struct LedgerTransactionHashesCf; -impl DefaultCf for LedgerTransactionHashesCf { - type Key = LedgerTransactionHash; - type Value = StateVersion; - - const DEFAULT_NAME: &'static str = "ledger_transaction_hashes"; - type KeyCodec = HashDbCodec; - type ValueCodec = StateVersionDbCodec; -} - -/// Radix Engine's runtime Substate database. -/// Schema: `encode_to_rocksdb_bytes(DbPartitionKey, DbSortKey)` -> `Vec` -/// Note: This table does not use explicit versioning wrapper, since each serialized substate -/// value is already versioned. -struct SubstatesCf; -impl DefaultCf for SubstatesCf { - type Key = DbSubstateKey; - type Value = DbSubstateValue; - - const DEFAULT_NAME: &'static str = "substates"; - type KeyCodec = SubstateKeyDbCodec; - type ValueCodec = DirectDbCodec; -} - -/// Ancestor information for the RE Nodes of [`Substates`] (which is useful and can be computed, -/// but is not provided by the Engine itself). -/// Schema: `NodeId.0` -> `scrypto_encode(VersionedSubstateNodeAncestryRecord)` -/// Note: we do not persist records of root Nodes (which do not have any ancestor). -struct SubstateNodeAncestryRecordsCf; -impl VersionedCf for SubstateNodeAncestryRecordsCf { - type Key = NodeId; - type Value = SubstateNodeAncestryRecord; - - const VERSIONED_NAME: &'static str = "substate_node_ancestry_records"; - type KeyCodec = NodeIdDbCodec; - type VersionedValue = VersionedSubstateNodeAncestryRecord; -} - -/// Vertex store. -/// Schema: `[]` -> `scrypto_encode(VersionedVertexStoreBlob)` -/// Note: This is a single-entry table (i.e. the empty key is the only allowed key). -struct VertexStoreCf; -impl VersionedCf for VertexStoreCf { - type Key = (); - type Value = VertexStoreBlob; - - const VERSIONED_NAME: &'static str = "vertex_store"; - type KeyCodec = UnitDbCodec; - type VersionedValue = VersionedVertexStoreBlob; -} - -/// Individual nodes of the Substate database's hash tree. -/// Schema: `encode_key(StoredTreeNodeKey)` -> `scrypto_encode(VersionedTreeNode)`. -struct StateTreeNodesCf; -impl VersionedCf for StateTreeNodesCf { - type Key = StoredTreeNodeKey; - type Value = TreeNode; - - // Note: the legacy `state_hash_tree` name lives on here because it already got persisted. - const VERSIONED_NAME: &'static str = "state_hash_tree_nodes"; - type KeyCodec = StoredTreeNodeKeyDbCodec; - type VersionedValue = VersionedTreeNode; -} - -/// Parts of the Substate database's hash tree that became stale at a specific state version. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedStaleTreeParts)`. -struct StaleStateTreePartsCf; -impl VersionedCf for StaleStateTreePartsCf { - type Key = StateVersion; - type Value = StaleTreeParts; - - // Note: the legacy `state_hash_tree` name lives on here because it already got persisted. - const VERSIONED_NAME: &'static str = "stale_state_hash_tree_parts"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedStaleTreeParts; -} - -/// Transaction accumulator tree slices added at a specific state version. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedTransactionAccuTreeSlice)`. -struct TransactionAccuTreeSlicesCf; -impl VersionedCf for TransactionAccuTreeSlicesCf { - type Key = StateVersion; - type Value = TransactionAccuTreeSlice; - - const VERSIONED_NAME: &'static str = "transaction_accu_tree_slices"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedTransactionAccuTreeSlice; -} - -/// Receipt accumulator tree slices added at a specific state version. -/// Schema: `StateVersion.to_bytes()` -> `scrypto_encode(VersionedReceiptAccuTreeSlice)`. -struct ReceiptAccuTreeSlicesCf; -impl VersionedCf for ReceiptAccuTreeSlicesCf { - type Key = StateVersion; - type Value = ReceiptAccuTreeSlice; - - const VERSIONED_NAME: &'static str = "receipt_accu_tree_slices"; - type KeyCodec = StateVersionDbCodec; - type VersionedValue = VersionedReceiptAccuTreeSlice; -} - -/// Various data needed by extensions. -/// Schema: `ExtensionsDataKeys.to_string().as_bytes() -> Vec`. -/// Note: This table does not use explicit versioning wrapper, since each extension manages the -/// serialization of their data (of its custom type). -struct ExtensionsDataCf; -impl TypedCf for ExtensionsDataCf { - type Key = ExtensionsDataKey; - type Value = Vec; - - type KeyCodec = PredefinedDbCodec; - type ValueCodec = DirectDbCodec; - - const NAME: &'static str = "extensions_data"; - - fn key_codec(&self) -> PredefinedDbCodec { - PredefinedDbCodec::new_from_string_representations(vec![ - ExtensionsDataKey::AccountChangeIndexEnabled, - ExtensionsDataKey::AccountChangeIndexLastProcessedStateVersion, - ExtensionsDataKey::LocalTransactionExecutionIndexEnabled, - ExtensionsDataKey::December2023LostSubstatesRestored, - ExtensionsDataKey::StateTreeAssociatedValuesStatus, - ExtensionsDataKey::EntityListingIndicesLastProcessedStateVersion, - ]) - } - - fn value_codec(&self) -> DirectDbCodec { - DirectDbCodec::default() - } -} - -/// Account addresses and state versions at which they were changed. -/// Schema: `[GlobalAddress.0, StateVersion.to_bytes()].concat() -> []`. -/// Note: This is a key-only table (i.e. the empty value is the only allowed value). Given fast -/// prefix iterator from RocksDB this emulates a `Map>`. -struct AccountChangeStateVersionsCf; -impl TypedCf for AccountChangeStateVersionsCf { - type Key = (GlobalAddress, StateVersion); - type Value = (); - - type KeyCodec = PrefixGlobalAddressDbCodec; - type ValueCodec = UnitDbCodec; - - const NAME: &'static str = "account_change_state_versions"; - - fn key_codec(&self) -> PrefixGlobalAddressDbCodec { - PrefixGlobalAddressDbCodec::new(StateVersionDbCodec::default()) - } - - fn value_codec(&self) -> UnitDbCodec { - UnitDbCodec::default() - } -} - -/// Additional details of "Scenarios" (and their transactions) executed as part of Genesis, -/// keyed by their sequence number (i.e. their index in the list of Scenarios to execute). -/// Schema: `ScenarioSequenceNumber.to_be_bytes()` -> `scrypto_encode(VersionedExecutedScenario)` -struct ExecutedScenariosCf; -impl VersionedCf for ExecutedScenariosCf { - type Key = ScenarioSequenceNumber; - type Value = ExecutedScenario; - - // Note: a legacy name is still used here, even though we now have scenarios run outside Genesis - const VERSIONED_NAME: &'static str = "executed_genesis_scenarios"; - type KeyCodec = ScenarioSequenceNumberDbCodec; - type VersionedValue = VersionedExecutedScenario; -} - -/// A progress of the GC process pruning the [`LedgerProofsCf`]. -/// Schema: `[]` -> `scrypto_encode(VersionedLedgerProofsGcProgress)` -/// Note: This is a single-entry table (i.e. the empty key is the only allowed key). -struct LedgerProofsGcProgressCf; -impl VersionedCf for LedgerProofsGcProgressCf { - type Key = (); - type Value = LedgerProofsGcProgress; - - const VERSIONED_NAME: &'static str = "ledger_proofs_gc_progress"; - type KeyCodec = UnitDbCodec; - type VersionedValue = VersionedLedgerProofsGcProgress; -} - -/// Node IDs and blueprints of all entities, indexed by their type and creation order. -/// Schema: `[EntityType as u8, StateVersion.to_be_bytes(), (index_within_txn as u32).to_be_bytes()].concat()` -> `scrypto_encode(VersionedEntityBlueprintId)` -struct TypeAndCreationIndexedEntitiesCf; -impl VersionedCf for TypeAndCreationIndexedEntitiesCf { - type Key = (EntityType, CreationId); - type Value = EntityBlueprintId; - - const VERSIONED_NAME: &'static str = "type_and_creation_indexed_entities"; - type KeyCodec = TypeAndCreationIndexKeyDbCodec; - type VersionedValue = VersionedEntityBlueprintId; -} - -/// Node IDs and blueprints of all objects, indexed by their blueprint ID and creation order. -/// Schema: `[PackageAddress.0, hash(blueprint_name), StateVersion.to_be_bytes(), (index_within_txn as u32).to_be_bytes()].concat()` -> `scrypto_encode(VersionedObjectBlueprintName)` -struct BlueprintAndCreationIndexedObjectsCf; -impl VersionedCf for BlueprintAndCreationIndexedObjectsCf { - type Key = (PackageAddress, Hash, CreationId); - type Value = ObjectBlueprintName; - - const VERSIONED_NAME: &'static str = "blueprint_and_creation_indexed_objects"; - type KeyCodec = BlueprintAndCreationIndexKeyDbCodec; - type VersionedValue = VersionedObjectBlueprintName; -} - -/// Substate values associated with leaf nodes of the state hash tree's Substate Tier. -/// Needed for [`LeafSubstateValueStore`]. -/// Note: This table does not use explicit versioning wrapper, since each serialized substate -/// value is already versioned. -struct AssociatedStateTreeValuesCf; -impl DefaultCf for AssociatedStateTreeValuesCf { - type Key = StoredTreeNodeKey; - type Value = DbSubstateValue; - - const DEFAULT_NAME: &'static str = "associated_state_tree_values"; - type KeyCodec = StoredTreeNodeKeyDbCodec; - type ValueCodec = DirectDbCodec; -} - -/// An enum key for [`ExtensionsDataCf`]. -#[derive(Eq, PartialEq, Hash, PartialOrd, Ord, Clone, Debug)] -enum ExtensionsDataKey { - AccountChangeIndexLastProcessedStateVersion, - AccountChangeIndexEnabled, - LocalTransactionExecutionIndexEnabled, - December2023LostSubstatesRestored, - StateTreeAssociatedValuesStatus, - EntityListingIndicesLastProcessedStateVersion, -} - -// IMPORTANT NOTE: the strings defined below are used as database identifiers. Any change would -// effectively clear the associated extension's state in DB. For this reason, we choose to -// define them manually (rather than using the enum's `Into`, which is refactor-sensitive). -impl fmt::Display for ExtensionsDataKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let str = match self { - Self::AccountChangeIndexLastProcessedStateVersion => { - "account_change_index_last_processed_state_version" - } - Self::AccountChangeIndexEnabled => "account_change_index_enabled", - Self::LocalTransactionExecutionIndexEnabled => { - "local_transaction_execution_index_enabled" - } - Self::December2023LostSubstatesRestored => "december_2023_lost_substates_restored", - Self::StateTreeAssociatedValuesStatus => "state_tree_associated_values_status", - Self::EntityListingIndicesLastProcessedStateVersion => { - "entity_listing_indices_last_processed_state_version" - } - }; - write!(f, "{str}") - } -} +const ALL_NODE_COLUMN_FAMILIES: [&str; 3] = [ + MigrationStatusCf::DEFAULT_NAME, + AddressBookCf::DEFAULT_NAME, + SafetyStoreCf::DEFAULT_NAME, +]; /// A redefined RocksDB's "key and value bytes" tuple (the original one lives in a private module). pub type KVBytes = (Box<[u8]>, Box<[u8]>); @@ -740,6 +353,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> { } pub type ActualStateManagerDatabase = StateManagerDatabase; +pub type ActualNodeDatabase = NodeDatabase; impl<'db> Snapshottable<'db> for StateManagerDatabase { type Snapshot = StateManagerDatabase>; @@ -760,6 +374,108 @@ impl<'db> Snapshottable<'db> for StateManagerDatabase { } } +/// A RocksDB-backed persistence layer for node-specific address book and safety store. +pub struct NodeDatabase { + /// Underlying RocksDB instance. + rocks: R, +} + +impl ActualNodeDatabase { + pub fn new( + root_path: PathBuf, + ) -> ActualNodeDatabase { + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + + let column_families: Vec = ALL_NODE_COLUMN_FAMILIES + .iter() + .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) + .collect(); + + let db = DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap(); + + NodeDatabase { + rocks: DirectRocks { db }, + } + } +} + +impl NodeDatabase { + fn open_rw_context(&self) -> TypedDbContext> { + TypedDbContext::new(&self.rocks, BufferedWriteSupport::new(&self.rocks)) + } +} + +impl AddressBookStore for NodeDatabase { + fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { + let binding = self.open_rw_context(); + let context = binding.cf(AddressBookCf); + let exists = context.get(node_id).is_some(); + + if exists { + context.delete(node_id); + } + + exists + } + + fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { + let binding = self.open_rw_context(); + let context = binding.cf(AddressBookCf); + let exists = context.get(node_id).is_some(); + + context.put(node_id, &entry.to_vec()); + + exists + } + + fn reset(&self) { + todo!("reset") + } + + fn get_all(&self) -> Vec { + todo!("get_all") + } +} + +impl HighPriorityPeersStore for NodeDatabase { + fn upsert_all_peers(&self, peers: &[u8]) { + self.open_rw_context().cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); + } + + fn get_all_peers(&self) -> Vec { + self.open_rw_context().cf(HighPriorityPeersCf) + .get(&()) + .unwrap_or_default() + } +} + +impl SafetyStateStore for NodeDatabase { + fn upsert_safety_state(&self, safety_state: &[u8]) { + self.open_rw_context().cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); + } + + fn get_safety_state(&self) -> Vec { + self.open_rw_context().cf(SafetyStoreCf) + .get(&()) + .unwrap_or_default() + } +} + +impl MigrationStore for NodeDatabase { + fn is_migration_required(&self, store_id: MigrationId) -> bool { + self.open_rw_context().cf(MigrationStatusCf) + .get(&store_id).is_none() + } + + fn migration_done(&self, store_id: MigrationId) { + self.open_rw_context().cf(MigrationStatusCf) + .put(&store_id, &MigrationStatus::Completed) + } +} + + /// A RocksDB-backed persistence layer for state manager. pub struct StateManagerDatabase { /// Database config. @@ -782,7 +498,7 @@ impl ActualStateManagerDatabase { db_opts.create_if_missing(true); db_opts.create_missing_column_families(true); - let column_families: Vec = ALL_COLUMN_FAMILIES + let column_families: Vec = ALL_STATE_MANAGER_COLUMN_FAMILIES .iter() .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) .collect(); @@ -819,7 +535,7 @@ impl ActualStateManagerDatabase { db_opts.create_if_missing(false); db_opts.create_missing_column_families(false); - let column_families: Vec = ALL_COLUMN_FAMILIES + let column_families: Vec = ALL_STATE_MANAGER_COLUMN_FAMILIES .iter() .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) .collect(); @@ -1071,7 +787,7 @@ impl ConfigurableDatabase for StateManagerDatabase { impl MeasurableDatabase for ActualStateManagerDatabase { fn get_data_volume_statistics(&self) -> Vec { - let mut statistics = ALL_COLUMN_FAMILIES + let mut statistics = ALL_STATE_MANAGER_COLUMN_FAMILIES .iter() .map(|cf_name| { ( @@ -1080,13 +796,11 @@ impl MeasurableDatabase for ActualStateManagerDatabase { ) }) .collect::>(); - let live_files = match self.rocks.db.live_files() { - Ok(live_files) => live_files, - Err(err) => { - warn!("could not get DB live files; returning 0: {:?}", err); - Vec::new() - } - }; + let live_files = self.rocks.db.live_files().unwrap_or_else(|err| { + warn!("could not get DB live files; returning 0: {:?}", err); + Vec::new() + }); + for live_file in live_files { let Some(statistic) = statistics.get_mut(&live_file.column_family_name) else { warn!("LiveFile of unknown column family: {:?}", live_file); diff --git a/core-rust/state-manager/src/store/traits.rs b/core-rust/state-manager/src/store/traits.rs index c041434502..16d9d21f87 100644 --- a/core-rust/state-manager/src/store/traits.rs +++ b/core-rust/state-manager/src/store/traits.rs @@ -169,6 +169,35 @@ pub struct LeafSubstateKeyAssociation { pub substate_value: Vec, } +pub mod node { + use crate::address_book_components::AddressBookNodeId; + use crate::migration::MigrationId; + + use super::*; + + pub trait AddressBookStore { + fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; + fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; + fn reset(&self); + fn get_all(&self, ) -> Vec; + } + + pub trait HighPriorityPeersStore { + fn upsert_all_peers(&self, peers: &[u8]); + fn get_all_peers(&self) -> Vec; + } + + pub trait SafetyStateStore { + fn upsert_safety_state(&self, safety_state: &[u8]); + fn get_safety_state(&self) -> Vec; + } + + pub trait MigrationStore { + fn is_migration_required(&self, store_id: MigrationId) -> bool; + fn migration_done(&self, store_id: MigrationId); + } +} + pub mod vertex { use super::*; diff --git a/core-rust/state-manager/src/store/typed_cf_api.rs b/core-rust/state-manager/src/store/typed_cf_api.rs index 3d6efea5c3..49caa3ac6a 100644 --- a/core-rust/state-manager/src/store/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/typed_cf_api.rs @@ -62,7 +62,9 @@ * permissions under this License. */ +use crate::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; +use crate::migration::{MigrationId, MigrationStatus}; use crate::store::rocks_db::{ReadableRocks, WriteableRocks}; use itertools::Itertools; use rocksdb::{ColumnFamily, Direction, IteratorMode, WriteBatch}; @@ -722,6 +724,25 @@ impl DbCodec> for DirectDbCodec { } } +#[derive(Clone, Default)] +pub struct AddressBookNodeIdDbCodec {} + +impl DbCodec for AddressBookNodeIdDbCodec { + fn encode(&self, value: &AddressBookNodeId) -> Vec { + value.as_bytes().to_vec() + } + + fn decode(&self, bytes: &[u8]) -> AddressBookNodeId { + AddressBookNodeId::new(bytes.try_into().expect("Invalid NodeId")) + } +} + +impl BoundedDbCodec for AddressBookNodeIdDbCodec { + fn upper_bound_encoding(&self) -> Vec { + vec![0xFF; AddressBookNodeId::LENGTH] + } +} + /// A [`DbCodec]` capable of representing only a unit `()` (as an empty array). /// This is useful e.g. for "single-row" column families (which do not need keys), or "key-only" /// column families (which do not need values). @@ -738,6 +759,41 @@ impl DbCodec<()> for UnitDbCodec { } } +/// A [`DbCodec]` capable of representing [`MigrationId]`. +#[derive(Clone, Default)] +pub struct MigrationIdDbCodec; + +impl DbCodec for MigrationIdDbCodec { + fn encode(&self, key: &MigrationId) -> Vec { + vec![*key as u8] + } + + fn decode(&self, bytes: &[u8]) -> MigrationId { + match bytes[0] { + 0 => MigrationId::AddressBook, + 1 => MigrationId::SafetyState, + _ => panic!("Invalid MigrationId"), + } + } +} + +/// A [`DbCodec]` capable of representing [`MigrationStatus]`. +#[derive(Clone, Default)] +pub struct MigrationStatusDbCodec; + +impl DbCodec for MigrationStatusDbCodec { + fn encode(&self, key: &MigrationStatus) -> Vec { + vec![*key as u8] + } + + fn decode(&self, bytes: &[u8]) -> MigrationStatus { + match bytes[0] { + 0 => MigrationStatus::Completed, + _ => panic!("Invalid MigrationId"), + } + } +} + /// A [`DbCodec]` based on a predefined set of mappings. #[derive(Default)] pub struct PredefinedDbCodec { diff --git a/core/src/main/java/com/radixdlt/RadixNodeModule.java b/core/src/main/java/com/radixdlt/RadixNodeModule.java index fc9a035560..7e2675de94 100644 --- a/core/src/main/java/com/radixdlt/RadixNodeModule.java +++ b/core/src/main/java/com/radixdlt/RadixNodeModule.java @@ -97,7 +97,7 @@ import com.radixdlt.protocol.ProtocolConfig; import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.rev2.modules.*; -import com.radixdlt.store.NodeStorageLocationFromPropertiesModule; +import com.radixdlt.store.StorageLocationFromPropertiesModule; import com.radixdlt.sync.SyncRelayConfig; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.utils.BooleanUtils; @@ -252,7 +252,7 @@ protected void configure() { install(new EpochsSyncModule()); // Storage directory - install(new NodeStorageLocationFromPropertiesModule()); + install(new StorageLocationFromPropertiesModule()); // State Computer var mempoolMaxMemory = properties.get( diff --git a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java index a973196386..abe3ca01e4 100644 --- a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java +++ b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java @@ -70,7 +70,7 @@ import com.radixdlt.api.system.health.HealthInfoService; import com.radixdlt.api.system.health.HealthInfoServiceImpl; import com.radixdlt.api.system.routes.*; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import io.undertow.server.HttpHandler; import java.nio.file.Path; @@ -116,7 +116,7 @@ public SystemApi systemApi(@SystemApiEndpoints Map ha @Provides @Singleton public SystemApiConfig systemApiConfig( - RuntimeProperties runtimeProperties, @NodeStorageLocation String nodeStorageLocation) { + RuntimeProperties runtimeProperties, @StateManagerStorageLocation String nodeStorageLocation) { final var dbCheckpointEnabled = runtimeProperties.get("api.system.enable_db_checkpoint", false); final var dbCheckpointsPath = runtimeProperties.get( diff --git a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java index 9486e6724b..d43d96df96 100644 --- a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java +++ b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java @@ -79,7 +79,7 @@ import com.radixdlt.lang.Unit; import com.radixdlt.networks.FixedNetworkGenesis; import com.radixdlt.networks.Network; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; import java.io.IOException; @@ -111,7 +111,7 @@ public static RadixNode runNewNode(RuntimeProperties properties) { RuntimeProperties properties, GenesisFromPropertiesLoader genesisFromPropertiesLoader, GenesisStore genesisStore, - @NodeStorageLocation String nodeStorageLocation) { + @StateManagerStorageLocation String nodeStorageLocation) { this.network = network; this.hasher = hasher; this.properties = properties; diff --git a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java index 96106f18e8..54d3d483b5 100644 --- a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java +++ b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java @@ -74,8 +74,8 @@ import com.radixdlt.modules.CryptoModule; import com.radixdlt.modules.MetricsModule; import com.radixdlt.networks.Network; -import com.radixdlt.store.NodeStorageLocation; -import com.radixdlt.store.NodeStorageLocationFromPropertiesModule; +import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.StorageLocationFromPropertiesModule; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; import java.util.Optional; @@ -98,12 +98,12 @@ public void configure() { bind(RadixNodeBootstrapper.class).in(Scopes.SINGLETON); install(new MetricsModule()); install(new CryptoModule()); - install(new NodeStorageLocationFromPropertiesModule()); + install(new StorageLocationFromPropertiesModule()); } @Provides @Singleton - GenesisStore genesisStore(@NodeStorageLocation String nodeStorageLocation) { + GenesisStore genesisStore(@StateManagerStorageLocation String nodeStorageLocation) { return new GenesisFileStore(new File(nodeStorageLocation, "executed_genesis")); } diff --git a/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java index d66f1260c8..3c57a6d906 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java @@ -71,7 +71,7 @@ import com.radixdlt.serialization.DeserializeException; import com.radixdlt.serialization.DsonOutput; import com.radixdlt.serialization.Serialization; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.Longs; import com.sleepycat.je.*; import java.io.File; @@ -101,7 +101,7 @@ public final class BerkeleySafetyStateStore implements PersistentSafetyStateStor public BerkeleySafetyStateStore( Serialization serialization, Metrics metrics, - @NodeStorageLocation String nodeStorageLocation, + @StateManagerStorageLocation String nodeStorageLocation, EnvironmentConfig envConfig) { this.serialization = Objects.requireNonNull(serialization); this.metrics = Objects.requireNonNull(metrics); diff --git a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java index 9fd1896fca..2075d26da1 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java @@ -69,7 +69,10 @@ import com.radixdlt.consensus.Vote; import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.consensus.bft.Round; +import com.radixdlt.safety.SafetyStateDTO; +import com.radixdlt.serialization.DeserializeException; import com.radixdlt.serialization.DsonOutput; +import com.radixdlt.serialization.Serialization; import com.radixdlt.serialization.SerializerConstants; import com.radixdlt.serialization.SerializerDummy; import com.radixdlt.serialization.SerializerId2; @@ -208,4 +211,12 @@ private Long getSerializerLockedRound() { public Vote getSerializerLastVote() { return lastVote.orElse(null); } + + public static SafetyState fromDto(Serialization serialization, SafetyStateDTO dto) throws DeserializeException { + return serialization.fromDson(dto.dsonEncodedContent(), SafetyState.class); + } + + public static SafetyStateDTO toDto(Serialization serialization, SafetyState safetyState) { + return new SafetyStateDTO(serialization.toDson(safetyState, DsonOutput.Output.PERSIST)); + } } diff --git a/core/src/main/java/com/radixdlt/p2p/P2PModule.java b/core/src/main/java/com/radixdlt/p2p/P2PModule.java index d82e7b2a02..8171c001a5 100644 --- a/core/src/main/java/com/radixdlt/p2p/P2PModule.java +++ b/core/src/main/java/com/radixdlt/p2p/P2PModule.java @@ -88,7 +88,7 @@ import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.ProtocolState; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; public final class P2PModule extends AbstractModule { @@ -163,7 +163,7 @@ public RadixNodeUri selfUri( BerkeleyAddressBookStore addressBookStore( Serialization serialization, Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { + @StateManagerStorageLocation String nodeStorageLocation) { return new BerkeleyAddressBookStore( serialization, metrics, diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java b/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java index 53f8dc732d..286e86ca5c 100644 --- a/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java @@ -72,7 +72,7 @@ import com.radixdlt.p2p.NodeId; import com.radixdlt.serialization.DsonOutput.Output; import com.radixdlt.serialization.Serialization; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.sleepycat.je.*; import java.io.File; import java.io.IOException; @@ -101,7 +101,7 @@ public final class BerkeleyAddressBookStore implements AddressBookPersistence { public BerkeleyAddressBookStore( Serialization serialization, Metrics metrics, - @NodeStorageLocation String nodeStorageLocation, + @StateManagerStorageLocation String nodeStorageLocation, EnvironmentConfig envConfig) { this.serialization = Objects.requireNonNull(serialization); this.metrics = Objects.requireNonNull(metrics); diff --git a/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java b/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java index fbedeeb34d..18f30bf25c 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java @@ -74,7 +74,7 @@ import com.radixdlt.monitoring.Metrics; import com.radixdlt.serialization.Serialization; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; public class BerkeleySafetyStoreModule extends AbstractModule { @@ -92,7 +92,7 @@ BerkeleySafetyStateStore safetyStateStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { + @StateManagerStorageLocation String nodeStorageLocation) { return new BerkeleySafetyStateStore( serialization, metrics, diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index 9d85ff52c5..11c09cb20f 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -94,6 +94,7 @@ import com.radixdlt.state.RustStateReader; import com.radixdlt.statecomputer.RustStateComputer; import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.sync.TransactionsAndProofReader; import com.radixdlt.testutil.TestStateReader; import com.radixdlt.transaction.LedgerSyncLimitsConfig; @@ -205,19 +206,29 @@ public void configure() { new AbstractModule() { @Provides @Singleton - DatabaseBackendConfig databaseBackendConfig( - @NodeStorageLocation String nodeStorageLocation) { + @StateManagerStorageLocation + DatabaseBackendConfig stateManagerDatabaseBackendConfig( + @StateManagerStorageLocation String nodeStorageLocation) { return new DatabaseBackendConfig( new File(nodeStorageLocation, "state_manager").getPath()); } + @Provides + @Singleton + @NodeStorageLocation + DatabaseBackendConfig NodeDatabaseBackendConfig( + @NodeStorageLocation String nodeStorageLocation) { + return new DatabaseBackendConfig(new File(nodeStorageLocation, "node").getPath()); + } + @Provides @Singleton private NodeRustEnvironment stateManager( MempoolRelayDispatcher mempoolRelayDispatcher, FatalPanicHandler fatalPanicHandler, Network network, - DatabaseBackendConfig databaseBackendConfig, + @StateManagerStorageLocation DatabaseBackendConfig stateManagerDatabaseBackendConfig, + @NodeStorageLocation DatabaseBackendConfig nodeDatabaseBackendConfig, DatabaseConfig databaseConfig) { return new NodeRustEnvironment( mempoolRelayDispatcher, @@ -226,7 +237,7 @@ private NodeRustEnvironment stateManager( NetworkDefinition.from(network), mempoolConfig, vertexLimitsConfigOpt, - databaseBackendConfig, + stateManagerDatabaseBackendConfig, databaseConfig, getLoggingConfig(), stateTreeGcConfig, @@ -234,7 +245,8 @@ private NodeRustEnvironment stateManager( ledgerSyncLimitsConfig, protocolConfig, noFees, - scenariosExecutionConfig)); + scenariosExecutionConfig), + new NodeConfig(nodeDatabaseBackendConfig)); } @Provides diff --git a/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java b/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java new file mode 100644 index 0000000000..f06ff9c29a --- /dev/null +++ b/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java @@ -0,0 +1,80 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.store; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import javax.inject.Qualifier; + +/** Specifies a folder where the state manager can store its files */ +@Qualifier +@Target({FIELD, PARAMETER, METHOD}) +@Retention(RUNTIME) +public @interface StateManagerStorageLocation {} diff --git a/core/src/main/java/com/radixdlt/store/NodeStorageLocationFromPropertiesModule.java b/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java similarity index 93% rename from core/src/main/java/com/radixdlt/store/NodeStorageLocationFromPropertiesModule.java rename to core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java index c42726ec35..ebcdec71e0 100644 --- a/core/src/main/java/com/radixdlt/store/NodeStorageLocationFromPropertiesModule.java +++ b/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java @@ -70,11 +70,18 @@ import com.radixdlt.utils.properties.RuntimeProperties; /** Reads node's storage location from properties */ -public final class NodeStorageLocationFromPropertiesModule extends AbstractModule { +public final class StorageLocationFromPropertiesModule extends AbstractModule { + @Provides + @Singleton + @StateManagerStorageLocation + private String stateManagerStorageLocation(RuntimeProperties properties) { + return properties.get("db.location", ".//RADIXDB"); + } + @Provides @Singleton @NodeStorageLocation private String nodeStorageLocation(RuntimeProperties properties) { - return properties.get("db.location", ".//RADIXDB"); + return properties.get("db.location.node", ".//RADIXDB/node"); } } diff --git a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java index 8535976384..b22e615f86 100644 --- a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java @@ -69,7 +69,7 @@ import com.google.inject.Singleton; import com.radixdlt.consensus.bft.Self; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import java.io.File; /** @@ -85,7 +85,7 @@ public PrefixedNodeStorageLocationModule(String baseLocation) { @Provides @Singleton - @NodeStorageLocation + @StateManagerStorageLocation private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { return new File(baseLocation, publicKey.toHex()).getPath(); } diff --git a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java index 9a8581252c..94331f146b 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java @@ -97,7 +97,7 @@ import com.radixdlt.rev2.modules.REv2StateManagerModule; import com.radixdlt.statecomputer.commit.ActiveValidatorInfo; import com.radixdlt.statecomputer.commit.LedgerHeader; -import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.transaction.REv2TransactionAndProofStore; import com.radixdlt.transactions.RawNotarizedTransaction; @@ -162,7 +162,7 @@ protected void configure() { new SelfValidatorInfo( ONLY_VALIDATOR_ID.getKey(), Optional.of(ONLY_VALIDATOR_ID))); bind(String.class) - .annotatedWith(NodeStorageLocation.class) + .annotatedWith(StateManagerStorageLocation.class) .toInstance(folder.getRoot().getAbsolutePath()); bind(FatalPanicHandler.class).toInstance(() -> {}); } diff --git a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java index 099e50e1ec..910387c5e0 100644 --- a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java +++ b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java @@ -80,10 +80,12 @@ import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.transaction.REv2TransactionAndProofStore; import com.radixdlt.transactions.PreparedNotarizedTransaction; -import com.radixdlt.transactions.RawNotarizedTransaction; + +import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; + import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -91,59 +93,13 @@ public final class RustMempoolTest { - private static final DatabaseConfig TEST_DATABASE_CONFIG = - new DatabaseConfig(false, false, false, false); - @Rule public TemporaryFolder folder = new TemporaryFolder(); - /** A no-op dispatcher of transactions to be relayed. */ - private static final MempoolRelayDispatcher NOOP_DISPATCHER = tx -> {}; - - /** - * A no-op fatal panic handler. Please note that a JNI-invoking test (like this one) will observe - * panics as runtime exceptions propagated up the stack (through JNI), which will fail the test - * gracefully anyway. - */ - private static final FatalPanicHandler NOOP_HANDLER = () -> {}; - - private static void initStateComputer(NodeRustEnvironment nodeRustEnvironment) { - final var metrics = new MetricsInitializer().initialize(); - final var genesisProvider = - RawGenesisDataWithHash.fromGenesisData(GenesisData.testingDefaultEmpty()); - new REv2LedgerInitializer( - new Blake2b256Hasher(DefaultSerialization.getInstance()), - new RustStateComputer(metrics, nodeRustEnvironment), - new REv2TransactionsAndProofReader( - new REv2TransactionAndProofStore(metrics, nodeRustEnvironment), - LedgerSyncLimitsConfig.defaults(), - metrics)) - .initialize(genesisProvider); - } - @Test public void test_rust_mempool_add() throws Exception { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - new DatabaseBackendConfig(folder.newFolder().getPath()), - TEST_DATABASE_CONFIG, - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - final var metrics = new MetricsInitializer().initialize(); - - try (var stateManager = new NodeRustEnvironment(NOOP_DISPATCHER, NOOP_HANDLER, config)) { - initStateComputer(stateManager); - final var rustMempool = new RustMempool(metrics, stateManager); + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + initStateComputer(nodeRustEnvironment); + final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); @@ -181,28 +137,9 @@ public void test_rust_mempool_add() throws Exception { @Test public void test_rust_mempool_getTxns() throws Exception { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - new DatabaseBackendConfig(folder.newFolder().getPath()), - TEST_DATABASE_CONFIG, - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - final var metrics = new MetricsInitializer().initialize(); - - try (var stateManager = new NodeRustEnvironment(NOOP_DISPATCHER, NOOP_HANDLER, config)) { - initStateComputer(stateManager); - final var rustMempool = new RustMempool(metrics, stateManager); + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + initStateComputer(nodeRustEnvironment); + final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); final var transaction3 = constructValidTransaction(0, 2); @@ -323,28 +260,9 @@ public void test_rust_mempool_getTxns() throws Exception { @Test public void test_rust_mempool_getRelayTxns() throws Exception { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - new DatabaseBackendConfig(folder.newFolder().getPath()), - TEST_DATABASE_CONFIG, - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - final var metrics = new MetricsInitializer().initialize(); - - try (var stateManager = new NodeRustEnvironment(NOOP_DISPATCHER, NOOP_HANDLER, config)) { - initStateComputer(stateManager); - final var rustMempool = new RustMempool(metrics, stateManager); + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + initStateComputer(nodeRustEnvironment); + final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); final var transaction3 = constructValidTransaction(0, 2); @@ -383,6 +301,51 @@ public void test_rust_mempool_getRelayTxns() throws Exception { } } + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } + + private static void initStateComputer(NodeRustEnvironment nodeRustEnvironment) { + final var metrics = new MetricsInitializer().initialize(); + final var genesisProvider = + RawGenesisDataWithHash.fromGenesisData(GenesisData.testingDefaultEmpty()); + new REv2LedgerInitializer( + new Blake2b256Hasher(DefaultSerialization.getInstance()), + new RustStateComputer(metrics, nodeRustEnvironment), + new REv2TransactionsAndProofReader( + new REv2TransactionAndProofStore(metrics, nodeRustEnvironment), + LedgerSyncLimitsConfig.defaults(), + metrics)) + .initialize(genesisProvider); + } + private static PreparedNotarizedTransaction constructValidTransaction(long fromEpoch, int nonce) { return TransactionBuilder.forTests().fromEpoch(fromEpoch).nonce(nonce).prepare(); } From 7d7e9925a1bc48f8d5f3607961d1c1214e2dde4e Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Fri, 19 Jul 2024 16:44:30 +0200 Subject: [PATCH 02/40] Intermediate commit: migration store, safety store and address book are working (address book is only partially tested) --- .../radixdlt/db/RocksDbMigrationStore.java | 3 +- .../radixdlt/p2p/RocksDbAddressBookStore.java | 118 ++++++++++-------- .../radixdlt/safety/RocksDbSafetyStore.java | 14 ++- .../db/RocksDbMigrationStoreTest.java | 69 ++++++++++ .../p2p/RocksDbAddressBookStoreTest.java | 76 +++++++++++ .../safety/RocksDbSafetyStoreTest.java | 92 +++++++++----- core-rust/node-common/src/java/utils.rs | 15 +++ .../state-manager/src/jni/address_book.rs | 12 +- .../state-manager/src/jni/migration_store.rs | 16 +-- .../state-manager/src/jni/safety_store.rs | 13 +- core-rust/state-manager/src/node.rs | 10 -- .../src/store/address_book_components.rs | 8 +- core-rust/state-manager/src/store/rocks_db.rs | 18 ++- core-rust/state-manager/src/store/traits.rs | 6 +- .../state-manager/src/store/typed_cf_api.rs | 9 ++ .../consensus/safety/SafetyState.java | 16 +-- 16 files changed, 350 insertions(+), 145 deletions(-) create mode 100644 core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java create mode 100644 core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java index 7731d62bc3..b374e0bca2 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java @@ -2,6 +2,7 @@ import com.google.common.reflect.TypeToken; import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.lang.Tuple.Tuple0; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; @@ -51,5 +52,5 @@ public void migrationDone(StoreId storeId) { } private final Natives.Call1 isMigratedFunc; - private final Natives.Call1 migrationDoneFunc; + private final Natives.Call1 migrationDoneFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java index 171db45666..56bf8c3471 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java @@ -7,60 +7,72 @@ import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; +import com.radixdlt.sbor.NodeSborCodecs; +import com.radixdlt.sbor.codec.Codec; import java.util.List; public class RocksDbAddressBookStore { - static { - System.loadLibrary("corerust"); - } - - private static native byte[] removeOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private static native byte[] upsertOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private static native byte[] reset(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private static native byte[] getAll(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - - public static RocksDbAddressBookStore create(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - return new RocksDbAddressBookStore(metrics, nodeRustEnvironment); - } - - private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - final var timer = metrics.stateManager().nativeCall(); - removeOneFunc = - Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::removeOne) - .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "removeOne"))) - .build(new TypeToken<>() {}); - upsertOneFunc = - Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::upsertOne) - .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "upsertOne"))) - .build(new TypeToken<>() {}); - resetFunc = - Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::reset) - .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "reset"))) - .build(new TypeToken<>() {}); - getAllFunc = - Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::getAll) - .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "getAll"))) - .build(new TypeToken<>() {}); - } - - boolean upsertEntry(AddressBookEntryDTO entry) { - return this.upsertOneFunc.call(entry); - } - - boolean removeEntry(NodeIdDTO nodeId) { - return this.removeOneFunc.call(nodeId); - } - - void reset() { - this.resetFunc.call(Tuple0.of()); - } - - ImmutableList getAllEntries() { - return ImmutableList.copyOf(this.getAllFunc.call(Tuple0.of())); - } - - private final Natives.Call1 removeOneFunc; - private final Natives.Call1 upsertOneFunc; - private final Natives.Call1 resetFunc; - private final Natives.Call1> getAllFunc; + static { + System.loadLibrary("corerust"); + } + + private static native byte[] removeOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] upsertOne(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] reset(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] getAll(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private final Codec dtoCodec; + + public static RocksDbAddressBookStore create( + Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbAddressBookStore(metrics, nodeRustEnvironment); + } + + private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + removeOneFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::removeOne) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "removeOne"))) + .build(new TypeToken<>() {}); + upsertOneFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::upsertOne) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "upsertOne"))) + .build(new TypeToken<>() {}); + resetFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::reset) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "reset"))) + .build(new TypeToken<>() {}); + getAllFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::getAll) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "getAll"))) + .build(new TypeToken<>() {}); + + dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); + } + + boolean upsertEntry(AddressBookEntryDTO entry) { + return this.upsertOneFunc.call(entry); + } + + boolean removeEntry(NodeIdDTO nodeId) { + return this.removeOneFunc.call(nodeId); + } + + void reset() { + this.resetFunc.call(Tuple0.of()); + } + + ImmutableList getAllEntries() { + return this.getAllFunc.call(tuple()).stream() + .map(value -> NodeSborCodecs.decode(value, dtoCodec)) + .collect(ImmutableList.toImmutableList()); + } + + private final Natives.Call1 removeOneFunc; + private final Natives.Call1 upsertOneFunc; + private final Natives.Call1 resetFunc; + private final Natives.Call1> getAllFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java index 19d18f2561..b6e088f093 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java @@ -4,9 +4,11 @@ import com.google.common.reflect.TypeToken; import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.lang.Option; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; -import java.util.Optional; +import com.radixdlt.sbor.NodeSborCodecs; +import com.radixdlt.sbor.codec.Codec; public class RocksDbSafetyStore { static { @@ -17,6 +19,8 @@ public class RocksDbSafetyStore { private static native byte[] get(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private final Codec dtoCodec; + public static RocksDbSafetyStore create( Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { return new RocksDbSafetyStore(metrics, nodeRustEnvironment); @@ -32,16 +36,18 @@ private RocksDbSafetyStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironm Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::get) .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "get"))) .build(new TypeToken<>() {}); + + dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } public void upsert(SafetyStateDTO state) { this.upsertFunc.call(state); } - public Optional get() { - return Optional.ofNullable(this.getFunc.call(Tuple0.of())); + public Option get() { + return this.getFunc.call(tuple()).map(value -> NodeSborCodecs.decode(value, dtoCodec)); } private final Natives.Call1 upsertFunc; - private final Natives.Call1 getFunc; + private final Natives.Call1> getFunc; } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java new file mode 100644 index 0000000000..d69fb2d874 --- /dev/null +++ b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java @@ -0,0 +1,69 @@ +package com.radixdlt.db; + +import static org.junit.Assert.*; + +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbMigrationStoreTest { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void migration_status_is_properly_reported() throws Exception { + var migrationStore = + RocksDbMigrationStore.create( + new MetricsInitializer().initialize(), createNodeRustEnvironment()); + + assertFalse(migrationStore.isMigrated(StoreId.SAFETY_STORE)); + assertFalse(migrationStore.isMigrated(StoreId.ADDRESS_BOOK)); + + migrationStore.migrationDone(StoreId.SAFETY_STORE); + assertTrue(migrationStore.isMigrated(StoreId.SAFETY_STORE)); + + migrationStore.migrationDone(StoreId.ADDRESS_BOOK); + assertTrue(migrationStore.isMigrated(StoreId.ADDRESS_BOOK)); + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } +} diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java new file mode 100644 index 0000000000..5c37ff8ffb --- /dev/null +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -0,0 +1,76 @@ +package com.radixdlt.p2p; + +import static com.radixdlt.lang.Option.some; +import static org.junit.Assert.*; + +import com.radixdlt.crypto.ECKeyPair; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import java.util.Set; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbAddressBookStoreTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test_address_book_entry_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + var nodeId = new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); + var entry = new AddressBookEntryDTO(nodeId, some(123L), Set.of("addr1", "addr2")); + + addressBookStore.upsertEntry(entry); + + var allEntries = addressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry, allEntries.get(0)); + } + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } +} diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index ff34477375..897796886d 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -1,10 +1,13 @@ package com.radixdlt.safety; +import com.google.common.primitives.Bytes; +import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.environment.*; import com.radixdlt.lang.Option; import com.radixdlt.mempool.RustMempoolConfig; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.ComponentAddress; import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import org.junit.Rule; @@ -12,49 +15,72 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; +import java.util.Collections; import static org.junit.Assert.*; public class RocksDbSafetyStoreTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test public void test_safety_state_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var safetyStore = RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); - var originalSafetyState = new SafetyStateDTO(new byte[] {1, 2, 3}); + var safetyStore = + RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + var address = + new ComponentAddress( + Bytes.toArray( + Collections.nCopies( + ComponentAddress.BYTE_LENGTH, + ComponentAddress.VALIDATOR_COMPONENT_ADDRESS_ENTITY_ID))); + + var originalSafetyState = + new SafetyStateDTO( + new BFTValidatorIdDTO(ECKeyPair.generateNew().getPublicKey(), address, "validator1"), + new RoundDTO(10L), + Option.none()); + + var empty = safetyStore.get(); + assertTrue(empty.isEmpty()); + + safetyStore.upsert(originalSafetyState); + var present = safetyStore.get(); + + assertTrue(present.isPresent()); + assertEquals(originalSafetyState, present.orElseThrow()); } } private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the test - // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); - } + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } } diff --git a/core-rust/node-common/src/java/utils.rs b/core-rust/node-common/src/java/utils.rs index 5af40d96c1..a6bcb45874 100644 --- a/core-rust/node-common/src/java/utils.rs +++ b/core-rust/node-common/src/java/utils.rs @@ -115,6 +115,21 @@ pub fn jni_sbor_coded_fallible_call( + env: &JNIEnv, + encoded_request: jbyteArray, + method: impl FnOnce(Vec) -> JavaResult, +) -> jbyteArray { + jni_call(env, || { + let result = jni_jbytearray_to_vector(env, encoded_request) + .and_then(method); + jni_slice_to_jbytearray(env, &result.to_java().unwrap()) + }) + .unwrap_or_else(std::ptr::null_mut) +} + /// Executes the given function in a way that is panic-safe on a JNI-originating stack. /// This is achieved by intercepting any panic (i.e. `catch_unwind()`) and throwing a Java-side /// `RustPanicException` instead. diff --git a/core-rust/state-manager/src/jni/address_book.rs b/core-rust/state-manager/src/jni/address_book.rs index 98cd0b5bd9..43c5a5d3e2 100644 --- a/core-rust/state-manager/src/jni/address_book.rs +++ b/core-rust/state-manager/src/jni/address_book.rs @@ -90,16 +90,16 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_upsertOne( env: JNIEnv, _class: JClass, j_rust_global_context: JObject, - node_data: jbyteArray, + address_entry: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_data, |request: AddressBookEntry| { + jni_sbor_coded_call(&env, address_entry, |request: AddressBookEntry| { // It should be safe to unwrap here, as this operation was already successful inside // jni_sbor_coded_call. - let entity = jni_jbytearray_to_vector(&env, node_data).unwrap(); + let entity = jni_jbytearray_to_vector(&env, address_entry).unwrap(); JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() - .upsert_one(&request.node_id, &entity); + .upsert_one(&request.node_id, &entity) }) } @@ -110,7 +110,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_reset( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .reset(); @@ -125,7 +125,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec> { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .get_all() diff --git a/core-rust/state-manager/src/jni/migration_store.rs b/core-rust/state-manager/src/jni/migration_store.rs index c04de6b7bd..fc1bdf1179 100644 --- a/core-rust/state-manager/src/jni/migration_store.rs +++ b/core-rust/state-manager/src/jni/migration_store.rs @@ -62,16 +62,18 @@ * permissions under this License. */ -use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::traits::node::MigrationStore; +use jni::JNIEnv; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; -use jni::JNIEnv; + use node_common::java::*; + +use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::migration::MigrationId; +use crate::traits::node::MigrationStore; #[no_mangle] -extern "system" fn Java_com_radixdlt_p2p_RocksDbMigrationStore_migrationDone( +extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_migrationDone( env: JNIEnv, _class: JClass, j_rust_global_context: JObject, @@ -85,16 +87,16 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbMigrationStore_migrationDone( } #[no_mangle] -extern "system" fn Java_com_radixdlt_p2p_RocksDbMigrationStore_isMigrated( +extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_isMigrated( env: JNIEnv, _class: JClass, j_rust_global_context: JObject, migration_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| { + jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| -> bool { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() - .is_migration_required(migration_store_id) + .is_migration_done(migration_store_id) }) } diff --git a/core-rust/state-manager/src/jni/safety_store.rs b/core-rust/state-manager/src/jni/safety_store.rs index 67657b564b..697c9fc1ee 100644 --- a/core-rust/state-manager/src/jni/safety_store.rs +++ b/core-rust/state-manager/src/jni/safety_store.rs @@ -71,27 +71,28 @@ use jni::JNIEnv; use node_common::java::*; #[no_mangle] -extern "system" fn Java_com_radixdlt_p2p_RocksDbSafetyStore_upsert( +extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_upsert( env: JNIEnv, _class: JClass, j_rust_global_context: JObject, - node_id: jbyteArray, + request: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |request: Vec| { + jni_raw_sbor_fallible_call(&env, request, |bytes| { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() - .upsert_safety_state(&request); + .upsert_safety_state(&bytes); + Ok(()) }) } #[no_mangle] -extern "system" fn Java_com_radixdlt_p2p_RocksDbSafetyStore_get( +extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_get( env: JNIEnv, _class: JClass, j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + jni_sbor_coded_call(&env, node_id, |_: ()| -> Option> { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .get_safety_state() diff --git a/core-rust/state-manager/src/node.rs b/core-rust/state-manager/src/node.rs index fac86c13c9..1be1d5b661 100644 --- a/core-rust/state-manager/src/node.rs +++ b/core-rust/state-manager/src/node.rs @@ -69,13 +69,3 @@ use crate::store::DatabaseBackendConfig; pub struct NodeConfig { pub database_backend_config: DatabaseBackendConfig, } - -impl NodeConfig { - pub fn new_for_testing(rocks_db_path: impl Into) -> Self { - NodeConfig { - database_backend_config: DatabaseBackendConfig { - rocks_db_path: rocks_db_path.into(), - }, - } - } -} diff --git a/core-rust/state-manager/src/store/address_book_components.rs b/core-rust/state-manager/src/store/address_book_components.rs index f2c7915447..90e1201a90 100644 --- a/core-rust/state-manager/src/store/address_book_components.rs +++ b/core-rust/state-manager/src/store/address_book_components.rs @@ -1,18 +1,18 @@ use crate::engine_prelude::Sbor; -/// Id of the node stored in the Address book (Secp256k1 public key) + +/// The ID of the node stored in the Address book (Secp256k1 public key) #[derive(Clone, Copy, Sbor)] -#[sbor(transparent)] pub struct AddressBookNodeId(pub [u8; AddressBookNodeId::LENGTH]); impl AddressBookNodeId { pub const LENGTH: usize = 33; - pub fn new(id: [u8; 33]) -> Self { + pub fn new(id: [u8; Self::LENGTH]) -> Self { Self(id) } - pub fn as_bytes(&self) -> &[u8; AddressBookNodeId::LENGTH] { + pub fn as_bytes(&self) -> &[u8; Self::LENGTH] { &self.0 } } diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs index 3ff08fe04d..e699ea2ce2 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -63,7 +63,6 @@ */ use std::collections::HashSet; - use crate::engine_prelude::*; use crate::store::traits::*; use crate::{ @@ -79,10 +78,8 @@ use rocksdb::{ }; use std::path::PathBuf; - use node_common::locks::Snapshottable; use tracing::{error, info, warn}; - use super::traits::extensions::*; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; use crate::address_book_components::AddressBookNodeId; @@ -431,11 +428,13 @@ impl AddressBookStore for NodeDatabase { } fn reset(&self) { - todo!("reset") + self.open_rw_context().cf(AddressBookCf).delete_all(); } - fn get_all(&self) -> Vec { - todo!("get_all") + fn get_all(&self) -> Vec> { + self.open_rw_context() + .cf(AddressBookCf) + .get_all() } } @@ -456,17 +455,16 @@ impl SafetyStateStore for NodeDatabase { self.open_rw_context().cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); } - fn get_safety_state(&self) -> Vec { + fn get_safety_state(&self) -> Option> { self.open_rw_context().cf(SafetyStoreCf) .get(&()) - .unwrap_or_default() } } impl MigrationStore for NodeDatabase { - fn is_migration_required(&self, store_id: MigrationId) -> bool { + fn is_migration_done(&self, store_id: MigrationId) -> bool { self.open_rw_context().cf(MigrationStatusCf) - .get(&store_id).is_none() + .get(&store_id).is_some() } fn migration_done(&self, store_id: MigrationId) { diff --git a/core-rust/state-manager/src/store/traits.rs b/core-rust/state-manager/src/store/traits.rs index 16d9d21f87..6361043e57 100644 --- a/core-rust/state-manager/src/store/traits.rs +++ b/core-rust/state-manager/src/store/traits.rs @@ -179,7 +179,7 @@ pub mod node { fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; fn reset(&self); - fn get_all(&self, ) -> Vec; + fn get_all(&self, ) -> Vec>; } pub trait HighPriorityPeersStore { @@ -189,11 +189,11 @@ pub mod node { pub trait SafetyStateStore { fn upsert_safety_state(&self, safety_state: &[u8]); - fn get_safety_state(&self) -> Vec; + fn get_safety_state(&self) -> Option>; } pub trait MigrationStore { - fn is_migration_required(&self, store_id: MigrationId) -> bool; + fn is_migration_done(&self, store_id: MigrationId) -> bool; fn migration_done(&self, store_id: MigrationId); } } diff --git a/core-rust/state-manager/src/store/typed_cf_api.rs b/core-rust/state-manager/src/store/typed_cf_api.rs index 49caa3ac6a..84ddfbdfa9 100644 --- a/core-rust/state-manager/src/store/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/typed_cf_api.rs @@ -240,6 +240,15 @@ impl<'r, 'w, KC: BoundedDbCodec, CF: TypedCf, R: WriteableRocks> self.cf.key_codec.upper_bound_encoding(), ); } + + /// Retrieves all entries. + pub fn get_all(&self) -> Vec { + self.rocks + .iterator_cf(self.cf.handle, IteratorMode::Start) + .into_iter() + .map(|(_, value)| self.cf.value_codec.decode(value.as_ref())) + .collect() + } } impl<'r, 'w, KC: GroupPreservingDbCodec, CF: TypedCf, R: WriteableRocks> diff --git a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java index 2075d26da1..c9a9b1e135 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java @@ -211,12 +211,12 @@ private Long getSerializerLockedRound() { public Vote getSerializerLastVote() { return lastVote.orElse(null); } - - public static SafetyState fromDto(Serialization serialization, SafetyStateDTO dto) throws DeserializeException { - return serialization.fromDson(dto.dsonEncodedContent(), SafetyState.class); - } - - public static SafetyStateDTO toDto(Serialization serialization, SafetyState safetyState) { - return new SafetyStateDTO(serialization.toDson(safetyState, DsonOutput.Output.PERSIST)); - } +// +// public static SafetyState fromDto(Serialization serialization, SafetyStateDTO dto) throws DeserializeException { +// return serialization.fromDson(dto.dsonEncodedContent(), SafetyState.class); +// } +// +// public static SafetyStateDTO toDto(Serialization serialization, SafetyState safetyState) { +// return new SafetyStateDTO(serialization.toDson(safetyState, DsonOutput.Output.PERSIST)); +// } } From ef03a2b1b8107764d327570b0e51934ee527e27c Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Tue, 23 Jul 2024 15:24:07 +0200 Subject: [PATCH 03/40] Intermediate commit: all stores are working and tested --- .../p2p/RocksDbHighPriorityPeersStore.java | 87 ++++++++++------- .../p2p/RocksDbAddressBookStoreTest.java | 96 ++++++++++++++++++- .../RocksDbHighPriorityPeersStoreTest.java | 95 ++++++++++++++++++ .../state-manager/src/jni/address_book.rs | 9 +- .../src/jni/high_priority_peers.rs | 14 ++- core-rust/state-manager/src/store/rocks_db.rs | 6 +- core-rust/state-manager/src/store/traits.rs | 2 +- .../state-manager/src/store/typed_cf_api.rs | 1 - .../consensus/safety/SafetyState.java | 11 --- 9 files changed, 257 insertions(+), 64 deletions(-) create mode 100644 core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java index cad0871806..12f02a034f 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java @@ -4,42 +4,63 @@ import com.google.common.reflect.TypeToken; import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.lang.Option; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; +import com.radixdlt.sbor.NodeSborCodecs; +import com.radixdlt.sbor.codec.Codec; import java.util.List; public class RocksDbHighPriorityPeersStore { - static { - System.loadLibrary("corerust"); - } - - private static native byte[] upsertAllHighPriorityPeers(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private static native byte[] getAllHighPriorityPeers(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - - public static RocksDbHighPriorityPeersStore create(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - return new RocksDbHighPriorityPeersStore(metrics, nodeRustEnvironment); - } - - private RocksDbHighPriorityPeersStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - final var timer = metrics.stateManager().nativeCall(); - upsertAllHighPriorityPeersFunc = - Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::upsertAllHighPriorityPeers) - .measure(timer.label(new Metrics.MethodId(RocksDbHighPriorityPeersStore.class, "upsertAllHighPriorityPeers"))) - .build(new TypeToken<>() {}); - getAllHighPriorityPeersFunc = - Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::getAllHighPriorityPeers) - .measure(timer.label(new Metrics.MethodId(RocksDbHighPriorityPeersStore.class, "getAllHighPriorityPeers"))) - .build(new TypeToken<>() {}); - } - - void storeHighPriorityPeers(List ids) { - this.upsertAllHighPriorityPeersFunc.call(ids); - } - - List getHighPriorityPeers() { - return this.getAllHighPriorityPeersFunc.call(Tuple0.of()); - } - - private final Natives.Call1, Tuple0> upsertAllHighPriorityPeersFunc; - private final Natives.Call1> getAllHighPriorityPeersFunc; + static { + System.loadLibrary("corerust"); + } + + private static native byte[] upsertAllHighPriorityPeers( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] getAllHighPriorityPeers( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private final Codec> dtoCodec; + + public static RocksDbHighPriorityPeersStore create( + Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + return new RocksDbHighPriorityPeersStore(metrics, nodeRustEnvironment); + } + + private RocksDbHighPriorityPeersStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { + final var timer = metrics.stateManager().nativeCall(); + upsertAllHighPriorityPeersFunc = + Natives.builder( + nodeRustEnvironment, RocksDbHighPriorityPeersStore::upsertAllHighPriorityPeers) + .measure( + timer.label( + new Metrics.MethodId( + RocksDbHighPriorityPeersStore.class, "upsertAllHighPriorityPeers"))) + .build(new TypeToken<>() {}); + getAllHighPriorityPeersFunc = + Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::getAllHighPriorityPeers) + .measure( + timer.label( + new Metrics.MethodId( + RocksDbHighPriorityPeersStore.class, "getAllHighPriorityPeers"))) + .build(new TypeToken<>() {}); + + dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); + } + + void storeHighPriorityPeers(List ids) { + this.upsertAllHighPriorityPeersFunc.call(ids); + } + + List getHighPriorityPeers() { + return this.getAllHighPriorityPeersFunc + .call(tuple()) + .map(value -> NodeSborCodecs.decode(value, dtoCodec)) + .or(List.of()); + } + + private final Natives.Call1, Tuple0> upsertAllHighPriorityPeersFunc; + private final Natives.Call1> getAllHighPriorityPeersFunc; } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index 5c37ff8ffb..f75281bf0e 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -1,5 +1,6 @@ package com.radixdlt.p2p; +import static com.radixdlt.lang.Option.none; import static com.radixdlt.lang.Option.some; import static org.junit.Assert.*; @@ -22,22 +23,107 @@ public class RocksDbAddressBookStoreTest { public TemporaryFolder folder = new TemporaryFolder(); @Test - public void test_address_book_entry_can_be_saved_and_restored() throws Exception { + public void test_address_book_entries_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { var addressBookStore = RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + // New store is empty var empty = addressBookStore.getAllEntries(); assertTrue(empty.isEmpty()); - var nodeId = new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); - var entry = new AddressBookEntryDTO(nodeId, some(123L), Set.of("addr1", "addr2")); + // Ensure keys are repeatable to make test deterministic + var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); + var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); - addressBookStore.upsertEntry(entry); + addressBookStore.upsertEntry(entry1); + // Store now contains one entry var allEntries = addressBookStore.getAllEntries(); assertEquals(1L, allEntries.size()); - assertEquals(entry, allEntries.get(0)); + assertEquals(entry1, allEntries.get(0)); + + // Ensure keys are repeatable to make test deterministic + var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); + var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); + + // Add another entry + addressBookStore.upsertEntry(entry2); + + allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + } + } + + @Test + public void test_address_book_entry_can_be_added_and_removed() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // Ensure keys are repeatable to make test deterministic + var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); + var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); + var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); + var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); + + addressBookStore.upsertEntry(entry1); + addressBookStore.upsertEntry(entry2); + + // Check that entries were added + var allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // Remove entry1 + var removed = addressBookStore.removeEntry(nodeId1); + assertTrue(removed); + + // Check that entry1 was removed + allEntries = addressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry2, allEntries.get(0)); + } + } + + @Test + public void test_address_book_can_be_reset() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // Ensure keys are repeatable to make test deterministic + var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); + var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); + var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); + var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); + + addressBookStore.upsertEntry(entry1); + addressBookStore.upsertEntry(entry2); + + // Check that entries were added + var allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // Reset store + addressBookStore.reset(); + + // Check that entry1 was removed + empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java new file mode 100644 index 0000000000..b757384398 --- /dev/null +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java @@ -0,0 +1,95 @@ +package com.radixdlt.p2p; + +import static com.radixdlt.lang.Option.some; +import static org.junit.Assert.*; + +import com.radixdlt.crypto.ECKeyPair; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbHighPriorityPeersStoreTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test_high_priority_peers_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var highPriorityPeersStore = + RocksDbHighPriorityPeersStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = highPriorityPeersStore.getHighPriorityPeers(); + assertTrue(empty.isEmpty()); + + var inputList1 = List.of(newNodeId(), newNodeId(), newNodeId()); + + // Store new list + highPriorityPeersStore.storeHighPriorityPeers(inputList1); + + // Retrieve same list back + var peers = highPriorityPeersStore.getHighPriorityPeers(); + assertEquals(3L, peers.size()); + assertEquals(inputList1, peers); + + // Overwrite with a new list + var inputList2 = List.of(newNodeId(), newNodeId(), newNodeId()); + + // Ensure lists are different + assertNotEquals(inputList1, inputList2); + + // Store new list + highPriorityPeersStore.storeHighPriorityPeers(inputList2); + + peers = highPriorityPeersStore.getHighPriorityPeers(); + assertEquals(3L, peers.size()); + assertEquals(inputList2, peers); + } + } + + private static NodeIdDTO newNodeId() { + return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } +} diff --git a/core-rust/state-manager/src/jni/address_book.rs b/core-rust/state-manager/src/jni/address_book.rs index 43c5a5d3e2..3353703c13 100644 --- a/core-rust/state-manager/src/jni/address_book.rs +++ b/core-rust/state-manager/src/jni/address_book.rs @@ -78,10 +78,10 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |request: AddressBookNodeId| { + jni_sbor_coded_call(&env, node_id, |request: AddressBookNodeId| -> bool { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() - .remove_one(&request); + .remove_one(&request) }) } @@ -92,7 +92,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_upsertOne( j_rust_global_context: JObject, address_entry: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, address_entry, |request: AddressBookEntry| { + jni_sbor_coded_call(&env, address_entry, |request: AddressBookEntry| -> bool { // It should be safe to unwrap here, as this operation was already successful inside // jni_sbor_coded_call. let entity = jni_jbytearray_to_vector(&env, address_entry).unwrap(); @@ -110,11 +110,10 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_reset( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec { + jni_sbor_coded_call(&env, node_id, |_: ()| { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .reset(); - vec![0] }) } diff --git a/core-rust/state-manager/src/jni/high_priority_peers.rs b/core-rust/state-manager/src/jni/high_priority_peers.rs index cd7b2a898f..b5b24a9154 100644 --- a/core-rust/state-manager/src/jni/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/high_priority_peers.rs @@ -75,12 +75,16 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAll env: JNIEnv, _class: JClass, j_rust_global_context: JObject, - node_id: jbyteArray, + payload: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |request: Vec| { + println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers"); + jni_raw_sbor_fallible_call(&env, payload, |bytes| { + println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers: bytes: {:?}", bytes); JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() - .upsert_all_peers(&request); + .upsert_all_peers(&bytes); + println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers: done"); + Ok(()) }) } @@ -89,9 +93,9 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHig env: JNIEnv, _class: JClass, j_rust_global_context: JObject, - node_id: jbyteArray, + payload: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_ : Vec| -> Vec { + jni_sbor_coded_call(&env, payload, |_: ()| -> Option> { JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .get_all_peers() diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs index e699ea2ce2..fd8a0583b8 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -147,10 +147,11 @@ const ALL_STATE_MANAGER_COLUMN_FAMILIES: [&str; 25] = [ BlueprintAndCreationIndexedObjectsCf::VERSIONED_NAME, ]; -const ALL_NODE_COLUMN_FAMILIES: [&str; 3] = [ +const ALL_NODE_COLUMN_FAMILIES: [&str; 4] = [ MigrationStatusCf::DEFAULT_NAME, AddressBookCf::DEFAULT_NAME, SafetyStoreCf::DEFAULT_NAME, + HighPriorityPeersCf::DEFAULT_NAME, ]; /// A redefined RocksDB's "key and value bytes" tuple (the original one lives in a private module). @@ -443,10 +444,9 @@ impl HighPriorityPeersStore for NodeDatabase { self.open_rw_context().cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); } - fn get_all_peers(&self) -> Vec { + fn get_all_peers(&self) -> Option> { self.open_rw_context().cf(HighPriorityPeersCf) .get(&()) - .unwrap_or_default() } } diff --git a/core-rust/state-manager/src/store/traits.rs b/core-rust/state-manager/src/store/traits.rs index 6361043e57..e573f814b9 100644 --- a/core-rust/state-manager/src/store/traits.rs +++ b/core-rust/state-manager/src/store/traits.rs @@ -184,7 +184,7 @@ pub mod node { pub trait HighPriorityPeersStore { fn upsert_all_peers(&self, peers: &[u8]); - fn get_all_peers(&self) -> Vec; + fn get_all_peers(&self) -> Option>; } pub trait SafetyStateStore { diff --git a/core-rust/state-manager/src/store/typed_cf_api.rs b/core-rust/state-manager/src/store/typed_cf_api.rs index 84ddfbdfa9..18de58727c 100644 --- a/core-rust/state-manager/src/store/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/typed_cf_api.rs @@ -245,7 +245,6 @@ impl<'r, 'w, KC: BoundedDbCodec, CF: TypedCf, R: WriteableRocks> pub fn get_all(&self) -> Vec { self.rocks .iterator_cf(self.cf.handle, IteratorMode::Start) - .into_iter() .map(|(_, value)| self.cf.value_codec.decode(value.as_ref())) .collect() } diff --git a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java index c9a9b1e135..9fd1896fca 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java @@ -69,10 +69,7 @@ import com.radixdlt.consensus.Vote; import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.consensus.bft.Round; -import com.radixdlt.safety.SafetyStateDTO; -import com.radixdlt.serialization.DeserializeException; import com.radixdlt.serialization.DsonOutput; -import com.radixdlt.serialization.Serialization; import com.radixdlt.serialization.SerializerConstants; import com.radixdlt.serialization.SerializerDummy; import com.radixdlt.serialization.SerializerId2; @@ -211,12 +208,4 @@ private Long getSerializerLockedRound() { public Vote getSerializerLastVote() { return lastVote.orElse(null); } -// -// public static SafetyState fromDto(Serialization serialization, SafetyStateDTO dto) throws DeserializeException { -// return serialization.fromDson(dto.dsonEncodedContent(), SafetyState.class); -// } -// -// public static SafetyStateDTO toDto(Serialization serialization, SafetyState safetyState) { -// return new SafetyStateDTO(serialization.toDson(safetyState, DsonOutput.Output.PERSIST)); -// } } From 641f31d85724b6c453e46d24e8b00a776863d39b Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Thu, 25 Jul 2024 16:56:10 +0200 Subject: [PATCH 04/40] Intermediate commit: address book implementation --- .../radixdlt/db/RocksDbMigrationStore.java | 64 ++++ .../main/java/com/radixdlt/db/StoreId.java | 68 +++- .../com/radixdlt/environment/NodeConfig.java | 3 +- .../environment/NodeRustEnvironment.java | 3 +- .../com/radixdlt/p2p/AddressBookEntryDTO.java | 78 +++- .../main/java/com/radixdlt/p2p/NodeIdDTO.java | 73 +++- .../com/radixdlt/p2p/PeerAddressEntryDTO.java | 112 ++++++ .../radixdlt/p2p/RocksDbAddressBookStore.java | 72 +++- .../p2p/RocksDbHighPriorityPeersStore.java | 83 +++- .../com/radixdlt/safety/BFTHeaderDTO.java | 64 ++++ .../radixdlt/safety/BFTValidatorIdDTO.java | 64 ++++ .../java/com/radixdlt/safety/HighQCDTO.java | 64 ++++ .../radixdlt/safety/QuorumCertificateDTO.java | 64 ++++ .../radixdlt/safety/RocksDbSafetyStore.java | 64 ++++ .../java/com/radixdlt/safety/RoundDTO.java | 64 ++++ .../com/radixdlt/safety/SafetyStateDTO.java | 77 +++- .../safety/TimeoutCertificateDTO.java | 64 ++++ .../safety/TimestampedECDSASignatureDTO.java | 64 ++++ .../safety/TimestampedECDSASignaturesDTO.java | 64 ++++ .../java/com/radixdlt/safety/VertexIdDTO.java | 64 ++++ .../java/com/radixdlt/safety/VoteDTO.java | 64 ++++ .../java/com/radixdlt/safety/VoteDataDTO.java | 64 ++++ .../com/radixdlt/sbor/NodeSborCodecs.java | 2 + .../db/RocksDbMigrationStoreTest.java | 66 +++- .../p2p/RocksDbAddressBookStoreTest.java | 362 +++++++++++------- .../RocksDbHighPriorityPeersStoreTest.java | 195 +++++++--- .../safety/RocksDbSafetyStoreTest.java | 73 +++- .../src/jni/high_priority_peers.rs | 28 +- .../src/store/address_book_components.rs | 16 +- core-rust/state-manager/src/store/rocks_db.rs | 76 ++-- core-rust/state-manager/src/store/traits.rs | 1 + .../radixdlt/api/system/SystemApiModule.java | 3 +- .../java/com/radixdlt/p2p/RadixNodeUri.java | 2 +- .../p2p/addressbook/AddressBookEntry.java | 4 + .../addressbook/RocksAddressBookStore.java | 211 ++++++++++ .../PrefixedNodeStorageLocationModule.java | 10 +- .../RocksAddressBookStoreTest.java | 220 +++++++++++ .../radixdlt/rev2/REv2StateComputerTest.java | 4 + .../com/radixdlt/rev2/RustMempoolTest.java | 31 +- 39 files changed, 2408 insertions(+), 297 deletions(-) create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java create mode 100644 core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java create mode 100644 core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java index b374e0bca2..f2be16a5ee 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.db; import com.google.common.reflect.TypeToken; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java b/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java index 9a8d18059c..48a6506361 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java @@ -1,11 +1,73 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.db; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.EnumCodec; -/** - * Java side representation of the StoreId enum in Rust - */ +/** Java side representation of the StoreId enum in Rust */ public sealed interface StoreId { record AddressBookStoreId() implements StoreId {} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java index d06a0d378c..79f2a74328 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java @@ -70,7 +70,6 @@ public record NodeConfig(DatabaseBackendConfig databaseBackendConfig) { public static void registerCodec(CodecMap codecMap) { codecMap.register( - NodeConfig.class, - codecs -> StructCodec.fromRecordComponents(NodeConfig.class, codecs)); + NodeConfig.class, codecs -> StructCodec.fromRecordComponents(NodeConfig.class, codecs)); } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java index 1ca32dadf0..996f2e98d0 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java @@ -131,7 +131,8 @@ public void handleFatalPanic() { this.fatalPanicHandler.handleFatalPanic(); } - private static native void init(NodeRustEnvironment nodeRustEnvironment, byte[] config, byte[] nodeConfig); + private static native void init( + NodeRustEnvironment nodeRustEnvironment, byte[] config, byte[] nodeConfig); private static native void cleanup(NodeRustEnvironment nodeRustEnvironment); } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java index e19a9af307..595625317e 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/AddressBookEntryDTO.java @@ -1,15 +1,79 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import com.radixdlt.lang.Option; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; - import java.util.Set; -public record AddressBookEntryDTO(NodeIdDTO nodeId, Option bannedUntil, Set knownAddresses) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - AddressBookEntryDTO.class, - codecs -> StructCodec.fromRecordComponents(AddressBookEntryDTO.class, codecs)); - } +public record AddressBookEntryDTO( + NodeIdDTO nodeId, Option bannedUntil, Set knownAddresses) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + AddressBookEntryDTO.class, + codecs -> StructCodec.fromRecordComponents(AddressBookEntryDTO.class, codecs)); + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java index 70534af617..b634b40580 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/NodeIdDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; @@ -5,9 +69,8 @@ import com.radixdlt.sbor.codec.StructCodec; public record NodeIdDTO(ECDSASecp256k1PublicKey publicKey) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - NodeIdDTO.class, - codecs -> StructCodec.fromRecordComponents(NodeIdDTO.class, codecs)); - } + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + NodeIdDTO.class, codecs -> StructCodec.fromRecordComponents(NodeIdDTO.class, codecs)); + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java new file mode 100644 index 0000000000..4cdf5b1863 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java @@ -0,0 +1,112 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.p2p; + +import com.radixdlt.lang.Option; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.EnumCodec; +import com.radixdlt.sbor.codec.StructCodec; +import java.util.Arrays; +import java.util.Objects; + +public record PeerAddressEntryDTO( + byte[] address, + Option latestConnectionStatus, + Option maybeFailedHandshake) { + public sealed interface ConnectionStatus { + record Success() implements ConnectionStatus {} + + record Failure() implements ConnectionStatus {} + + Success SUCCESS = new Success(); + Failure FAILURE = new Failure(); + } + + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + ConnectionStatus.class, + codecs -> EnumCodec.fromPermittedRecordSubclasses(ConnectionStatus.class, codecs)); + codecMap.register( + PeerAddressEntryDTO.class, + codecs -> StructCodec.fromRecordComponents(PeerAddressEntryDTO.class, codecs)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + return o instanceof PeerAddressEntryDTO that + && Objects.deepEquals(address, that.address) + && Objects.equals(maybeFailedHandshake, that.maybeFailedHandshake) + && Objects.equals(latestConnectionStatus, that.latestConnectionStatus); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(address), latestConnectionStatus, maybeFailedHandshake); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java index 56bf8c3471..aaafe4eb20 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import static com.radixdlt.lang.Tuple.*; @@ -53,19 +117,19 @@ private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnv dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } - boolean upsertEntry(AddressBookEntryDTO entry) { + public boolean upsertEntry(AddressBookEntryDTO entry) { return this.upsertOneFunc.call(entry); } - boolean removeEntry(NodeIdDTO nodeId) { + public boolean removeEntry(NodeIdDTO nodeId) { return this.removeOneFunc.call(nodeId); } - void reset() { + public void reset() { this.resetFunc.call(Tuple0.of()); } - ImmutableList getAllEntries() { + public ImmutableList getAllEntries() { return this.getAllFunc.call(tuple()).stream() .map(value -> NodeSborCodecs.decode(value, dtoCodec)) .collect(ImmutableList.toImmutableList()); diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java index 12f02a034f..689297f9d0 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import static com.radixdlt.lang.Tuple.*; @@ -22,6 +86,9 @@ private static native byte[] upsertAllHighPriorityPeers( private static native byte[] getAllHighPriorityPeers( NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] resetHighPriorityPeers( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private final Codec> dtoCodec; public static RocksDbHighPriorityPeersStore create( @@ -46,21 +113,33 @@ private RocksDbHighPriorityPeersStore(Metrics metrics, NodeRustEnvironment nodeR new Metrics.MethodId( RocksDbHighPriorityPeersStore.class, "getAllHighPriorityPeers"))) .build(new TypeToken<>() {}); + resetFunc = + Natives.builder(nodeRustEnvironment, RocksDbHighPriorityPeersStore::resetHighPriorityPeers) + .measure( + timer.label( + new Metrics.MethodId( + RocksDbHighPriorityPeersStore.class, "resetHighPriorityPeers"))) + .build(new TypeToken<>() {}); dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } - void storeHighPriorityPeers(List ids) { + public void storeHighPriorityPeers(List ids) { this.upsertAllHighPriorityPeersFunc.call(ids); } - List getHighPriorityPeers() { + public List getHighPriorityPeers() { return this.getAllHighPriorityPeersFunc .call(tuple()) .map(value -> NodeSborCodecs.decode(value, dtoCodec)) .or(List.of()); } + public void reset() { + this.resetFunc.call(tuple()); + } + private final Natives.Call1, Tuple0> upsertAllHighPriorityPeersFunc; private final Natives.Call1> getAllHighPriorityPeersFunc; + private final Natives.Call1 resetFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java index ba230b8e35..42d75f7e37 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java index 60c23de318..c65f7b9eb8 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java index 234cf0afa4..18c1d40b36 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java index c9f480801c..ed63b86657 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/QuorumCertificateDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java index b6e088f093..7338b9de51 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import static com.radixdlt.lang.Tuple.*; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java index 95c3496ac1..f26298b5b1 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RoundDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java index d1e0b40ba8..ca3e216c06 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/SafetyStateDTO.java @@ -1,13 +1,78 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.lang.Option; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; -public record SafetyStateDTO(BFTValidatorIdDTO validatorId, RoundDTO round, Option lastVote) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - SafetyStateDTO.class, - codecs -> StructCodec.fromRecordComponents(SafetyStateDTO.class, codecs)); - } +public record SafetyStateDTO( + BFTValidatorIdDTO validatorId, RoundDTO round, Option lastVote) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + SafetyStateDTO.class, + codecs -> StructCodec.fromRecordComponents(SafetyStateDTO.class, codecs)); + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java index d30d384c27..58d735d537 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimeoutCertificateDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java index 987715be15..d6f5fb87a9 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignatureDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.crypto.ECDSASecp256k1Signature; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java index 5a3c78a42a..d5cdc06e77 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/TimestampedECDSASignaturesDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java index 7ccb666b63..3948b5b852 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java index 57877693c4..9f5752a756 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.crypto.ECDSASecp256k1Signature; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java index 9821b91fdf..889e02bc4a 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; import com.radixdlt.sbor.codec.CodecMap; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index 4ea7afaed3..a68b9ddca1 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -78,6 +78,7 @@ import com.radixdlt.message.*; import com.radixdlt.p2p.AddressBookEntryDTO; import com.radixdlt.p2p.NodeIdDTO; +import com.radixdlt.p2p.PeerAddressEntryDTO; import com.radixdlt.prometheus.LedgerStatus; import com.radixdlt.prometheus.RecentSelfProposalMissStatistic; import com.radixdlt.protocol.*; @@ -225,6 +226,7 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { QuorumCertificateDTO.registerCodec(codecMap); RoundDTO.registerCodec(codecMap); SafetyStateDTO.registerCodec(codecMap); + PeerAddressEntryDTO.registerCodec(codecMap); TimeoutCertificateDTO.registerCodec(codecMap); TimestampedECDSASignatureDTO.registerCodec(codecMap); TimestampedECDSASignaturesDTO.registerCodec(codecMap); diff --git a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java index d69fb2d874..d9c54c0ffa 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.db; import static org.junit.Assert.*; @@ -59,7 +123,7 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { return new NodeRustEnvironment( tx -> {}, // A no-op dispatcher of transactions to be relayed. () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe + // one) will observe // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index f75281bf0e..03b6b6561b 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import static com.radixdlt.lang.Option.none; @@ -9,154 +73,180 @@ import com.radixdlt.lang.Option; import com.radixdlt.mempool.RustMempoolConfig; import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.p2p.PeerAddressEntryDTO.ConnectionStatus; import com.radixdlt.protocol.ProtocolConfig; import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import java.io.IOException; +import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; public class RocksDbAddressBookStoreTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void test_address_book_entries_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var addressBookStore = - RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); - - // New store is empty - var empty = addressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); - - // Ensure keys are repeatable to make test deterministic - var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); - var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); - - addressBookStore.upsertEntry(entry1); - - // Store now contains one entry - var allEntries = addressBookStore.getAllEntries(); - assertEquals(1L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - - // Ensure keys are repeatable to make test deterministic - var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); - var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); - - // Add another entry - addressBookStore.upsertEntry(entry2); - - allEntries = addressBookStore.getAllEntries(); - assertEquals(2L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - assertEquals(entry2, allEntries.get(1)); - } - } + private static final Random RANDOM = new Random(); - @Test - public void test_address_book_entry_can_be_added_and_removed() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var addressBookStore = - RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); - - // New store is empty - var empty = addressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); - - // Ensure keys are repeatable to make test deterministic - var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); - var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); - var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); - var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); - - addressBookStore.upsertEntry(entry1); - addressBookStore.upsertEntry(entry2); - - // Check that entries were added - var allEntries = addressBookStore.getAllEntries(); - assertEquals(2L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - assertEquals(entry2, allEntries.get(1)); - - // Remove entry1 - var removed = addressBookStore.removeEntry(nodeId1); - assertTrue(removed); - - // Check that entry1 was removed - allEntries = addressBookStore.getAllEntries(); - assertEquals(1L, allEntries.size()); - assertEquals(entry2, allEntries.get(0)); - } - } + @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Test - public void test_address_book_can_be_reset() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var addressBookStore = - RocksDbAddressBookStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); - - // New store is empty - var empty = addressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); - - // Ensure keys are repeatable to make test deterministic - var nodeId1 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {1}).getPublicKey()); - var entry1 = new AddressBookEntryDTO(nodeId1, some(123L), Set.of("addr1", "addr2")); - var nodeId2 = new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {2}).getPublicKey()); - var entry2 = new AddressBookEntryDTO(nodeId2, none(), Set.of("addr3", "addr4", "addr5")); - - addressBookStore.upsertEntry(entry1); - addressBookStore.upsertEntry(entry2); - - // Check that entries were added - var allEntries = addressBookStore.getAllEntries(); - assertEquals(2L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - assertEquals(entry2, allEntries.get(1)); - - // Reset store - addressBookStore.reset(); - - // Check that entry1 was removed - empty = addressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); - } - } + @Test + public void test_address_book_entries_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // Ensure keys are repeatable to make test deterministic + var entry1 = newAddressBookEntry(1); - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + addressBookStore.upsertEntry(entry1); + + // Store now contains one entry + var allEntries = addressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + + // Ensure keys are repeatable to make test deterministic + var entry2 = newAddressBookEntry(2); + + // Add another entry + addressBookStore.upsertEntry(entry2); + + allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + } + } + + @Test + public void test_address_book_entry_can_be_added_and_removed() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // Ensure keys are repeatable to make test deterministic + var entry1 = newAddressBookEntry(1); + var entry2 = newAddressBookEntry(2); + + addressBookStore.upsertEntry(entry1); + addressBookStore.upsertEntry(entry2); + + // Check that entries were added + var allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // Remove entry1 + var removed = addressBookStore.removeEntry(entry1.nodeId()); + assertTrue(removed); + + // Check that entry1 was removed + allEntries = addressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry2, allEntries.get(0)); + } + } + + @Test + public void test_address_book_can_be_reset() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + + // New store is empty + var empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // Ensure keys are repeatable to make test deterministic + var entry1 = newAddressBookEntry(1); + var entry2 = newAddressBookEntry(2); + + addressBookStore.upsertEntry(entry1); + addressBookStore.upsertEntry(entry2); + + // Check that entries were added + var allEntries = addressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // Reset store + addressBookStore.reset(); + + // Check that entry1 was removed + empty = addressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); } + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } + + private static AddressBookEntryDTO newAddressBookEntry(int id) { + return new AddressBookEntryDTO( + new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey()), + RANDOM.nextBoolean() ? some(RANDOM.nextLong()) : none(), + peerAddresses()); + } + + private static Set peerAddresses() { + return IntStream.range(0, RANDOM.nextInt(3)) + .mapToObj(__ -> newPeerAddressEntry()) + .collect(Collectors.toSet()); + } + + private static PeerAddressEntryDTO newPeerAddressEntry() { + var pubKey = ECKeyPair.generateNew().getPublicKey().getBytes(); + return new PeerAddressEntryDTO(pubKey, connectionStatus(), Option.some(RANDOM.nextLong())); + } + + private static Option connectionStatus() { + return RANDOM.nextBoolean() + ? Option.some(RANDOM.nextBoolean() ? ConnectionStatus.SUCCESS : ConnectionStatus.FAILURE) + : Option.none(); + } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java index b757384398..3031ed070f 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.p2p; import static com.radixdlt.lang.Option.some; @@ -18,78 +82,83 @@ import org.junit.rules.TemporaryFolder; public class RocksDbHighPriorityPeersStoreTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void test_high_priority_peers_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var highPriorityPeersStore = - RocksDbHighPriorityPeersStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + @Rule public TemporaryFolder folder = new TemporaryFolder(); - // New store is empty - var empty = highPriorityPeersStore.getHighPriorityPeers(); - assertTrue(empty.isEmpty()); + @Test + public void test_high_priority_peers_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var highPriorityPeersStore = + RocksDbHighPriorityPeersStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); - var inputList1 = List.of(newNodeId(), newNodeId(), newNodeId()); + // New store is empty + var empty = highPriorityPeersStore.getHighPriorityPeers(); + assertTrue(empty.isEmpty()); - // Store new list - highPriorityPeersStore.storeHighPriorityPeers(inputList1); + var inputList1 = List.of(newNodeId(), newNodeId(), newNodeId()); - // Retrieve same list back - var peers = highPriorityPeersStore.getHighPriorityPeers(); - assertEquals(3L, peers.size()); - assertEquals(inputList1, peers); + // Store new list + highPriorityPeersStore.storeHighPriorityPeers(inputList1); - // Overwrite with a new list - var inputList2 = List.of(newNodeId(), newNodeId(), newNodeId()); + // Retrieve same list back + var peers = highPriorityPeersStore.getHighPriorityPeers(); + assertEquals(3L, peers.size()); + assertEquals(inputList1, peers); - // Ensure lists are different - assertNotEquals(inputList1, inputList2); + // Overwrite with a new list + var inputList2 = List.of(newNodeId(), newNodeId(), newNodeId()); - // Store new list - highPriorityPeersStore.storeHighPriorityPeers(inputList2); + // Ensure lists are different + assertNotEquals(inputList1, inputList2); - peers = highPriorityPeersStore.getHighPriorityPeers(); - assertEquals(3L, peers.size()); - assertEquals(inputList2, peers); - } - } + // Store new list + highPriorityPeersStore.storeHighPriorityPeers(inputList2); - private static NodeIdDTO newNodeId() { - return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); - } + peers = highPriorityPeersStore.getHighPriorityPeers(); + assertEquals(3L, peers.size()); + assertEquals(inputList2, peers); - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + // Reset store + highPriorityPeersStore.reset(); + empty = highPriorityPeersStore.getHighPriorityPeers(); + assertTrue(empty.isEmpty()); } + } + + private static NodeIdDTO newNodeId() { + return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 897796886d..784e0aaf16 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -1,5 +1,71 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + package com.radixdlt.safety; +import static org.junit.Assert.*; + import com.google.common.primitives.Bytes; import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.environment.*; @@ -10,15 +76,12 @@ import com.radixdlt.rev2.ComponentAddress; import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import java.util.Collections; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import java.io.IOException; -import java.util.Collections; - -import static org.junit.Assert.*; - public class RocksDbSafetyStoreTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); diff --git a/core-rust/state-manager/src/jni/high_priority_peers.rs b/core-rust/state-manager/src/jni/high_priority_peers.rs index b5b24a9154..32df425cca 100644 --- a/core-rust/state-manager/src/jni/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/high_priority_peers.rs @@ -62,14 +62,16 @@ * permissions under this License. */ -use crate::engine_prelude::*; -use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::traits::node::HighPriorityPeersStore; +use jni::JNIEnv; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; -use jni::JNIEnv; + use node_common::java::*; +use crate::engine_prelude::*; +use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use crate::traits::node::HighPriorityPeersStore; + #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers( env: JNIEnv, @@ -77,13 +79,10 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAll j_rust_global_context: JObject, payload: jbyteArray, ) -> jbyteArray { - println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers"); jni_raw_sbor_fallible_call(&env, payload, |bytes| { - println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers: bytes: {:?}", bytes); JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) .lock() .upsert_all_peers(&bytes); - println!("Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers: done"); Ok(()) }) } @@ -102,4 +101,19 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHig }) } +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_resetHighPriorityPeers( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_: ()| { + JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) + .lock() + .reset_high_priority_peers(); + }) +} + + pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/store/address_book_components.rs b/core-rust/state-manager/src/store/address_book_components.rs index 90e1201a90..946202bef3 100644 --- a/core-rust/state-manager/src/store/address_book_components.rs +++ b/core-rust/state-manager/src/store/address_book_components.rs @@ -17,10 +17,24 @@ impl AddressBookNodeId { } } +/// Peer address entry with all components +#[derive(Clone, Sbor)] +pub struct PeerAddress { + pub encoded_uri: Vec, + pub latest_connection_status: Option, + pub last_seen: Option, +} + +#[derive(Clone, Copy, Sbor)] +pub enum ConnectionStatus { + Connected, + Disconnected, +} + /// Address book entry #[derive(Clone, Sbor)] pub struct AddressBookEntry { pub node_id: AddressBookNodeId, pub banned_until: Option, - pub known_addresses: Vec, + pub known_addresses: Vec, } diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs index fd8a0583b8..51b5ba6009 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -176,7 +176,7 @@ pub trait ReadableRocks { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_>; + ) -> Box + '_>; /// Gets a single value by key. fn get_pinned_cf( @@ -193,7 +193,7 @@ pub trait ReadableRocks { /// TODO(when the rustc feature mentioned above becomes stable): get rid of the `<'a>`. fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>>; } @@ -234,7 +234,7 @@ impl ReadableRocks for DirectRocks { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.db .iterator_cf(cf, mode) @@ -252,7 +252,7 @@ impl ReadableRocks for DirectRocks { fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>> { self.db .multi_get_cf(keys) @@ -320,7 +320,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.snapshot .iterator_cf(cf, mode) @@ -340,7 +340,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> { fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>> { self.snapshot .multi_get_cf(keys) @@ -409,23 +409,21 @@ impl AddressBookStore for NodeDatabase { fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { let binding = self.open_rw_context(); let context = binding.cf(AddressBookCf); - let exists = context.get(node_id).is_some(); - if exists { + if context.get(node_id).is_some() { context.delete(node_id); } - exists + true } fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { let binding = self.open_rw_context(); let context = binding.cf(AddressBookCf); - let exists = context.get(node_id).is_some(); context.put(node_id, &entry.to_vec()); - exists + true } fn reset(&self) { @@ -448,6 +446,10 @@ impl HighPriorityPeersStore for NodeDatabase { self.open_rw_context().cf(HighPriorityPeersCf) .get(&()) } + + fn reset_high_priority_peers(&self) { + self.open_rw_context().cf(HighPriorityPeersCf).delete(&()); + } } impl SafetyStateStore for NodeDatabase { @@ -544,7 +546,7 @@ impl ActualStateManagerDatabase { column_families, false, ) - .unwrap(); + .unwrap(); StateManagerDatabase { config: DatabaseConfig { @@ -579,7 +581,7 @@ impl ActualStateManagerDatabase { temp_path.as_path(), column_families, ) - .unwrap(); + .unwrap(); StateManagerDatabase { config: DatabaseConfig { @@ -1059,11 +1061,11 @@ impl ExecutedScenarioStore for StateManagerDatabase { pub struct RocksDBCommittedTransactionBundleIterator<'r> { state_version: StateVersion, - txns_iter: Box + 'r>, - ledger_receipts_iter: Box + 'r>, - local_executions_iter: Box + 'r>, + txns_iter: Box + 'r>, + ledger_receipts_iter: Box + 'r>, + local_executions_iter: Box + 'r>, identifiers_iter: - Box + 'r>, + Box + 'r>, } impl<'r> RocksDBCommittedTransactionBundleIterator<'r> { @@ -1141,7 +1143,7 @@ impl IterableTransactionStore for StateManagerDatabase { fn get_committed_transaction_bundle_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { // This should not happen. This interface should be used after checking (e.g. `core-api-server/src/core-api/handlers/`). // However, with or without this debug_assert there would still be a panic if LocalTransactionExecution is missing. debug_assert!(self.is_local_transaction_execution_index_enabled()); @@ -1260,7 +1262,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(LedgerProofsCf) @@ -1272,7 +1274,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_next_epoch_proof_iter( &self, from_epoch: Epoch, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(EpochLedgerProofsCf) @@ -1284,7 +1286,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_protocol_update_init_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(ProtocolUpdateInitLedgerProofsCf) @@ -1296,7 +1298,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_protocol_update_execution_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(ProtocolUpdateExecutionLedgerProofsCf) @@ -1377,8 +1379,8 @@ impl QueryableProofStore for StateManagerDatabase { 'proof_txns_loop: while payload_size_including_next_proof_txns <= max_payload_size_in_bytes && (latest_usable_proof.is_none() - || txns.len() + next_proof_txns.len() - <= (max_number_of_txns_if_more_than_one_proof as usize)) + || txns.len() + next_proof_txns.len() + <= (max_number_of_txns_if_more_than_one_proof as usize)) { match txns_iter.next() { Some((next_txn_state_version, next_txn)) => { @@ -1404,8 +1406,8 @@ impl QueryableProofStore for StateManagerDatabase { // that they can all fit in the response (the last txn could have crossed the limit) if payload_size_including_next_proof_txns <= max_payload_size_in_bytes && (latest_usable_proof.is_none() - || txns.len() + next_proof_txns.len() - <= (max_number_of_txns_if_more_than_one_proof as usize)) + || txns.len() + next_proof_txns.len() + <= (max_number_of_txns_if_more_than_one_proof as usize)) { // Yup, all good, use next_proof as the result and add its txns let next_proof_is_a_protocol_update = @@ -1540,7 +1542,7 @@ impl SubstateDatabase for StateManagerDatabase { &self, partition_key: &DbPartitionKey, from_sort_key: Option<&DbSortKey>, - ) -> Box + '_> { + ) -> Box + '_> { let partition_key = partition_key.clone(); let from_sort_key = from_sort_key.cloned().unwrap_or(DbSortKey(vec![])); Box::new( @@ -1553,7 +1555,7 @@ impl SubstateDatabase for StateManagerDatabase { } impl ListableSubstateDatabase for StateManagerDatabase { - fn list_partition_keys(&self) -> Box + '_> { + fn list_partition_keys(&self) -> Box + '_> { self.open_read_context() .cf(SubstatesCf) .iterate_key_groups() @@ -1563,7 +1565,7 @@ impl ListableSubstateDatabase for StateManagerDatabase { impl SubstateNodeAncestryStore for StateManagerDatabase { fn batch_get_ancestry<'a>( &self, - node_ids: impl IntoIterator, + node_ids: impl IntoIterator, ) -> Vec> { self.open_read_context() .cf(SubstateNodeAncestryRecordsCf) @@ -1580,7 +1582,7 @@ impl ReadableTreeStore for StateManagerDatabase { impl StateTreeGcStore for StateManagerDatabase { fn get_stale_tree_parts_iter( &self, - ) -> Box + '_> { + ) -> Box + '_> { self.open_read_context() .cf(StaleStateTreePartsCf) .iterate(Direction::Forward) @@ -1611,7 +1613,7 @@ impl StateTreeGcStore for StateManagerDatabase { ); } - fn batch_delete_node<'a>(&self, keys: impl IntoIterator) { + fn batch_delete_node<'a>(&self, keys: impl IntoIterator) { let db_context = self.open_rw_context(); let tree_nodes_cf = db_context.cf(StateTreeNodesCf); let associated_values_cf = db_context.cf(AssociatedStateTreeValuesCf); @@ -1653,7 +1655,7 @@ impl LedgerProofsGcStore for StateManagerDatabase { } impl ReadableAccuTreeStore - for StateManagerDatabase +for StateManagerDatabase { fn get_tree_slice( &self, @@ -1667,7 +1669,7 @@ impl ReadableAccuTreeStore } impl ReadableAccuTreeStore - for StateManagerDatabase +for StateManagerDatabase { fn get_tree_slice(&self, state_version: &StateVersion) -> Option> { self.open_read_context() @@ -1983,7 +1985,7 @@ impl RestoreDecember2023LostSubstates for StateManagerDatabas let substates_cf = db_context.cf(SubstatesCf); - let receipts_iter: Box> = + let receipts_iter: Box> = db_context .cf(TransactionReceiptsCf) .iterate_from(&StateVersion::of(1u64), Direction::Forward); @@ -2037,7 +2039,7 @@ impl IterableAccountChangeIndex for StateManagerDatabase { &self, account: GlobalAddress, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(AccountChangeStateVersionsCf) @@ -2053,7 +2055,7 @@ impl EntityListingIndex for StateManagerDatabase { &self, entity_type: EntityType, from_creation_id: Option<&CreationId>, - ) -> Box + '_> { + ) -> Box + '_> { let from_creation_id = from_creation_id.cloned().unwrap_or_else(CreationId::zero); Box::new( self.open_read_context() @@ -2067,7 +2069,7 @@ impl EntityListingIndex for StateManagerDatabase { &self, blueprint_id: &BlueprintId, from_creation_id: Option<&CreationId>, - ) -> Box + '_> { + ) -> Box + '_> { let BlueprintId { package_address, blueprint_name, diff --git a/core-rust/state-manager/src/store/traits.rs b/core-rust/state-manager/src/store/traits.rs index e573f814b9..c96e3e4df6 100644 --- a/core-rust/state-manager/src/store/traits.rs +++ b/core-rust/state-manager/src/store/traits.rs @@ -185,6 +185,7 @@ pub mod node { pub trait HighPriorityPeersStore { fn upsert_all_peers(&self, peers: &[u8]); fn get_all_peers(&self) -> Option>; + fn reset_high_priority_peers(&self); } pub trait SafetyStateStore { diff --git a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java index abe3ca01e4..e7cb60fcab 100644 --- a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java +++ b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java @@ -116,7 +116,8 @@ public SystemApi systemApi(@SystemApiEndpoints Map ha @Provides @Singleton public SystemApiConfig systemApiConfig( - RuntimeProperties runtimeProperties, @StateManagerStorageLocation String nodeStorageLocation) { + RuntimeProperties runtimeProperties, + @StateManagerStorageLocation String nodeStorageLocation) { final var dbCheckpointEnabled = runtimeProperties.get("api.system.enable_db_checkpoint", false); final var dbCheckpointsPath = runtimeProperties.get( diff --git a/core/src/main/java/com/radixdlt/p2p/RadixNodeUri.java b/core/src/main/java/com/radixdlt/p2p/RadixNodeUri.java index 0a2d30f3db..0591ab6115 100644 --- a/core/src/main/java/com/radixdlt/p2p/RadixNodeUri.java +++ b/core/src/main/java/com/radixdlt/p2p/RadixNodeUri.java @@ -127,7 +127,7 @@ public NodeId getNodeId() { } @JsonValue - private byte[] getSerializedValue() { + public byte[] getSerializedValue() { return getUriString().getBytes(StandardCharsets.UTF_8); } diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/AddressBookEntry.java b/core/src/main/java/com/radixdlt/p2p/addressbook/AddressBookEntry.java index cf5d061ad6..edced5f5c1 100644 --- a/core/src/main/java/com/radixdlt/p2p/addressbook/AddressBookEntry.java +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/AddressBookEntry.java @@ -393,6 +393,10 @@ public boolean failedHandshakeIsEmptyOrExpired() { return maybeFailedHandshake.isEmpty() || maybeFailedHandshake.get().isExpired(); } + public Optional getMaybeFailedHandshake() { + return maybeFailedHandshake; + } + @JsonProperty("latestConnectionStatus") @DsonOutput(DsonOutput.Output.ALL) private String getLatestConnectionStatusForSerializer() { diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java new file mode 100644 index 0000000000..1080cc62ab --- /dev/null +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java @@ -0,0 +1,211 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.p2p.addressbook; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.radixdlt.lang.Option; +import com.radixdlt.p2p.*; +import com.radixdlt.serialization.DeserializeException; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +public class RocksAddressBookStore implements AddressBookPersistence { + private final RocksDbAddressBookStore addressBookStore; + private final RocksDbHighPriorityPeersStore highPriorityPeersStore; + + public RocksAddressBookStore( + RocksDbAddressBookStore addressBookStore, + RocksDbHighPriorityPeersStore highPriorityPeersStore) { + this.addressBookStore = addressBookStore; + this.highPriorityPeersStore = highPriorityPeersStore; + } + + public void open() { + // no-op + } + + public void close() { + // no-op + } + + public void reset() { + addressBookStore.reset(); + highPriorityPeersStore.reset(); + } + + public boolean upsertEntry(AddressBookEntry entry) { + return addressBookStore.upsertEntry(toDTO(entry)); + } + + public boolean removeEntry(NodeId nodeId) { + return addressBookStore.removeEntry(toDTO(nodeId)); + } + + public ImmutableList getAllEntries() { + return ImmutableList.copyOf( + addressBookStore.getAllEntries().stream().map(RocksAddressBookStore::fromDTO).iterator()); + } + + public void storeHighPriorityPeers(List ids) { + highPriorityPeersStore.storeHighPriorityPeers( + ids.stream().map(RocksAddressBookStore::toDTO).collect(ImmutableList.toImmutableList())); + } + + public List getHighPriorityPeers() { + return highPriorityPeersStore.getHighPriorityPeers().stream() + .map(RocksAddressBookStore::fromDTO) + .collect(ImmutableList.toImmutableList()); + } + + private static AddressBookEntryDTO toDTO(AddressBookEntry entry) { + return new AddressBookEntryDTO( + new NodeIdDTO(entry.getNodeId().getPublicKey()), + Option.from(entry.bannedUntil().map(Instant::toEpochMilli)), + toDTO(entry.getKnownAddresses())); + } + + private static AddressBookEntry fromDTO(AddressBookEntryDTO dto) { + return new AddressBookEntry( + NodeId.fromPublicKey(dto.nodeId().publicKey()), + dto.bannedUntil().map(Instant::ofEpochMilli).toOptional(), + fromDTO(dto.knownAddresses())); + } + + private static NodeIdDTO toDTO(NodeId nodeId) { + return new NodeIdDTO(nodeId.getPublicKey()); + } + + private static NodeId fromDTO(NodeIdDTO dto) { + return NodeId.fromPublicKey(dto.publicKey()); + } + + private static PeerAddressEntryDTO toDTO(AddressBookEntry.PeerAddressEntry entry) { + var connectionStatus = + entry.getLatestConnectionStatus().map(RocksAddressBookStore::toConnectionStatus); + + var instant = + entry + .getMaybeFailedHandshake() + .map(AddressBookEntry.PeerAddressEntry.FailedHandshake::retainUntil) + .map(Instant::toEpochMilli); + return new PeerAddressEntryDTO( + entry.getUri().getSerializedValue(), Option.from(connectionStatus), Option.from(instant)); + } + + private static PeerAddressEntryDTO.ConnectionStatus toConnectionStatus( + AddressBookEntry.PeerAddressEntry.LatestConnectionStatus status) { + return switch (status) { + case SUCCESS -> PeerAddressEntryDTO.ConnectionStatus.SUCCESS; + case FAILURE -> PeerAddressEntryDTO.ConnectionStatus.FAILURE; + }; + } + + private static AddressBookEntry.PeerAddressEntry fromDTO(PeerAddressEntryDTO dto) { + return new AddressBookEntry.PeerAddressEntry( + deserializeRadixNodeUri(dto), + dto.latestConnectionStatus() + .map(RocksAddressBookStore::toLatestConnectionStatus) + .toOptional(), + dto.maybeFailedHandshake() + .map(Instant::ofEpochMilli) + .map(AddressBookEntry.PeerAddressEntry.FailedHandshake::new) + .toOptional()); + } + + private static AddressBookEntry.PeerAddressEntry.LatestConnectionStatus toLatestConnectionStatus( + PeerAddressEntryDTO.ConnectionStatus status) { + if (status instanceof PeerAddressEntryDTO.ConnectionStatus.Success) { + return AddressBookEntry.PeerAddressEntry.LatestConnectionStatus.SUCCESS; + } + if (status instanceof PeerAddressEntryDTO.ConnectionStatus.Failure) { + return AddressBookEntry.PeerAddressEntry.LatestConnectionStatus.FAILURE; + } + throw new IllegalStateException("Unknown connection status: " + status); + } + + private static RadixNodeUri deserializeRadixNodeUri(PeerAddressEntryDTO dto) { + RadixNodeUri nodeUri; + try { + nodeUri = RadixNodeUri.deserialize(dto.address()); + } catch (URISyntaxException | DeserializeException e) { + throw new RuntimeException(e); + } + return nodeUri; + } + + private static Set toDTO(Set input) { + return input.stream().map(RocksAddressBookStore::toDTO).collect(ImmutableSet.toImmutableSet()); + } + + private static ImmutableSet fromDTO( + Set input) { + return input.stream() + .map(RocksAddressBookStore::fromDTO) + .collect(ImmutableSet.toImmutableSet()); + } +} diff --git a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java index b22e615f86..c17e0c2ab2 100644 --- a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java @@ -69,6 +69,7 @@ import com.google.inject.Singleton; import com.radixdlt.consensus.bft.Self; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.store.StateManagerStorageLocation; import java.io.File; @@ -86,7 +87,14 @@ public PrefixedNodeStorageLocationModule(String baseLocation) { @Provides @Singleton @StateManagerStorageLocation - private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { + private String stateManagerStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { return new File(baseLocation, publicKey.toHex()).getPath(); } + + @Provides + @Singleton + @NodeStorageLocation + private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { + return new File(baseLocation + "/NODE", publicKey.toHex()).getPath(); + } } diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java new file mode 100644 index 0000000000..58cf401d4e --- /dev/null +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -0,0 +1,220 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.p2p.addressbook; + +import static com.radixdlt.lang.Option.some; +import static org.junit.Assert.*; + +import com.google.common.collect.ImmutableSet; +import com.radixdlt.crypto.ECDSASecp256k1PublicKey; +import com.radixdlt.crypto.ECKeyPair; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.p2p.*; +import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry; +import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.FailedHandshake; +import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.LatestConnectionStatus; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import java.time.Instant; +import java.util.Optional; +import java.util.Random; +import java.util.stream.IntStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksAddressBookStoreTest { + private static final Random RANDOM = new Random(); + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test_address_book_entries_can_be_saved_and_restored() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + var highPriorityPeersStore = + RocksDbHighPriorityPeersStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + try (var rocksAddressBookStore = + new RocksAddressBookStore(addressBookStore, highPriorityPeersStore)) { + + // New store is empty + var empty = rocksAddressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // One entry can be stored and restored + var entry1 = newAddressBookEntry(1); + assertTrue(rocksAddressBookStore.upsertEntry(entry1)); + + var allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + + // Another entry can be stored and restored + var entry2 = newAddressBookEntry(2); + assertTrue(rocksAddressBookStore.upsertEntry(entry2)); + + allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // An entry can be deleted + assertTrue(rocksAddressBookStore.removeEntry(entry1.getNodeId())); + + allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry2, allEntries.get(0)); + + // Address book can be reset + rocksAddressBookStore.reset(); + + empty = rocksAddressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + } + } + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } + + private static AddressBookEntry newAddressBookEntry(int id) { + var pubKey = ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey(); + var bannedUntil = newBannedUntil(); + + return new AddressBookEntry( + NodeId.fromPublicKey(pubKey), bannedUntil, newKnownAddresses(pubKey)); + } + + private static ImmutableSet newKnownAddresses(ECDSASecp256k1PublicKey pubKey) { + return IntStream.range(0, RANDOM.nextInt(3)) + .mapToObj(__ -> newPeerAddressEntry(pubKey)) + .collect(ImmutableSet.toImmutableSet()); + } + + private static PeerAddressEntry newPeerAddressEntry(ECDSASecp256k1PublicKey pubKey) { + var failedHandshake = newFailedHandshake(); + var latestConnectionStatus = newLatestConnectionStatus(); + var uri = RadixNodeUri.fromPubKeyAndAddress(1, pubKey, "127.0.0.1", 30000); + + return new PeerAddressEntry(uri, latestConnectionStatus, failedHandshake); + } + + private static Optional newLatestConnectionStatus() { + return RANDOM.nextBoolean() + ? Optional.of( + RANDOM.nextBoolean() ? LatestConnectionStatus.FAILURE : LatestConnectionStatus.SUCCESS) + : Optional.empty(); + } + + private static Optional newBannedUntil() { + return RANDOM.nextBoolean() + ? Optional.of(Instant.ofEpochMilli(Math.abs(RANDOM.nextLong()))) + : Optional.empty(); + } + + private static Optional newFailedHandshake() { + return RANDOM.nextBoolean() + ? Optional.of(new FailedHandshake(Instant.ofEpochMilli(Math.abs(RANDOM.nextLong())))) + : Optional.empty(); + } +} diff --git a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java index 94331f146b..edf7bdef62 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java @@ -97,6 +97,7 @@ import com.radixdlt.rev2.modules.REv2StateManagerModule; import com.radixdlt.statecomputer.commit.ActiveValidatorInfo; import com.radixdlt.statecomputer.commit.LedgerHeader; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.transaction.REv2TransactionAndProofStore; @@ -164,6 +165,9 @@ protected void configure() { bind(String.class) .annotatedWith(StateManagerStorageLocation.class) .toInstance(folder.getRoot().getAbsolutePath()); + bind(String.class) + .annotatedWith(NodeStorageLocation.class) + .toInstance(folder.getRoot().getAbsolutePath() + "/NODE"); bind(FatalPanicHandler.class).toInstance(() -> {}); } diff --git a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java index 910387c5e0..b937c49855 100644 --- a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java +++ b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java @@ -80,12 +80,10 @@ import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.transaction.REv2TransactionAndProofStore; import com.radixdlt.transactions.PreparedNotarizedTransaction; - import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; - import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -99,7 +97,8 @@ public final class RustMempoolTest { public void test_rust_mempool_add() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { initStateComputer(nodeRustEnvironment); - final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); + final var rustMempool = + new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); @@ -139,7 +138,8 @@ public void test_rust_mempool_add() throws Exception { public void test_rust_mempool_getTxns() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { initStateComputer(nodeRustEnvironment); - final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); + final var rustMempool = + new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); final var transaction3 = constructValidTransaction(0, 2); @@ -262,7 +262,8 @@ public void test_rust_mempool_getTxns() throws Exception { public void test_rust_mempool_getRelayTxns() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { initStateComputer(nodeRustEnvironment); - final var rustMempool = new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); + final var rustMempool = + new RustMempool(new MetricsInitializer().initialize(), nodeRustEnvironment); final var transaction1 = constructValidTransaction(0, 0); final var transaction2 = constructValidTransaction(0, 1); final var transaction3 = constructValidTransaction(0, 2); @@ -325,9 +326,11 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { return new NodeRustEnvironment( tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the test - // gracefully anyway. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. config, new NodeConfig(nodeDbConfig)); } @@ -337,12 +340,12 @@ private static void initStateComputer(NodeRustEnvironment nodeRustEnvironment) { final var genesisProvider = RawGenesisDataWithHash.fromGenesisData(GenesisData.testingDefaultEmpty()); new REv2LedgerInitializer( - new Blake2b256Hasher(DefaultSerialization.getInstance()), - new RustStateComputer(metrics, nodeRustEnvironment), - new REv2TransactionsAndProofReader( - new REv2TransactionAndProofStore(metrics, nodeRustEnvironment), - LedgerSyncLimitsConfig.defaults(), - metrics)) + new Blake2b256Hasher(DefaultSerialization.getInstance()), + new RustStateComputer(metrics, nodeRustEnvironment), + new REv2TransactionsAndProofReader( + new REv2TransactionAndProofStore(metrics, nodeRustEnvironment), + LedgerSyncLimitsConfig.defaults(), + metrics)) .initialize(genesisProvider); } From e45fde84e7e2117962b36b2989134449f99e583f Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 09:04:47 +0200 Subject: [PATCH 05/40] Intermediate commit: dependency injection issues, 8 tests failing. --- .../main/java/com/radixdlt/lang/Option.java | 10 + .../com/radixdlt/safety/BFTHeaderDTO.java | 3 +- .../com/radixdlt/safety/BFTValidatorDTO.java | 54 ++-- .../radixdlt/safety/BFTValidatorIdDTO.java | 3 +- .../java/com/radixdlt/safety/HighQCDTO.java | 3 +- .../com/radixdlt/safety/LedgerHeaderDTO.java | 87 +++++ .../com/radixdlt/safety/NextEpochDTO.java | 78 +++++ .../java/com/radixdlt/safety/VoteDataDTO.java | 3 +- .../com/radixdlt/sbor/NodeSborCodecs.java | 3 + .../safety/RocksDbSafetyStoreTest.java | 2 +- .../rev2/IncreasingValidatorsTest.java | 2 +- .../rev2/RandomValidatorsTest.java | 2 +- .../rev2/RandomVoteDropperTest.java | 2 +- .../deterministic/rev2/SanityTest.java | 2 +- .../MultiNodeRecoveryTest.java | 2 +- .../MultiNodeRebootTest.java | 8 +- .../RecoveryAfterTimeoutQuorumTest.java | 2 +- .../storage/TransactionDBSizeStressTest.java | 2 +- .../java/com/radixdlt/RadixNodeModule.java | 1 - .../safety/MigratingSafetyStore.java | 81 +++++ .../safety/RocksSafetyStateStore.java | 304 ++++++++++++++++++ .../consensus/safety/SafetyState.java | 5 + .../main/java/com/radixdlt/p2p/P2PModule.java | 20 -- .../addressbook/RocksAddressBookStore.java | 2 + .../rev2/modules/NodePersistenceModule.java | 178 ++++++++++ .../rev2/modules/REv2StateManagerModule.java | 16 +- .../radixdlt/store/SafetyStoreMigrator.java | 67 ++++ .../modules/FunctionalRadixNodeModule.java | 36 ++- .../safety/RocksSafetyStateStoreTest.java | 157 +++++++++ .../RocksAddressBookStoreTest.java | 97 +++--- .../p2p/test/P2PTestNetworkRunner.java | 28 ++ .../REv2ConsensusLedgerRecoveryTest.java | 2 +- .../recovery/REv2LedgerRecoveryTest.java | 2 +- .../rev2/REv2IncreasingEpochTest.java | 2 +- .../rev2/REv2LargeTransactionTest.java | 2 +- .../rev2/REv2RegisterValidatorTest.java | 2 +- .../rev2/REv2RejectMultipleIntentsTest.java | 2 +- .../REv2RejectedTransactionMempoolTest.java | 2 +- .../rev2/REv2RejectedTransactionTest.java | 2 +- .../protocol/AnemoneProtocolUpdateTest.java | 2 +- .../BottlenoseProtocolUpdateTest.java | 2 +- .../ProtocolUpdateWithEpochBoundsTest.java | 2 +- 42 files changed, 1151 insertions(+), 131 deletions(-) rename core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java => core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java (74%) create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java create mode 100644 core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java create mode 100644 core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java create mode 100644 core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java create mode 100644 core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java create mode 100644 core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java create mode 100644 core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java diff --git a/common/src/main/java/com/radixdlt/lang/Option.java b/common/src/main/java/com/radixdlt/lang/Option.java index 447c60c769..bf2e1a435c 100644 --- a/common/src/main/java/com/radixdlt/lang/Option.java +++ b/common/src/main/java/com/radixdlt/lang/Option.java @@ -218,6 +218,16 @@ default T or(T replacement) { return fold(Functions::id, () -> replacement); } + /** + * Return current value stored in current instance if current instance is present. If current + * instance is empty then return {@code null}. + * + * @return either value stored in current instance or {@code null} if current instance is empty + */ + default T toNullable() { + return fold(Functions::id, () -> null); + } + /** * Return current value stored in current instance if current instance is present. If current * instance is empty then return value returned by provided supplier. If current instance is not diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java index 42d75f7e37..50ece65e45 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTHeaderDTO.java @@ -66,9 +66,8 @@ import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; -import com.radixdlt.statecomputer.commit.LedgerHeader; -public record BFTHeaderDTO(RoundDTO round, VertexIdDTO vertexId, LedgerHeader ledgerHeader) { +public record BFTHeaderDTO(RoundDTO round, VertexIdDTO vertexId, LedgerHeaderDTO ledgerHeader) { public static void registerCodec(CodecMap codecMap) { codecMap.register( BFTHeaderDTO.class, codecs -> StructCodec.fromRecordComponents(BFTHeaderDTO.class, codecs)); diff --git a/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java similarity index 74% rename from core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java rename to core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java index 18f30bf25c..58a1ec208d 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/BerkeleySafetyStoreModule.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java @@ -62,41 +62,33 @@ * permissions under this License. */ -package com.radixdlt.rev2.modules; +package com.radixdlt.safety; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; -import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; -import com.radixdlt.consensus.safety.PersistentSafetyStateStore; -import com.radixdlt.environment.NodeAutoCloseable; -import com.radixdlt.monitoring.Metrics; -import com.radixdlt.serialization.Serialization; -import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.StateManagerStorageLocation; -import com.radixdlt.utils.properties.RuntimeProperties; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +import java.util.Arrays; +import java.util.Objects; + +public record BFTValidatorDTO(byte[] power, BFTValidatorIdDTO validatorId) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + BFTValidatorDTO.class, + codecs -> StructCodec.fromRecordComponents(BFTValidatorDTO.class, codecs)); + } -public class BerkeleySafetyStoreModule extends AbstractModule { @Override - protected void configure() { - bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); - Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) - .addBinding() - .to(BerkeleySafetyStateStore.class); + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o instanceof BFTValidatorDTO that + && Objects.deepEquals(power, that.power) + && Objects.equals(validatorId, that.validatorId); } - @Provides - @Singleton - BerkeleySafetyStateStore safetyStateStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { - return new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(power), validatorId); } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java index c65f7b9eb8..4e8157c6cc 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorIdDTO.java @@ -70,8 +70,7 @@ import com.radixdlt.sbor.codec.StructCodec; /** Representation of the validator suitable for Java-Rust exchange. */ -public record BFTValidatorIdDTO( - ECDSASecp256k1PublicKey key, ComponentAddress validatorAddress, String shortenedName) { +public record BFTValidatorIdDTO(ECDSASecp256k1PublicKey key, ComponentAddress validatorAddress) { public static void registerCodec(CodecMap codecMap) { codecMap.register( BFTValidatorIdDTO.class, diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java index 18c1d40b36..065c3b6085 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/HighQCDTO.java @@ -64,13 +64,14 @@ package com.radixdlt.safety; +import com.radixdlt.lang.Option; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; public record HighQCDTO( QuorumCertificateDTO highestQC, QuorumCertificateDTO highestCommittedQC, - TimeoutCertificateDTO highestTC) { + Option highestTC) { public static void registerCodec(CodecMap codecMap) { codecMap.register( HighQCDTO.class, codecs -> StructCodec.fromRecordComponents(HighQCDTO.class, codecs)); diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java new file mode 100644 index 0000000000..79d6d42548 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java @@ -0,0 +1,87 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.safety; + +import com.radixdlt.lang.Option; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; +import com.radixdlt.statecomputer.commit.LedgerHashes; + +public record LedgerHeaderDTO( + long epoch, + RoundDTO round, + long stateVersion, + LedgerHashes hashes, + long consensusParentRoundTimestampMs, + long proposerTimestampMs, + Option nextEpoch, + Option nextProtocolVersion +) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + LedgerHeaderDTO.class, + codecs -> StructCodec.fromRecordComponents(LedgerHeaderDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java new file mode 100644 index 0000000000..3ebac84d92 --- /dev/null +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java @@ -0,0 +1,78 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.safety; + +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; + +import java.util.Set; + +public record NextEpochDTO(long epoch, Set validators) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + NextEpochDTO.class, + codecs -> StructCodec.fromRecordComponents(NextEpochDTO.class, codecs)); + } +} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java index 889e02bc4a..310aa27146 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java @@ -64,10 +64,11 @@ package com.radixdlt.safety; +import com.radixdlt.lang.Option; import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; -public record VoteDataDTO(BFTHeaderDTO proposed, BFTHeaderDTO parent, BFTHeaderDTO committed) { +public record VoteDataDTO(BFTHeaderDTO proposed, BFTHeaderDTO parent, Option committed) { public static void registerCodec(CodecMap codecMap) { codecMap.register( VoteDataDTO.class, codecs -> StructCodec.fromRecordComponents(VoteDataDTO.class, codecs)); diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index a68b9ddca1..556b2b045d 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -221,8 +221,11 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { NodeConfig.registerCodec(codecMap); BFTHeaderDTO.registerCodec(codecMap); + BFTValidatorDTO.registerCodec(codecMap); BFTValidatorIdDTO.registerCodec(codecMap); HighQCDTO.registerCodec(codecMap); + LedgerHeaderDTO.registerCodec(codecMap); + NextEpochDTO.registerCodec(codecMap); QuorumCertificateDTO.registerCodec(codecMap); RoundDTO.registerCodec(codecMap); SafetyStateDTO.registerCodec(codecMap); diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 784e0aaf16..958a34c15c 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -99,7 +99,7 @@ public void test_safety_state_can_be_saved_and_restored() throws Exception { var originalSafetyState = new SafetyStateDTO( - new BFTValidatorIdDTO(ECKeyPair.generateNew().getPublicKey(), address, "validator1"), + new BFTValidatorIdDTO(ECKeyPair.generateNew().getPublicKey(), address), new RoundDTO(10L), Option.none()); diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/IncreasingValidatorsTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/IncreasingValidatorsTest.java index 8ec2acd5f8..8b38802e19 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/IncreasingValidatorsTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/IncreasingValidatorsTest.java @@ -115,7 +115,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - FunctionalRadixNodeModule.SafetyRecoveryConfig.BERKELEY_DB, + FunctionalRadixNodeModule.SafetyRecoveryConfig.REAL, FunctionalRadixNodeModule.ConsensusConfig.of(1000), FunctionalRadixNodeModule.LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomValidatorsTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomValidatorsTest.java index 4a402f190e..22d84eaae4 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomValidatorsTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomValidatorsTest.java @@ -118,7 +118,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - FunctionalRadixNodeModule.SafetyRecoveryConfig.BERKELEY_DB, + FunctionalRadixNodeModule.SafetyRecoveryConfig.REAL, FunctionalRadixNodeModule.ConsensusConfig.of(1000), FunctionalRadixNodeModule.LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomVoteDropperTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomVoteDropperTest.java index de017cfafa..bfa8597392 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomVoteDropperTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/RandomVoteDropperTest.java @@ -100,7 +100,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/SanityTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/SanityTest.java index 91b978709f..c572608802 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/SanityTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/SanityTest.java @@ -127,7 +127,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), epochs, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger/MultiNodeRecoveryTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger/MultiNodeRecoveryTest.java index 24442e703b..5d912c97d2 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger/MultiNodeRecoveryTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger/MultiNodeRecoveryTest.java @@ -134,7 +134,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), this.epochs, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger_sync/MultiNodeRebootTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger_sync/MultiNodeRebootTest.java index b4bb738c4f..9a92ac92a9 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger_sync/MultiNodeRebootTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/rev2/consensus_ledger_sync/MultiNodeRebootTest.java @@ -263,7 +263,7 @@ private void runTest( @Test public void restart_all_nodes_intermittently() { - runTest(new MixedLivenessEachRound(random, 0), SafetyRecoveryConfig.BERKELEY_DB); + runTest(new MixedLivenessEachRound(random, 0), SafetyRecoveryConfig.REAL); } @Test @@ -279,15 +279,13 @@ public void restart_all_nodes_intermittently_with_bad_liveness_recovery_should_f () -> runTest( new MixedLivenessEachRound(random, 0), - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, new MockedVertexStoreModule())) .isInstanceOf(DeterministicTest.NeverReachedStateException.class); } @Test public void restart_all_nodes_intermittently_while_f_nodes_down() { - runTest( - new MixedLivenessEachRound(random, (numValidators - 1) / 3), - SafetyRecoveryConfig.BERKELEY_DB); + runTest(new MixedLivenessEachRound(random, (numValidators - 1) / 3), SafetyRecoveryConfig.REAL); } } diff --git a/core/src/integration/java/com/radixdlt/integration/targeted/rev2/recovery/RecoveryAfterTimeoutQuorumTest.java b/core/src/integration/java/com/radixdlt/integration/targeted/rev2/recovery/RecoveryAfterTimeoutQuorumTest.java index 57558f5af6..a7f7940539 100644 --- a/core/src/integration/java/com/radixdlt/integration/targeted/rev2/recovery/RecoveryAfterTimeoutQuorumTest.java +++ b/core/src/integration/java/com/radixdlt/integration/targeted/rev2/recovery/RecoveryAfterTimeoutQuorumTest.java @@ -116,7 +116,7 @@ public void recovery_after_timeout_quorum_test() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/integration/java/com/radixdlt/integration/targeted/rev2/storage/TransactionDBSizeStressTest.java b/core/src/integration/java/com/radixdlt/integration/targeted/rev2/storage/TransactionDBSizeStressTest.java index a66a6e303d..247b95b820 100644 --- a/core/src/integration/java/com/radixdlt/integration/targeted/rev2/storage/TransactionDBSizeStressTest.java +++ b/core/src/integration/java/com/radixdlt/integration/targeted/rev2/storage/TransactionDBSizeStressTest.java @@ -106,7 +106,7 @@ private DeterministicTest buildTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/main/java/com/radixdlt/RadixNodeModule.java b/core/src/main/java/com/radixdlt/RadixNodeModule.java index 7e2675de94..2aeb488636 100644 --- a/core/src/main/java/com/radixdlt/RadixNodeModule.java +++ b/core/src/main/java/com/radixdlt/RadixNodeModule.java @@ -391,7 +391,6 @@ protected void configure() { ScenariosExecutionConfig.resolveForNetwork(network))); // Recovery - install(new BerkeleySafetyStoreModule()); install(new EpochsSafetyRecoveryModule()); install(new REv2LedgerRecoveryModule()); install(new REv2ConsensusRecoveryModule()); diff --git a/core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java b/core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java new file mode 100644 index 0000000000..248221a968 --- /dev/null +++ b/core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java @@ -0,0 +1,81 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.consensus.safety; + +import java.util.Optional; + +public class MigratingSafetyStore implements PersistentSafetyStateStore { + + @Override + public void commitState(SafetyState safetyState) {} + + @Override + public void close() {} + + @Override + public Optional get() { + return Optional.empty(); + } +} diff --git a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java new file mode 100644 index 0000000000..0dbc0cb177 --- /dev/null +++ b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java @@ -0,0 +1,304 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.consensus.safety; + +import com.google.common.hash.HashCode; +import com.radixdlt.consensus.*; +import com.radixdlt.consensus.bft.BFTValidator; +import com.radixdlt.consensus.bft.BFTValidatorId; +import com.radixdlt.consensus.bft.Round; +import com.radixdlt.crypto.ECDSASecp256k1Signature; +import com.radixdlt.lang.Option; +import com.radixdlt.safety.*; +import com.radixdlt.utils.UInt192; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.inject.Inject; + +public class RocksSafetyStateStore implements PersistentSafetyStateStore { + private final RocksDbSafetyStore rocksDbSafetyStore; + + @Inject + public RocksSafetyStateStore(RocksDbSafetyStore rocksDbSafetyStore) { + this.rocksDbSafetyStore = rocksDbSafetyStore; + } + + @Override + public void commitState(SafetyState safetyState) { + rocksDbSafetyStore.upsert(toDTO(safetyState)); + } + + @Override + public void close() { + // no-op + } + + @Override + public Optional get() { + return rocksDbSafetyStore.get().map(RocksSafetyStateStore::fromDTO).toOptional(); + } + + // ------------------------------------------------------------------------------------- + // Conversion to DTO + // ------------------------------------------------------------------------------------- + + private static SafetyStateDTO toDTO(SafetyState safetyState) { + var validatorId = toDTO(safetyState.getValidatorId()); + var round = toDTO(safetyState.getLockedRound()); + var vote = Option.from(safetyState.getLastVote().map(RocksSafetyStateStore::toDTO)); + return new SafetyStateDTO(validatorId, round, vote); + } + + private static VoteDTO toDTO(Vote vote) { + BFTValidatorIdDTO author = toDTO(vote.getAuthor()); + HighQCDTO highQC = toDTO(vote.highQC()); + VoteDataDTO voteData = toDTO(vote.getVoteData()); + long timestamp = vote.getTimestamp(); + ECDSASecp256k1Signature signature = vote.getSignature(); + Option timeoutSignature = Option.from(vote.getTimeoutSignature()); + + return new VoteDTO(author, highQC, voteData, timestamp, signature, timeoutSignature); + } + + private static HighQCDTO toDTO(HighQC highQC) { + return new HighQCDTO( + toDTO(highQC.highestQC()), + toDTO(highQC.highestCommittedQC()), + Option.from(highQC.highestTC().map(RocksSafetyStateStore::toDTO))); + } + + private static TimeoutCertificateDTO toDTO(TimeoutCertificate tc) { + return new TimeoutCertificateDTO( + tc.getEpoch(), toDTO(tc.getRound()), toDTO(tc.getTimestampedSignatures())); + } + + private static QuorumCertificateDTO toDTO(QuorumCertificate quorumCertificate) { + var signatures = toDTO(quorumCertificate.getTimestampedSignatures()); + var voteData = toDTO(quorumCertificate.getVoteData()); + + return new QuorumCertificateDTO(signatures, voteData); + } + + private static TimestampedECDSASignaturesDTO toDTO( + TimestampedECDSASignatures timestampedSignatures) { + return new TimestampedECDSASignaturesDTO( + timestampedSignatures.getSignatures().entrySet().stream() + .collect(Collectors.toMap(e -> toDTO(e.getKey()), e -> toDTO(e.getValue())))); + } + + private static TimestampedECDSASignatureDTO toDTO(TimestampedECDSASignature signature) { + return new TimestampedECDSASignatureDTO(signature.timestamp(), signature.signature()); + } + + private static VoteDataDTO toDTO(VoteData voteData) { + return new VoteDataDTO( + toDTO(voteData.getProposed()), + toDTO(voteData.getParent()), + Option.from(voteData.getCommitted().map(RocksSafetyStateStore::toDTO))); + } + + private static BFTHeaderDTO toDTO(BFTHeader header) { + return new BFTHeaderDTO( + toDTO(header.getRound()), toDTO(header.getVertexId()), toDTO(header.getLedgerHeader())); + } + + private static LedgerHeaderDTO toDTO(com.radixdlt.consensus.LedgerHeader ledgerHeader) { + return new LedgerHeaderDTO( + ledgerHeader.getEpoch(), + toDTO(ledgerHeader.getRound()), + ledgerHeader.getStateVersion(), + toDTO(ledgerHeader.getHashes()), + ledgerHeader.consensusParentRoundTimestamp(), + ledgerHeader.proposerTimestamp(), + ledgerHeader.getNextEpoch().map(RocksSafetyStateStore::toDTO), + ledgerHeader.nextProtocolVersion()); + } + + private static com.radixdlt.statecomputer.commit.LedgerHashes toDTO(LedgerHashes hashes) { + return new com.radixdlt.statecomputer.commit.LedgerHashes( + hashes.getStateRoot(), hashes.getTransactionRoot(), hashes.getReceiptRoot()); + } + + private static NextEpochDTO toDTO(NextEpoch epoch) { + return new NextEpochDTO( + epoch.getEpoch(), + epoch.getValidators().stream() + .map(RocksSafetyStateStore::toDTO) + .collect(Collectors.toSet())); + } + + private static BFTValidatorDTO toDTO(BFTValidator bftValidator) { + return new BFTValidatorDTO( + bftValidator.getPower().toBigEndianBytes(), toDTO(bftValidator.getValidatorId())); + } + + private static VertexIdDTO toDTO(HashCode vertexId) { + return new VertexIdDTO(vertexId.asBytes()); + } + + private static BFTValidatorIdDTO toDTO(BFTValidatorId validatorId) { + return new BFTValidatorIdDTO(validatorId.getKey(), validatorId.getValidatorAddress()); + } + + private static RoundDTO toDTO(Round round) { + return new RoundDTO(round.number()); + } + + // ------------------------------------------------------------------------------------- + // Conversion to domain object + // ------------------------------------------------------------------------------------- + + private static SafetyState fromDTO(SafetyStateDTO dto) { + return SafetyState.create( + fromDTO(dto.validatorId()), + fromDTO(dto.round()), + dto.lastVote().map(RocksSafetyStateStore::fromDTO).toOptional()); + } + + private static BFTValidatorId fromDTO(BFTValidatorIdDTO dto) { + return BFTValidatorId.create(dto.validatorAddress(), dto.key()); + } + + private static Round fromDTO(RoundDTO dto) { + return Round.of(dto.round()); + } + + private static Vote fromDTO(VoteDTO dto) { + return new Vote( + fromDTO(dto.author()), + fromDTO(dto.voteData()), + dto.timestamp(), + dto.signature(), + fromDTO(dto.highQC()), + dto.timeoutSignature().toOptional()); + } + + private static VoteData fromDTO(VoteDataDTO dto) { + return new VoteData( + fromDTO(dto.proposed()), + fromDTO(dto.parent()), + dto.committed().map(RocksSafetyStateStore::fromDTO).toNullable()); + } + + private static BFTHeader fromDTO(BFTHeaderDTO dto) { + return new BFTHeader( + fromDTO(dto.round()), fromDTO(dto.vertexId()), fromDTO(dto.ledgerHeader())); + } + + private static HashCode fromDTO(VertexIdDTO dto) { + return HashCode.fromBytes(dto.idBytes()); + } + + private static LedgerHeader fromDTO(LedgerHeaderDTO dto) { + return new LedgerHeader( + dto.epoch(), + fromDTO(dto.round()), + dto.stateVersion(), + fromDTO(dto.hashes()), + dto.consensusParentRoundTimestampMs(), + dto.proposerTimestampMs(), + dto.nextEpoch().map(RocksSafetyStateStore::fromDTO).toNullable(), + dto.nextProtocolVersion().toNullable()); + } + + private static LedgerHashes fromDTO(com.radixdlt.statecomputer.commit.LedgerHashes hashes) { + return LedgerHashes.create(hashes.stateRoot(), hashes.transactionRoot(), hashes.receiptRoot()); + } + + private static NextEpoch fromDTO(NextEpochDTO dto) { + return NextEpoch.create( + dto.epoch(), + dto.validators().stream().map(RocksSafetyStateStore::fromDTO).collect(Collectors.toSet())); + } + + private static BFTValidator fromDTO(BFTValidatorDTO dto) { + return BFTValidator.from(fromDTO(dto.validatorId()), UInt192.fromBigEndianBytes(dto.power())); + } + + private static HighQC fromDTO(HighQCDTO dto) { + return HighQC.from( + fromDTO(dto.highestQC()), + fromDTO(dto.highestCommittedQC()), + dto.highestTC().map(RocksSafetyStateStore::fromDTO).toOptional()); + } + + private static QuorumCertificate fromDTO(QuorumCertificateDTO dto) { + return new QuorumCertificate(fromDTO(dto.voteData()), fromDTO(dto.signatures())); + } + + private static TimestampedECDSASignatures fromDTO(TimestampedECDSASignaturesDTO dto) { + return new TimestampedECDSASignatures( + dto.nodeToTimestampedSignature().entrySet().stream() + .collect(Collectors.toMap(e -> fromDTO(e.getKey()), e -> fromDTO(e.getValue())))); + } + + private static TimestampedECDSASignature fromDTO(TimestampedECDSASignatureDTO dto) { + return TimestampedECDSASignature.from(dto.timestamp(), dto.signature()); + } + + private static TimeoutCertificate fromDTO(TimeoutCertificateDTO dto) { + return new TimeoutCertificate(dto.epoch(), fromDTO(dto.round()), fromDTO(dto.signatures())); + } +} diff --git a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java index 9fd1896fca..afac323ef2 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/SafetyState.java @@ -111,6 +111,11 @@ private SafetyState(BFTValidatorId validatorId, Round lockedRound, Optional lastVote) { + return new SafetyState(validatorId, lockedRound, lastVote); + } + static class Builder { private final SafetyState original; private Round lockedRound; diff --git a/core/src/main/java/com/radixdlt/p2p/P2PModule.java b/core/src/main/java/com/radixdlt/p2p/P2PModule.java index 8171c001a5..086b01dda9 100644 --- a/core/src/main/java/com/radixdlt/p2p/P2PModule.java +++ b/core/src/main/java/com/radixdlt/p2p/P2PModule.java @@ -71,12 +71,9 @@ import com.radixdlt.crypto.ECDSASecp256k1PublicKey; import com.radixdlt.environment.*; import com.radixdlt.ledger.LedgerProofBundle; -import com.radixdlt.monitoring.Metrics; import com.radixdlt.networks.Network; import com.radixdlt.p2p.addressbook.AddressBook; import com.radixdlt.p2p.addressbook.AddressBookPeerControl; -import com.radixdlt.p2p.addressbook.AddressBookPersistence; -import com.radixdlt.p2p.addressbook.BerkeleyAddressBookStore; import com.radixdlt.p2p.discovery.SeedNodesConfigParser; import com.radixdlt.p2p.hostip.HostIp; import com.radixdlt.p2p.hostip.HostIpModule; @@ -85,10 +82,7 @@ import com.radixdlt.p2p.transport.PeerServerBootstrap; import com.radixdlt.protocol.ProtocolUpdateEnactmentCondition.EnactAtStartOfEpochIfValidatorsReady; import com.radixdlt.protocol.ProtocolUpdateEnactmentCondition.EnactAtStartOfEpochUnconditionally; -import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.ProtocolState; -import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; public final class P2PModule extends AbstractModule { @@ -111,7 +105,6 @@ protected void configure() { bind(PeersView.class).to(PeerManagerPeersView.class).in(Scopes.SINGLETON); bind(PeerControl.class).to(AddressBookPeerControl.class).in(Scopes.SINGLETON); bind(PeerOutboundBootstrap.class).to(PeerOutboundBootstrapImpl.class).in(Scopes.SINGLETON); - bind(AddressBookPersistence.class).to(BerkeleyAddressBookStore.class); bind(PeerServerBootstrap.class).in(Scopes.SINGLETON); bind(PendingOutboundChannelsManager.class).in(Scopes.SINGLETON); bind(PeerManager.class).in(Scopes.SINGLETON); @@ -158,19 +151,6 @@ public RadixNodeUri selfUri( return RadixNodeUri.fromPubKeyAndAddress(network.getId(), selfKey, host, port); } - @Provides - @Singleton - BerkeleyAddressBookStore addressBookStore( - Serialization serialization, - Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { - return new BerkeleyAddressBookStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - } - /** * A start processor that unbans all peers if the node is booting within enactment bounds of an * upcoming protocol update. It is likely that any banned peers have been banned because the node diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java index 1080cc62ab..8a186447ed 100644 --- a/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java @@ -73,11 +73,13 @@ import java.time.Instant; import java.util.List; import java.util.Set; +import javax.inject.Inject; public class RocksAddressBookStore implements AddressBookPersistence { private final RocksDbAddressBookStore addressBookStore; private final RocksDbHighPriorityPeersStore highPriorityPeersStore; + @Inject public RocksAddressBookStore( RocksDbAddressBookStore addressBookStore, RocksDbHighPriorityPeersStore highPriorityPeersStore) { diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java new file mode 100644 index 0000000000..43bb31e880 --- /dev/null +++ b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java @@ -0,0 +1,178 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.rev2.modules; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; +import com.radixdlt.consensus.safety.PersistentSafetyStateStore; +import com.radixdlt.consensus.safety.RocksSafetyStateStore; +import com.radixdlt.db.RocksDbMigrationStore; +import com.radixdlt.db.StoreId; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.p2p.RocksDbAddressBookStore; +import com.radixdlt.p2p.RocksDbHighPriorityPeersStore; +import com.radixdlt.p2p.addressbook.AddressBookPersistence; +import com.radixdlt.p2p.addressbook.BerkeleyAddressBookStore; +import com.radixdlt.p2p.addressbook.RocksAddressBookStore; +import com.radixdlt.safety.RocksDbSafetyStore; +import com.radixdlt.serialization.Serialization; +import com.radixdlt.store.BerkeleyDbDefaults; +import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.utils.properties.RuntimeProperties; + +public class NodePersistenceModule extends AbstractModule { + @Override + protected void configure() {} + + @Provides + @Singleton + BerkeleySafetyStateStore bdbSafetyStateStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @StateManagerStorageLocation String nodeStorageLocation) { + return new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + + @Provides + @Singleton + BerkeleyAddressBookStore bdbAddressBookStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @StateManagerStorageLocation String nodeStorageLocation) { + return new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + + @Provides + @Singleton + RocksDbSafetyStore rocksDbSafetyStore(Metrics metrics, NodeRustEnvironment environment) { + return RocksDbSafetyStore.create(metrics, environment); + } + + @Provides + @Singleton + RocksDbAddressBookStore rocksDbAddressBookStore( + Metrics metrics, NodeRustEnvironment environment) { + return RocksDbAddressBookStore.create(metrics, environment); + } + + @Provides + @Singleton + RocksDbHighPriorityPeersStore rocksDbHighPriorityPeersStore( + Metrics metrics, NodeRustEnvironment environment) { + return RocksDbHighPriorityPeersStore.create(metrics, environment); + } + + @Provides + @Singleton + RocksDbMigrationStore rocksDbMigrationStore(Metrics metrics, NodeRustEnvironment environment) { + return RocksDbMigrationStore.create(metrics, environment); + } + + @Provides + @Singleton + AddressBookPersistence addressBookPersistence( + RocksDbMigrationStore migrationStore, + BerkeleyAddressBookStore berkeleyAddressBookStore, + RocksAddressBookStore addressBookStore) { + + try (berkeleyAddressBookStore) { + if (!migrationStore.isMigrated(StoreId.ADDRESS_BOOK)) { + berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); + addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); + } + } + + return addressBookStore; + } + + @Provides + @Singleton + PersistentSafetyStateStore persistentSafetyStateStore( + RocksDbMigrationStore migrationStore, + BerkeleySafetyStateStore berkeleySafetyStateStore, + RocksSafetyStateStore rocksSafetyStateStore) { + + try (berkeleySafetyStateStore) { + if (!migrationStore.isMigrated(StoreId.SAFETY_STORE)) { + berkeleySafetyStateStore.get().ifPresent(rocksSafetyStateStore::commitState); + } + } + + return rocksSafetyStateStore; + } +} diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index 11c09cb20f..4d20d8049f 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -117,6 +117,7 @@ public final class REv2StateManagerModule extends AbstractModule { private final ProtocolConfig protocolConfig; private final boolean noFees; private final ScenariosExecutionConfig scenariosExecutionConfig; + private final boolean installPersistence; private REv2StateManagerModule( ProposalLimitsConfig proposalLimitsConfig, @@ -129,7 +130,8 @@ private REv2StateManagerModule( LedgerSyncLimitsConfig ledgerSyncLimitsConfig, ProtocolConfig protocolConfig, boolean noFees, - ScenariosExecutionConfig scenariosExecutionConfig) { + ScenariosExecutionConfig scenariosExecutionConfig, + boolean installPersistence) { this.proposalLimitsConfig = proposalLimitsConfig; this.vertexLimitsConfigOpt = vertexLimitsConfigOpt; this.databaseConfig = databaseConfig; @@ -141,6 +143,7 @@ private REv2StateManagerModule( this.protocolConfig = protocolConfig; this.noFees = noFees; this.scenariosExecutionConfig = scenariosExecutionConfig; + this.installPersistence = installPersistence; } public static REv2StateManagerModule create( @@ -164,7 +167,8 @@ public static REv2StateManagerModule create( ledgerSyncLimitsConfig, protocolConfig, false, - scenariosExecutionConfig); + scenariosExecutionConfig, + true); } public static REv2StateManagerModule createForTesting( @@ -189,7 +193,8 @@ public static REv2StateManagerModule createForTesting( ledgerSyncLimitsConfig, protocolConfig, noFees, - scenariosExecutionConfig); + scenariosExecutionConfig, + false); } @Override @@ -326,6 +331,11 @@ EventProcessor onInsertUpdatePersistVertexStore( bind(new Key>() {}).to(RustMempool.class); bind(MempoolReevaluator.class).to(RustMempool.class); } + + if (installPersistence) { + // Moved here for convenience performing migration + install(new NodePersistenceModule()); + } } @ProvidesIntoSet diff --git a/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java b/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java new file mode 100644 index 0000000000..f982b8c07e --- /dev/null +++ b/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java @@ -0,0 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.store; + +public class SafetyStoreMigrator {} diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index 05c2ff8929..1d5975bf43 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -66,12 +66,16 @@ import com.google.inject.AbstractModule; import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; import com.radixdlt.consensus.*; import com.radixdlt.consensus.ProposalLimitsConfig; import com.radixdlt.consensus.bft.*; import com.radixdlt.consensus.epoch.EpochsConsensusModule; import com.radixdlt.consensus.liveness.ProposalGenerator; +import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; +import com.radixdlt.consensus.safety.PersistentSafetyStateStore; import com.radixdlt.consensus.sync.BFTSyncPatienceMillis; import com.radixdlt.environment.NoEpochsConsensusModule; import com.radixdlt.environment.NoEpochsSyncModule; @@ -83,13 +87,18 @@ import com.radixdlt.ledger.MockedLedgerRecoveryModule; import com.radixdlt.mempool.*; import com.radixdlt.modules.StateComputerConfig.*; +import com.radixdlt.monitoring.Metrics; import com.radixdlt.rev2.modules.*; +import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.MockedMempoolStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerWithEpochsModule; import com.radixdlt.statecomputer.RandomTransactionGenerator; +import com.radixdlt.store.BerkeleyDbDefaults; import com.radixdlt.store.InMemoryCommittedReaderModule; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.sync.SyncRelayConfig; +import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; import java.time.Duration; import org.junit.rules.TemporaryFolder; @@ -123,7 +132,7 @@ static NodeStorageConfig file(File folder) { public enum SafetyRecoveryConfig { MOCKED, - BERKELEY_DB, + REAL, } public static final class ConsensusConfig { @@ -316,7 +325,30 @@ public void configure() { switch (this.safetyRecoveryConfig) { case MOCKED -> install(new MockedSafetyStoreModule()); - case BERKELEY_DB -> install(new BerkeleySafetyStoreModule()); + case REAL -> install( + new AbstractModule() { + @Override + protected void configure() { + bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); + Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) + .addBinding() + .to(BerkeleySafetyStateStore.class); + } + + @Provides + @Singleton + BerkeleySafetyStateStore safetyStateStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @StateManagerStorageLocation String nodeStorageLocation) { + return new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + }); } // Consensus diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java new file mode 100644 index 0000000000..f98c382a11 --- /dev/null +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -0,0 +1,157 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.consensus.safety; + +import static com.radixdlt.lang.Option.some; +import static org.junit.Assert.*; + +import com.radixdlt.consensus.bft.BFTValidatorId; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.safety.RocksDbSafetyStore; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import java.util.Random; +import java.util.function.Consumer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksSafetyStateStoreTest { + private static final Random RANDOM = new Random(); + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test_address_book_entries_can_be_saved_and_restored() { + runTest( + safetyStore -> { + // Fresh store is empty + assertTrue(safetyStore.get().isEmpty()); + + // Save a safety state + var safetyState = SafetyState.initialState(BFTValidatorId.random()); + safetyStore.commitState(safetyState); + + // Retrieve the saved safety state + var restoredSafetyState = safetyStore.get(); + assertTrue(restoredSafetyState.isPresent()); + assertEquals(safetyState, restoredSafetyState.get()); + }); + } + + private void runTest(Consumer test) { + try (var environment = createNodeRustEnvironment()) { + var safetyStore = RocksDbSafetyStore.create(newMetrics(), environment); + try (var underTest = new RocksSafetyStateStore(safetyStore)) { + test.accept(underTest); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Metrics newMetrics() { + return new MetricsInitializer().initialize(); + } + + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config, + new NodeConfig(nodeDbConfig)); + } +} diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index 58cf401d4e..8b06949d9e 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -73,6 +73,7 @@ import com.radixdlt.environment.*; import com.radixdlt.lang.Option; import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.p2p.*; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry; @@ -85,6 +86,7 @@ import java.time.Instant; import java.util.Optional; import java.util.Random; +import java.util.function.Consumer; import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; @@ -96,54 +98,61 @@ public class RocksAddressBookStoreTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test - public void test_address_book_entries_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { - var addressBookStore = - RocksDbAddressBookStore.create( - new MetricsInitializer().initialize(), nodeRustEnvironment); - var highPriorityPeersStore = - RocksDbHighPriorityPeersStore.create( - new MetricsInitializer().initialize(), nodeRustEnvironment); - try (var rocksAddressBookStore = - new RocksAddressBookStore(addressBookStore, highPriorityPeersStore)) { - - // New store is empty - var empty = rocksAddressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); - - // One entry can be stored and restored - var entry1 = newAddressBookEntry(1); - assertTrue(rocksAddressBookStore.upsertEntry(entry1)); - - var allEntries = rocksAddressBookStore.getAllEntries(); - assertEquals(1L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - - // Another entry can be stored and restored - var entry2 = newAddressBookEntry(2); - assertTrue(rocksAddressBookStore.upsertEntry(entry2)); - - allEntries = rocksAddressBookStore.getAllEntries(); - assertEquals(2L, allEntries.size()); - assertEquals(entry1, allEntries.get(0)); - assertEquals(entry2, allEntries.get(1)); - - // An entry can be deleted - assertTrue(rocksAddressBookStore.removeEntry(entry1.getNodeId())); - - allEntries = rocksAddressBookStore.getAllEntries(); - assertEquals(1L, allEntries.size()); - assertEquals(entry2, allEntries.get(0)); - - // Address book can be reset - rocksAddressBookStore.reset(); - - empty = rocksAddressBookStore.getAllEntries(); - assertTrue(empty.isEmpty()); + public void test_address_book_entries_can_be_saved_and_restored() { + runTest( + rocksAddressBookStore -> { + // New store is empty + var empty = rocksAddressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + + // One entry can be stored and restored + var entry1 = newAddressBookEntry(1); + assertTrue(rocksAddressBookStore.upsertEntry(entry1)); + + var allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + + // Another entry can be stored and restored + var entry2 = newAddressBookEntry(2); + assertTrue(rocksAddressBookStore.upsertEntry(entry2)); + + allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(2L, allEntries.size()); + assertEquals(entry1, allEntries.get(0)); + assertEquals(entry2, allEntries.get(1)); + + // An entry can be deleted + assertTrue(rocksAddressBookStore.removeEntry(entry1.getNodeId())); + + allEntries = rocksAddressBookStore.getAllEntries(); + assertEquals(1L, allEntries.size()); + assertEquals(entry2, allEntries.get(0)); + + // Address book can be reset + rocksAddressBookStore.reset(); + + empty = rocksAddressBookStore.getAllEntries(); + assertTrue(empty.isEmpty()); + }); + } + + private void runTest(Consumer test) { + try (var environment = createNodeRustEnvironment()) { + var addressBook = RocksDbAddressBookStore.create(newMetrics(), environment); + var peersStore = RocksDbHighPriorityPeersStore.create(newMetrics(), environment); + try (var underTest = new RocksAddressBookStore(addressBook, peersStore)) { + test.accept(underTest); } + } catch (IOException e) { + throw new RuntimeException(e); } } + private static Metrics newMetrics() { + return new MetricsInitializer().initialize(); + } + private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index b7ebd6a73c..7e228e8803 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -74,10 +74,13 @@ import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.bft.Self; +import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; +import com.radixdlt.consensus.safety.PersistentSafetyStateStore; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; import com.radixdlt.crypto.ECKeyOps; import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.environment.Environment; +import com.radixdlt.environment.NodeAutoCloseable; import com.radixdlt.environment.StartProcessorOnRunner; import com.radixdlt.environment.deterministic.DeterministicProcessor; import com.radixdlt.environment.deterministic.network.ControlledDispatcher; @@ -101,6 +104,8 @@ import com.radixdlt.serialization.DefaultSerialization; import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.ProtocolState; +import com.radixdlt.store.BerkeleyDbDefaults; +import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.IOException; import java.util.Objects; @@ -188,6 +193,29 @@ private static Injector createInjector( throws ParseException { final var properties = RuntimeProperties.fromCommandLineArgs(new String[] {}); return Guice.createInjector( + new AbstractModule() { + @Override + protected void configure() { + bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); + Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) + .addBinding() + .to(BerkeleySafetyStateStore.class); + } + + @Provides + @Singleton + BerkeleySafetyStateStore safetyStateStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @StateManagerStorageLocation String nodeStorageLocation) { + return new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + }, Modules.override(new P2PModule(properties)) .with( new AbstractModule() { diff --git a/core/src/test/java/com/radixdlt/recovery/REv2ConsensusLedgerRecoveryTest.java b/core/src/test/java/com/radixdlt/recovery/REv2ConsensusLedgerRecoveryTest.java index 36f3f519fd..ada9a3ccfd 100644 --- a/core/src/test/java/com/radixdlt/recovery/REv2ConsensusLedgerRecoveryTest.java +++ b/core/src/test/java/com/radixdlt/recovery/REv2ConsensusLedgerRecoveryTest.java @@ -101,7 +101,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/recovery/REv2LedgerRecoveryTest.java b/core/src/test/java/com/radixdlt/recovery/REv2LedgerRecoveryTest.java index f1ff659d89..0d6c9eeac3 100644 --- a/core/src/test/java/com/radixdlt/recovery/REv2LedgerRecoveryTest.java +++ b/core/src/test/java/com/radixdlt/recovery/REv2LedgerRecoveryTest.java @@ -119,7 +119,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2IncreasingEpochTest.java b/core/src/test/java/com/radixdlt/rev2/REv2IncreasingEpochTest.java index 1f6cbb59d0..d60da7b6eb 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2IncreasingEpochTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2IncreasingEpochTest.java @@ -93,7 +93,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2LargeTransactionTest.java b/core/src/test/java/com/radixdlt/rev2/REv2LargeTransactionTest.java index cbc83068cb..3fade12bc3 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2LargeTransactionTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2LargeTransactionTest.java @@ -105,7 +105,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerWithSyncRelay( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2RegisterValidatorTest.java b/core/src/test/java/com/radixdlt/rev2/REv2RegisterValidatorTest.java index 77c58628a1..1483205151 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2RegisterValidatorTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2RegisterValidatorTest.java @@ -142,7 +142,7 @@ private DeterministicTest createTest() { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2RejectMultipleIntentsTest.java b/core/src/test/java/com/radixdlt/rev2/REv2RejectMultipleIntentsTest.java index d32f6e6420..73a566e8ca 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2RejectMultipleIntentsTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2RejectMultipleIntentsTest.java @@ -104,7 +104,7 @@ private DeterministicTest createTest(ProposalGenerator proposalGenerator) { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), false, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionMempoolTest.java b/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionMempoolTest.java index 89f4fe37c9..f8dfe44f47 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionMempoolTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionMempoolTest.java @@ -122,7 +122,7 @@ private DeterministicTest createTest(RustMempoolConfig mempoolConfig) { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), this.epochs, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionTest.java b/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionTest.java index 95c27f7757..941a3f38e3 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2RejectedTransactionTest.java @@ -119,7 +119,7 @@ private DeterministicTest createTest(ProposalGenerator proposalGenerator) { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), epochs, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(1000), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/protocol/AnemoneProtocolUpdateTest.java b/core/src/test/java/com/radixdlt/rev2/protocol/AnemoneProtocolUpdateTest.java index 640efc1250..1afe71b023 100644 --- a/core/src/test/java/com/radixdlt/rev2/protocol/AnemoneProtocolUpdateTest.java +++ b/core/src/test/java/com/radixdlt/rev2/protocol/AnemoneProtocolUpdateTest.java @@ -120,7 +120,7 @@ private DeterministicTest createTest(Module... extraModules) { new FunctionalRadixNodeModule( FunctionalRadixNodeModule.NodeStorageConfig.tempFolder(folder), true, - FunctionalRadixNodeModule.SafetyRecoveryConfig.BERKELEY_DB, + FunctionalRadixNodeModule.SafetyRecoveryConfig.REAL, FunctionalRadixNodeModule.ConsensusConfig.of(1000), FunctionalRadixNodeModule.LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() diff --git a/core/src/test/java/com/radixdlt/rev2/protocol/BottlenoseProtocolUpdateTest.java b/core/src/test/java/com/radixdlt/rev2/protocol/BottlenoseProtocolUpdateTest.java index 0816c263c5..0950b02b47 100644 --- a/core/src/test/java/com/radixdlt/rev2/protocol/BottlenoseProtocolUpdateTest.java +++ b/core/src/test/java/com/radixdlt/rev2/protocol/BottlenoseProtocolUpdateTest.java @@ -124,7 +124,7 @@ private DeterministicTest createTest(Module... extraModules) { new FunctionalRadixNodeModule( FunctionalRadixNodeModule.NodeStorageConfig.tempFolder(folder), true, - FunctionalRadixNodeModule.SafetyRecoveryConfig.BERKELEY_DB, + FunctionalRadixNodeModule.SafetyRecoveryConfig.REAL, FunctionalRadixNodeModule.ConsensusConfig.of(1000), FunctionalRadixNodeModule.LedgerConfig.stateComputerNoSync( new StateComputerConfig.REv2StateComputerConfig( diff --git a/core/src/test/java/com/radixdlt/rev2/protocol/ProtocolUpdateWithEpochBoundsTest.java b/core/src/test/java/com/radixdlt/rev2/protocol/ProtocolUpdateWithEpochBoundsTest.java index 0bb45d1a7b..384dd17dfe 100644 --- a/core/src/test/java/com/radixdlt/rev2/protocol/ProtocolUpdateWithEpochBoundsTest.java +++ b/core/src/test/java/com/radixdlt/rev2/protocol/ProtocolUpdateWithEpochBoundsTest.java @@ -291,7 +291,7 @@ private DeterministicTest createTest(ProtocolConfig protocolConfig) { new FunctionalRadixNodeModule( NodeStorageConfig.tempFolder(folder), true, - SafetyRecoveryConfig.BERKELEY_DB, + SafetyRecoveryConfig.REAL, ConsensusConfig.of(200), LedgerConfig.stateComputerNoSync( StateComputerConfig.rev2() From 19524e95e96d71cebfaa753465b99c4d51553b4b Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 11:25:23 +0200 Subject: [PATCH 06/40] Fix dependency injection in tests. --- .../com/radixdlt/safety/BFTValidatorDTO.java | 1 - .../com/radixdlt/safety/LedgerHeaderDTO.java | 13 ++++++------- .../java/com/radixdlt/safety/NextEpochDTO.java | 10 ++++------ .../java/com/radixdlt/safety/VoteDataDTO.java | 3 ++- .../radixdlt/p2p/test/P2PTestNetworkRunner.java | 16 ++++++++++++++++ 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java index 58a1ec208d..efc0465059 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java @@ -66,7 +66,6 @@ import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; - import java.util.Arrays; import java.util.Objects; diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java index 79d6d42548..6df20d8ede 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/LedgerHeaderDTO.java @@ -77,11 +77,10 @@ public record LedgerHeaderDTO( long consensusParentRoundTimestampMs, long proposerTimestampMs, Option nextEpoch, - Option nextProtocolVersion -) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - LedgerHeaderDTO.class, - codecs -> StructCodec.fromRecordComponents(LedgerHeaderDTO.class, codecs)); - } + Option nextProtocolVersion) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + LedgerHeaderDTO.class, + codecs -> StructCodec.fromRecordComponents(LedgerHeaderDTO.class, codecs)); + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java index 3ebac84d92..793aacfb1d 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java @@ -66,13 +66,11 @@ import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; - import java.util.Set; public record NextEpochDTO(long epoch, Set validators) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - NextEpochDTO.class, - codecs -> StructCodec.fromRecordComponents(NextEpochDTO.class, codecs)); - } + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + NextEpochDTO.class, codecs -> StructCodec.fromRecordComponents(NextEpochDTO.class, codecs)); + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java index 310aa27146..01e8d8d8f3 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VoteDataDTO.java @@ -68,7 +68,8 @@ import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; -public record VoteDataDTO(BFTHeaderDTO proposed, BFTHeaderDTO parent, Option committed) { +public record VoteDataDTO( + BFTHeaderDTO proposed, BFTHeaderDTO parent, Option committed) { public static void registerCodec(CodecMap codecMap) { codecMap.register( VoteDataDTO.class, codecs -> StructCodec.fromRecordComponents(VoteDataDTO.class, codecs)); diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index 7e228e8803..45b6320887 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -97,6 +97,7 @@ import com.radixdlt.p2p.*; import com.radixdlt.p2p.addressbook.AddressBook; import com.radixdlt.p2p.addressbook.AddressBookPersistence; +import com.radixdlt.p2p.addressbook.BerkeleyAddressBookStore; import com.radixdlt.p2p.capability.Capabilities; import com.radixdlt.p2p.transport.PeerOutboundBootstrap; import com.radixdlt.protocol.NewestProtocolVersion; @@ -196,6 +197,7 @@ private static Injector createInjector( new AbstractModule() { @Override protected void configure() { + bind(AddressBookPersistence.class).to(BerkeleyAddressBookStore.class); bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) .addBinding() @@ -215,6 +217,20 @@ BerkeleySafetyStateStore safetyStateStore( nodeStorageLocation, BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); } + + @Provides + @Singleton + BerkeleyAddressBookStore bdbAddressBookStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @StateManagerStorageLocation String nodeStorageLocation) { + return new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } }, Modules.override(new P2PModule(properties)) .with( From b830d179a6115810fee162732452bfc8932b92f9 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 11:39:07 +0200 Subject: [PATCH 07/40] Minor cleanup --- .../java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index 45b6320887..86330be65b 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -309,10 +309,7 @@ protected void configure() { } public void cleanup() { - this.nodes.forEach( - node -> { - node.injector.getInstance(AddressBookPersistence.class).close(); - }); + this.nodes.forEach(node -> node.injector.getInstance(AddressBookPersistence.class).close()); } public RadixNodeUri getUri(int nodeIndex) { From 9e55c3b8360ada64ed86b280e68bbd8c54841c65 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 11:53:55 +0200 Subject: [PATCH 08/40] Mark each store as migrated. --- .../java/com/radixdlt/rev2/modules/NodePersistenceModule.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java index 43bb31e880..7dfae3c494 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java @@ -154,6 +154,7 @@ AddressBookPersistence addressBookPersistence( if (!migrationStore.isMigrated(StoreId.ADDRESS_BOOK)) { berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); + migrationStore.migrationDone(StoreId.ADDRESS_BOOK); } } @@ -170,6 +171,7 @@ PersistentSafetyStateStore persistentSafetyStateStore( try (berkeleySafetyStateStore) { if (!migrationStore.isMigrated(StoreId.SAFETY_STORE)) { berkeleySafetyStateStore.get().ifPresent(rocksSafetyStateStore::commitState); + migrationStore.migrationDone(StoreId.SAFETY_STORE); } } From 34f667f619c1a4db0a4f4ab0f9faf811a03f2c7e Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 12:19:36 +0200 Subject: [PATCH 09/40] Remove unused class. --- .../radixdlt/store/SafetyStoreMigrator.java | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java diff --git a/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java b/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java deleted file mode 100644 index f982b8c07e..0000000000 --- a/core/src/main/java/com/radixdlt/store/SafetyStoreMigrator.java +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.store; - -public class SafetyStoreMigrator {} From b961bf3353b851493c8f10a6eeb03aa491e65bf4 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 29 Jul 2024 12:38:50 +0200 Subject: [PATCH 10/40] Update properties files and make all default paths consistent --- cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java | 3 ++- core/mainnet.config | 1 + core/src/main/resources/default.config | 3 +++ .../radixdlt/modules/PrefixedNodeStorageLocationModule.java | 2 +- core/stokenet.config | 1 + 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java b/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java index 71a8f662bb..c3160692d2 100644 --- a/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java +++ b/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java @@ -178,9 +178,10 @@ public Node build() throws Exception { dataDir = new File(Files.createTempDirectory("radix-shell-node-").toString()); } - customProperties.build().forEach((k, v) -> properties.set(k, v)); + customProperties.build().forEach(properties::set); properties.set("db.location", dataDir.toString()); + properties.set("db.location.node", new File(dataDir, "node").toString()); if (properties.get("node.key.path", "").isEmpty()) { properties.set("node.key.path", new File(dataDir, "node-keystore.ks").getAbsolutePath()); diff --git a/core/mainnet.config b/core/mainnet.config index a21fed8fdb..f4e30dab46 100644 --- a/core/mainnet.config +++ b/core/mainnet.config @@ -8,6 +8,7 @@ node.key.path=.//NODEMOUNT/mainnet/key/node.ks node.key.create_if_missing=true db.location=.//NODEMOUNT/mainnet/ledger +db.location.node=.//NODEMOUNT/mainnet/node network.p2p.seed_nodes=radix://node_rdx1qf2x63qx4jdaxj83kkw2yytehvvmu6r2xll5gcp6c9rancmrfsgfw0vnc65@babylon-mainnet-eu-west-1-node0.radixdlt.com,radix://node_rdx1qgxn3eeldj33kd98ha6wkjgk4k77z6xm0dv7mwnrkefknjcqsvhuu4gc609@babylon-mainnet-ap-southeast-2-node0.radixdlt.com,radix://node_rdx1qwrrnhzfu99fg3yqgk3ut9vev2pdssv7hxhff80msjmmcj968487uugc0t2@babylon-mainnet-ap-south-1-node0.radixdlt.com,radix://node_rdx1q0gnmwv0fmcp7ecq0znff7yzrt7ggwrp47sa9pssgyvrnl75tvxmvj78u7t@babylon-mainnet-us-east-1-node0.radixdlt.com diff --git a/core/src/main/resources/default.config b/core/src/main/resources/default.config index 1f3350a483..ebe4d18072 100755 --- a/core/src/main/resources/default.config +++ b/core/src/main/resources/default.config @@ -143,6 +143,9 @@ cp.port=8080 # Default: ./RADIXDB # db.location=./RADIXDB +# Location to create/find address book and safety store +# Default: ./RADIXDB/node +# db.location.node=./RADIXDB/node #### ## Debug configuration diff --git a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java index c17e0c2ab2..2f4c6211b7 100644 --- a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java @@ -95,6 +95,6 @@ private String stateManagerStorageLocation(@Self ECDSASecp256k1PublicKey publicK @Singleton @NodeStorageLocation private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { - return new File(baseLocation + "/NODE", publicKey.toHex()).getPath(); + return new File(new File(baseLocation, publicKey.toHex()), "node").getPath(); } } diff --git a/core/stokenet.config b/core/stokenet.config index 527d3ab41a..7689e70655 100644 --- a/core/stokenet.config +++ b/core/stokenet.config @@ -8,6 +8,7 @@ node.key.path=.//NODEMOUNT/stokenet/key/node.ks node.key.create_if_missing=true db.location=.//NODEMOUNT/stokenet/ledger +db.location.node=.//NODEMOUNT/stokenet/node network.p2p.seed_nodes=radix://node_tdx_2_1qv89yg0la2jt429vqp8sxtpg95hj637gards67gpgqy2vuvwe4s5ss0va2y@13.126.248.88,radix://node_tdx_2_1qvtd9ffdhxyg7meqggr2ezsdfgjre5aqs6jwk5amdhjg86xhurgn5c79t9t@13.210.209.103,radix://node_tdx_2_1qwfh2nn0zx8cut5fqfz6n7pau2f7vdyl89mypldnn4fwlhaeg2tvunp8s8h@54.229.126.97,radix://node_tdx_2_1qwz237kqdpct5l3yjhmna66uxja2ymrf3x6hh528ng3gtvnwndtn5rsrad4@3.210.187.161 From 5ddbfa261da540197c7153c87d02cd03def9ebea Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 7 Aug 2024 16:14:42 +0200 Subject: [PATCH 11/40] Separation of DB-related code at Rust side (step #1) --- .../src/core_api/conversions/lts.rs | 5 +- .../src/core_api/conversions/numerics.rs | 2 +- .../src/core_api/conversions/receipt.rs | 3 +- .../stream_account_transaction_outcomes.rs | 2 +- .../lts/stream_transaction_outcomes.rs | 2 +- .../handlers/lts/transaction_status.rs | 2 +- .../handlers/state_consensus_manager.rs | 3 +- .../handlers/status_network_status.rs | 2 +- .../src/core_api/handlers/status_scenarios.rs | 2 +- .../src/core_api/handlers/stream_proofs.rs | 5 +- .../core_api/handlers/stream_transactions.rs | 5 +- .../core_api/handlers/transaction_preview.rs | 2 +- .../core_api/handlers/transaction_receipt.rs | 2 +- .../core_api/handlers/transaction_status.rs | 2 +- .../core-api-server/src/core_api/helpers.rs | 5 +- .../src/engine_state_api/extras.rs | 2 +- .../engine_state_api/handlers/entity_info.rs | 2 +- .../handlers/extra_entity_search.rs | 4 +- core-rust/state-manager/src/commit_bundle.rs | 2 +- core-rust/state-manager/src/committer.rs | 4 +- .../state-manager/src/jni/address_book.rs | 4 +- .../src/jni/high_priority_peers.rs | 2 +- .../state-manager/src/jni/migration_store.rs | 4 +- .../src/jni/node_rust_environment.rs | 2 +- .../state-manager/src/jni/safety_store.rs | 2 +- .../src/jni/test_state_reader.rs | 4 +- .../src/jni/transaction_store.rs | 2 +- .../src/jni/vertex_store_recovery.rs | 2 +- core-rust/state-manager/src/metrics.rs | 4 +- .../src/protocol/protocol_state.rs | 4 +- .../definitions/anemone_definition.rs | 2 +- .../definitions/bottlenose_definition.rs | 2 +- .../definitions/custom_definition.rs | 2 +- .../definitions/default_definition.rs | 2 +- .../protocol_updates/definitions/mod.rs | 2 +- .../definitions/test_definition.rs | 2 +- .../protocol_update_definition.rs | 2 +- .../protocol_update_progress.rs | 2 +- core-rust/state-manager/src/protocol/test.rs | 2 +- core-rust/state-manager/src/staging/cache.rs | 6 +- core-rust/state-manager/src/staging/mod.rs | 4 +- .../src/staging/node_ancestry_resolver.rs | 2 +- .../state-manager/src/staging/overlays.rs | 2 +- core-rust/state-manager/src/staging/result.rs | 4 +- core-rust/state-manager/src/state_manager.rs | 6 +- core-rust/state-manager/src/store/codecs.rs | 4 +- .../src/store/column_families.rs | 12 +- .../state-manager/src/store/common/mod.rs | 1 + .../src/store/common/rocks_db.rs | 268 ++++++++++++++++++ .../state-manager/src/store/consensus/mod.rs | 2 + .../src/store/{ => consensus}/rocks_db.rs | 219 +------------- .../src/store/{ => consensus}/traits.rs | 32 +-- .../src/store/historical_state.rs | 7 +- core-rust/state-manager/src/store/jmt_gc.rs | 8 +- core-rust/state-manager/src/store/mod.rs | 28 +- .../{ => p2p}/address_book_components.rs | 0 .../src/store/{ => p2p}/migration.rs | 0 core-rust/state-manager/src/store/p2p/mod.rs | 3 + .../state-manager/src/store/p2p/traits.rs | 95 +++++++ .../state-manager/src/store/proofs_gc.rs | 14 +- .../state-manager/src/store/typed_cf_api.rs | 6 +- .../state-manager/src/system_executor.rs | 8 +- core-rust/state-manager/src/test/mod.rs | 2 +- .../src/transaction/preparation.rs | 2 +- .../state-manager/src/transaction/preview.rs | 2 +- .../src/transaction/series_execution.rs | 2 +- .../src/transaction/validation.rs | 6 +- 67 files changed, 501 insertions(+), 350 deletions(-) create mode 100644 core-rust/state-manager/src/store/common/mod.rs create mode 100644 core-rust/state-manager/src/store/common/rocks_db.rs create mode 100644 core-rust/state-manager/src/store/consensus/mod.rs rename core-rust/state-manager/src/store/{ => consensus}/rocks_db.rs (92%) rename core-rust/state-manager/src/store/{ => consensus}/traits.rs (97%) rename core-rust/state-manager/src/store/{ => p2p}/address_book_components.rs (100%) rename core-rust/state-manager/src/store/{ => p2p}/migration.rs (100%) create mode 100644 core-rust/state-manager/src/store/p2p/mod.rs create mode 100644 core-rust/state-manager/src/store/p2p/traits.rs diff --git a/core-rust/core-api-server/src/core_api/conversions/lts.rs b/core-rust/core-api-server/src/core_api/conversions/lts.rs index d9dfe70638..e6a10d47ca 100644 --- a/core-rust/core-api-server/src/core_api/conversions/lts.rs +++ b/core-rust/core-api-server/src/core_api/conversions/lts.rs @@ -1,6 +1,7 @@ use crate::engine_prelude::*; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; -use state_manager::store::traits::SubstateNodeAncestryStore; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::consensus::traits::SubstateNodeAncestryStore; use state_manager::{ CommittedTransactionIdentifiers, LedgerTransactionOutcome, LocalTransactionReceipt, StateVersion, TransactionTreeHash, diff --git a/core-rust/core-api-server/src/core_api/conversions/numerics.rs b/core-rust/core-api-server/src/core_api/conversions/numerics.rs index 575da7754c..45d55c4f19 100644 --- a/core-rust/core-api-server/src/core_api/conversions/numerics.rs +++ b/core-rust/core-api-server/src/core_api/conversions/numerics.rs @@ -3,7 +3,7 @@ use std::any::type_name; use std::ops::RangeInclusive; use crate::engine_prelude::*; -use state_manager::store::traits::scenario::ScenarioSequenceNumber; +use state_manager::store::consensus::traits::scenario::ScenarioSequenceNumber; use state_manager::StateVersion; use crate::core_api::models; diff --git a/core-rust/core-api-server/src/core_api/conversions/receipt.rs b/core-rust/core-api-server/src/core_api/conversions/receipt.rs index 982afd5ba7..df8c552064 100644 --- a/core-rust/core-api-server/src/core_api/conversions/receipt.rs +++ b/core-rust/core-api-server/src/core_api/conversions/receipt.rs @@ -4,7 +4,8 @@ use super::addressing::*; use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::{ ApplicationEvent, BySubstate, DetailedTransactionOutcome, LedgerStateChanges, LocalTransactionReceipt, PartitionChangeAction, PartitionReference, SubstateChangeAction, diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs b/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs index 6dbd3eefe8..aff88c7eb2 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs @@ -1,5 +1,5 @@ use crate::core_api::*; -use state_manager::store::traits::{ +use state_manager::store::consensus::traits::{ extensions::IterableAccountChangeIndex, ConfigurableDatabase, QueryableProofStore, QueryableTransactionStore, }; diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs b/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs index 8a8f717d73..c6f8d7f713 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs @@ -1,5 +1,5 @@ use crate::core_api::*; -use state_manager::store::traits::{ +use state_manager::store::consensus::traits::{ CommittedTransactionBundle, ConfigurableDatabase, IterableTransactionStore, QueryableProofStore, }; use std::ops::Deref; diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs b/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs index b24e70e848..41e618944a 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs @@ -9,7 +9,7 @@ use state_manager::{ use state_manager::mempool::pending_transaction_result_cache::PendingTransactionRecord; use state_manager::query::StateManagerSubstateQueries; -use state_manager::store::traits::*; +use state_manager::store::consensus::traits::*; #[tracing::instrument(skip(state))] pub(crate) async fn handle_lts_transaction_status( diff --git a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs index e487aa5b72..d1cea86e0b 100644 --- a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs +++ b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs @@ -2,7 +2,8 @@ use crate::core_api::*; use crate::engine_prelude::*; use state_manager::protocol::ProtocolVersionName; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; use std::ops::Deref; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs b/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs index 418f118713..8f18e9a760 100644 --- a/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs @@ -3,7 +3,7 @@ use crate::core_api::*; use crate::engine_prelude::*; use state_manager::query::TransactionIdentifierLoader; -use state_manager::store::traits::*; +use state_manager::store::consensus::traits::*; use state_manager::{LedgerHashes, LedgerProof, LedgerStateSummary, StateVersion}; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs b/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs index 7227fabcd9..bcf7403ada 100644 --- a/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs +++ b/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs @@ -2,7 +2,7 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::store::traits::scenario::{ +use state_manager::store::consensus::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ExecutedScenarioTransaction, ScenarioSequenceNumber, }; diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs index 45da8d8f5a..bab6f6d4da 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs @@ -1,8 +1,9 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; -use state_manager::store::traits::*; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::consensus::traits::*; use state_manager::{LedgerProof, LedgerProofOrigin, StateVersion}; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs index 5b18e355eb..08b84b8e2b 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs @@ -5,8 +5,9 @@ use std::ops::Deref; use crate::core_api::*; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; -use state_manager::store::traits::*; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::consensus::traits::*; use state_manager::transaction::*; use state_manager::{ CommittedTransactionIdentifiers, LedgerHeader, LedgerProof, LedgerProofOrigin, diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs index 4bbe016050..bbde1907ad 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs @@ -1,7 +1,7 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::rocks_db::ActualStateManagerDatabase; +use state_manager::store::consensus::rocks_db::ActualStateManagerDatabase; use state_manager::transaction::ProcessedPreviewResult; use state_manager::{ExecutionFeeData, LocalTransactionReceipt, PreviewRequest}; diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs index 8ded896fb1..8c9c8a313d 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs @@ -1,7 +1,7 @@ use crate::core_api::handlers::to_api_committed_transaction; use crate::core_api::*; -use state_manager::store::traits::*; +use state_manager::store::consensus::traits::*; use state_manager::transaction::*; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs index 75e1276df7..df2abed1e1 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs @@ -9,7 +9,7 @@ use state_manager::{ use state_manager::mempool::pending_transaction_result_cache::PendingTransactionRecord; use state_manager::query::StateManagerSubstateQueries; -use state_manager::store::traits::*; +use state_manager::store::consensus::traits::*; #[tracing::instrument(skip(state))] pub(crate) async fn handle_transaction_status( diff --git a/core-rust/core-api-server/src/core_api/helpers.rs b/core-rust/core-api-server/src/core_api/helpers.rs index e64ea4d53d..c4dadc76f9 100644 --- a/core-rust/core-api-server/src/core_api/helpers.rs +++ b/core-rust/core-api-server/src/core_api/helpers.rs @@ -1,7 +1,8 @@ use crate::engine_prelude::*; use serde::Serialize; -use state_manager::rocks_db::{ReadableRocks, StateManagerDatabase}; -use state_manager::store::traits::*; +use state_manager::common::rocks_db::ReadableRocks; +use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::consensus::traits::*; use state_manager::LedgerHeader; use std::io::Write; diff --git a/core-rust/engine-state-api-server/src/engine_state_api/extras.rs b/core-rust/engine-state-api-server/src/engine_state_api/extras.rs index 142932b351..a6bdfdacc5 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/extras.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/extras.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use itertools::Itertools; -use state_manager::store::traits::indices::{ +use state_manager::store::consensus::traits::indices::{ CreationId, EntityBlueprintId, EntityBlueprintIdV1, EntityListingIndex, }; diff --git a/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs b/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs index 6d403ae84f..73fd9c2937 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::engine_state_api::factories::EngineStateLoaderFactory; use crate::engine_state_api::*; use state_manager::historical_state::VersionScopingSupport; -use state_manager::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use state_manager::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; pub(crate) async fn handle_entity_info( state: State, diff --git a/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs b/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs index 7b1208f4d6..e031c4d43c 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs @@ -3,8 +3,8 @@ use crate::engine_state_api::*; use crate::engine_prelude::*; use state_manager::historical_state::VersionScopingSupport; -use state_manager::store::traits::indices::CreationId; -use state_manager::store::traits::ConfigurableDatabase; +use state_manager::store::consensus::traits::indices::CreationId; +use state_manager::store::consensus::traits::ConfigurableDatabase; use crate::engine_state_api::extras::{EngineEntityLister, EntitySummary}; use state_manager::StateVersion; diff --git a/core-rust/state-manager/src/commit_bundle.rs b/core-rust/state-manager/src/commit_bundle.rs index 9a873a6272..9664773e30 100644 --- a/core-rust/state-manager/src/commit_bundle.rs +++ b/core-rust/state-manager/src/commit_bundle.rs @@ -64,7 +64,7 @@ use crate::staging::epoch_handling::EpochAwareAccuTreeFactory; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::transaction::*; use crate::*; diff --git a/core-rust/state-manager/src/committer.rs b/core-rust/state-manager/src/committer.rs index 05da224c31..eefb55e6bc 100644 --- a/core-rust/state-manager/src/committer.rs +++ b/core-rust/state-manager/src/committer.rs @@ -64,7 +64,7 @@ use crate::mempool_manager::MempoolManager; use crate::staging::epoch_handling::EpochAwareAccuTreeFactory; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::transaction::*; use crate::types::CommitRequest; use crate::*; @@ -80,7 +80,7 @@ use crate::system_commits::*; use crate::accumulator_tree::storage::ReadableAccuTreeStore; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use std::ops::Deref; use std::sync::Arc; use std::time::SystemTime; diff --git a/core-rust/state-manager/src/jni/address_book.rs b/core-rust/state-manager/src/jni/address_book.rs index 3353703c13..8df2dd5761 100644 --- a/core-rust/state-manager/src/jni/address_book.rs +++ b/core-rust/state-manager/src/jni/address_book.rs @@ -64,12 +64,12 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::traits::node::AddressBookStore; +use crate::p2p::traits::node::AddressBookStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; -use crate::address_book_components::{AddressBookEntry, AddressBookNodeId}; +use crate::p2p::address_book_components::{AddressBookEntry, AddressBookNodeId}; #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( diff --git a/core-rust/state-manager/src/jni/high_priority_peers.rs b/core-rust/state-manager/src/jni/high_priority_peers.rs index 32df425cca..989636d43b 100644 --- a/core-rust/state-manager/src/jni/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/high_priority_peers.rs @@ -70,7 +70,7 @@ use node_common::java::*; use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::traits::node::HighPriorityPeersStore; +use crate::p2p::traits::node::HighPriorityPeersStore; #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers( diff --git a/core-rust/state-manager/src/jni/migration_store.rs b/core-rust/state-manager/src/jni/migration_store.rs index fc1bdf1179..2a29e6a628 100644 --- a/core-rust/state-manager/src/jni/migration_store.rs +++ b/core-rust/state-manager/src/jni/migration_store.rs @@ -69,8 +69,8 @@ use jni::sys::jbyteArray; use node_common::java::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::migration::MigrationId; -use crate::traits::node::MigrationStore; +use crate::store::p2p::migration::MigrationId; +use crate::p2p::traits::node::MigrationStore; #[no_mangle] extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_migrationDone( diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index b83c2bba90..9eb868a39d 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -87,7 +87,7 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use crate::rocks_db::{ActualNodeDatabase, ActualStateManagerDatabase}; +use crate::store::consensus::rocks_db::{ActualNodeDatabase, ActualStateManagerDatabase}; use crate::transaction::Preparator; use crate::{Committer, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; diff --git a/core-rust/state-manager/src/jni/safety_store.rs b/core-rust/state-manager/src/jni/safety_store.rs index 697c9fc1ee..d150e9fc24 100644 --- a/core-rust/state-manager/src/jni/safety_store.rs +++ b/core-rust/state-manager/src/jni/safety_store.rs @@ -64,7 +64,7 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::traits::node::SafetyStateStore; +use crate::store::p2p::traits::node::SafetyStateStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; diff --git a/core-rust/state-manager/src/jni/test_state_reader.rs b/core-rust/state-manager/src/jni/test_state_reader.rs index 2b2ccc6397..b0f987212a 100644 --- a/core-rust/state-manager/src/jni/test_state_reader.rs +++ b/core-rust/state-manager/src/jni/test_state_reader.rs @@ -73,11 +73,11 @@ use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::query::StateManagerSubstateQueries; use node_common::java::*; -use crate::store::traits::{ +use crate::store::consensus::traits::{ gc::StateTreeGcStore, IterableProofStore, QueryableProofStore, QueryableTransactionStore, SubstateNodeAncestryStore, }; -use crate::traits::measurement::MeasurableDatabase; +use crate::consensus::traits::measurement::MeasurableDatabase; use crate::transaction::LedgerTransactionHash; // diff --git a/core-rust/state-manager/src/jni/transaction_store.rs b/core-rust/state-manager/src/jni/transaction_store.rs index 1561380fbc..219b2eb9d3 100644 --- a/core-rust/state-manager/src/jni/transaction_store.rs +++ b/core-rust/state-manager/src/jni/transaction_store.rs @@ -66,7 +66,7 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::jni::LedgerSyncLimitsConfig; use crate::protocol::epoch_change_iter; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::{LedgerProof, StateVersion}; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; diff --git a/core-rust/state-manager/src/jni/vertex_store_recovery.rs b/core-rust/state-manager/src/jni/vertex_store_recovery.rs index a5b8ca0a47..b0f411519b 100644 --- a/core-rust/state-manager/src/jni/vertex_store_recovery.rs +++ b/core-rust/state-manager/src/jni/vertex_store_recovery.rs @@ -63,7 +63,7 @@ */ use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::store::traits::{RecoverableVertexStore, VertexStoreBlobV1, WriteableVertexStore}; +use crate::store::consensus::traits::{RecoverableVertexStore, VertexStoreBlobV1, WriteableVertexStore}; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; diff --git a/core-rust/state-manager/src/metrics.rs b/core-rust/state-manager/src/metrics.rs index 2267225801..445170bc7b 100644 --- a/core-rust/state-manager/src/metrics.rs +++ b/core-rust/state-manager/src/metrics.rs @@ -80,8 +80,8 @@ use crate::engine_prelude::*; use crate::protocol::{ PendingProtocolUpdateState, ProtocolState, ProtocolUpdateEnactmentCondition, }; -use crate::store::traits::measurement::CategoryDbVolumeStatistic; -use crate::traits::QueryableProofStore; +use crate::store::consensus::traits::measurement::CategoryDbVolumeStatistic; +use crate::consensus::traits::QueryableProofStore; pub struct LedgerMetrics { address_encoder: AddressBech32Encoder, // for label rendering only diff --git a/core-rust/state-manager/src/protocol/protocol_state.rs b/core-rust/state-manager/src/protocol/protocol_state.rs index 0471c3a9a3..2326abc08f 100644 --- a/core-rust/state-manager/src/protocol/protocol_state.rs +++ b/core-rust/state-manager/src/protocol/protocol_state.rs @@ -8,9 +8,9 @@ use std::sync::Arc; use tracing::info; use crate::protocol::*; -use crate::traits::{IterableProofStore, QueryableProofStore, QueryableTransactionStore}; +use crate::consensus::traits::{IterableProofStore, QueryableProofStore, QueryableTransactionStore}; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::{ LocalTransactionReceipt, ProtocolMetrics, ScenariosExecutionConfig, StateVersion, SystemExecutor, diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs index 38d63864b0..de4a89d3ca 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs index a5dbeabeae..301c966332 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs index 2ae872c30c..4aa56f0341 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs index 46211cc4f1..c9a05600a3 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs index 1dc9660b9b..b8d5d0f159 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs @@ -12,7 +12,7 @@ pub use test_definition::*; use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::transaction::*; use node_common::locks::DbLock; use std::ops::Deref; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs index aec38fa174..ac0d12e45d 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs @@ -1,5 +1,5 @@ use crate::engine_prelude::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::{protocol::*, transaction::FlashTransactionV1}; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs index 2f9795c40c..8b2d8a1d6a 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs index 23e46db125..18b84dd3e4 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::traits::*; +use crate::consensus::traits::*; use crate::LedgerProofOrigin; diff --git a/core-rust/state-manager/src/protocol/test.rs b/core-rust/state-manager/src/protocol/test.rs index cf280eb22a..ddfec4df5f 100644 --- a/core-rust/state-manager/src/protocol/test.rs +++ b/core-rust/state-manager/src/protocol/test.rs @@ -63,7 +63,7 @@ */ use crate::engine_prelude::*; -use crate::traits::QueryableProofStore; +use crate::store::consensus::traits::QueryableProofStore; use crate::protocol::*; use crate::{LedgerProof, LedgerProofOrigin, StateManagerConfig}; diff --git a/core-rust/state-manager/src/staging/cache.rs b/core-rust/state-manager/src/staging/cache.rs index 9f083682e5..294db4541d 100644 --- a/core-rust/state-manager/src/staging/cache.rs +++ b/core-rust/state-manager/src/staging/cache.rs @@ -86,9 +86,9 @@ use crate::{ use im::hashmap::HashMap as ImmutableHashMap; use itertools::Itertools; -use crate::rocks_db::ActualStateManagerDatabase; -use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; -use crate::traits::{ConfigurableDatabase, QueryableProofStore}; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::store::consensus::traits::{ConfigurableDatabase, QueryableProofStore}; use node_common::locks::{DbLock, LockFactory, Mutex}; use slotmap::SecondaryMap; diff --git a/core-rust/state-manager/src/staging/mod.rs b/core-rust/state-manager/src/staging/mod.rs index 6c182d2c78..487c5c049f 100644 --- a/core-rust/state-manager/src/staging/mod.rs +++ b/core-rust/state-manager/src/staging/mod.rs @@ -73,8 +73,8 @@ use crate::accumulator_tree::storage::ReadableAccuTreeStore; use crate::engine_prelude::*; use crate::{ReceiptTreeHash, StateVersion, TransactionTreeHash}; -use crate::store::traits::SubstateNodeAncestryStore; -use crate::traits::ConfigurableDatabase; +use crate::store::consensus::traits::SubstateNodeAncestryStore; +use crate::consensus::traits::ConfigurableDatabase; pub use cache::*; pub use result::*; diff --git a/core-rust/state-manager/src/staging/node_ancestry_resolver.rs b/core-rust/state-manager/src/staging/node_ancestry_resolver.rs index 763fd8d115..00c8ec3baa 100644 --- a/core-rust/state-manager/src/staging/node_ancestry_resolver.rs +++ b/core-rust/state-manager/src/staging/node_ancestry_resolver.rs @@ -63,7 +63,7 @@ */ use crate::engine_prelude::*; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::{SubstateChangeAction, SubstateReference}; use std::borrow::Borrow; use std::collections::hash_map::Entry; diff --git a/core-rust/state-manager/src/staging/overlays.rs b/core-rust/state-manager/src/staging/overlays.rs index ac54cbdc0e..c1fc1f2d93 100644 --- a/core-rust/state-manager/src/staging/overlays.rs +++ b/core-rust/state-manager/src/staging/overlays.rs @@ -3,7 +3,7 @@ use std::cmp::Ordering; use std::hash::Hash; use std::iter::Peekable; -use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; pub struct SubstateOverlayIterator<'a> { root_db: Peekable + 'a>>, diff --git a/core-rust/state-manager/src/staging/result.rs b/core-rust/state-manager/src/staging/result.rs index b134367d87..db15c58703 100644 --- a/core-rust/state-manager/src/staging/result.rs +++ b/core-rust/state-manager/src/staging/result.rs @@ -79,8 +79,8 @@ use crate::staging::ReadableStore; use crate::staging::node_ancestry_resolver::NodeAncestryResolver; use crate::staging::overlays::{MapSubstateNodeAncestryStore, StagedSubstateNodeAncestryStore}; -use crate::store::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; -use crate::traits::{ConfigurableDatabase, LeafSubstateKeyAssociation}; +use crate::store::consensus::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::consensus::traits::{ConfigurableDatabase, LeafSubstateKeyAssociation}; use node_common::utils::IsAccountExt; pub enum ProcessedTransactionReceipt { diff --git a/core-rust/state-manager/src/state_manager.rs b/core-rust/state-manager/src/state_manager.rs index a9dc368932..0a400034dc 100644 --- a/core-rust/state-manager/src/state_manager.rs +++ b/core-rust/state-manager/src/state_manager.rs @@ -72,11 +72,11 @@ use crate::jni::LedgerSyncLimitsConfig; use crate::protocol::{ ProtocolConfig, ProtocolManager, ProtocolUpdateExecutor, ProtocolVersionName, }; -use crate::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; +use crate::store::consensus::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; use crate::store::jmt_gc::StateTreeGcConfig; use crate::store::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; -use crate::store::traits::proofs::QueryableProofStore; -use crate::traits::DatabaseConfigValidationError; +use crate::store::consensus::traits::proofs::QueryableProofStore; +use crate::store::consensus::traits::DatabaseConfigValidationError; use crate::transaction::{ ExecutionConfigurator, LedgerTransactionValidator, Preparator, TransactionExecutorFactory, }; diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index f1cd8d2aa9..bd26a13d5a 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -67,8 +67,8 @@ use std::ops::Range; use crate::engine_prelude::*; -use crate::store::traits::indices::CreationId; -use crate::store::traits::scenario::ScenarioSequenceNumber; +use crate::store::consensus::traits::indices::CreationId; +use crate::store::consensus::traits::scenario::ScenarioSequenceNumber; use crate::store::typed_cf_api::*; use crate::transaction::RawLedgerTransaction; use crate::StateVersion; diff --git a/core-rust/state-manager/src/store/column_families.rs b/core-rust/state-manager/src/store/column_families.rs index dd1ea26152..927b38276b 100644 --- a/core-rust/state-manager/src/store/column_families.rs +++ b/core-rust/state-manager/src/store/column_families.rs @@ -1,5 +1,5 @@ -use crate::address_book_components::AddressBookNodeId; -use crate::migration::{MigrationId, MigrationStatus}; +use crate::p2p::address_book_components::AddressBookNodeId; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; use crate::store::codecs::{ BlueprintAndCreationIndexKeyDbCodec, EpochDbCodec, HashDbCodec, NodeIdDbCodec, PrefixGlobalAddressDbCodec, RawLedgerTransactionDbCodec, ScenarioSequenceNumberDbCodec, @@ -10,15 +10,15 @@ use crate::store::typed_cf_api::{ AddressBookNodeIdDbCodec, DefaultCf, DirectDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec, PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, }; -use crate::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; -use crate::traits::indices::{ +use crate::consensus::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; +use crate::consensus::traits::indices::{ CreationId, EntityBlueprintId, ObjectBlueprintName, VersionedEntityBlueprintId, VersionedObjectBlueprintName, }; -use crate::traits::scenario::{ +use crate::consensus::traits::scenario::{ ExecutedScenario, ScenarioSequenceNumber, VersionedExecutedScenario, }; -use crate::traits::{ +use crate::consensus::traits::{ ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, diff --git a/core-rust/state-manager/src/store/common/mod.rs b/core-rust/state-manager/src/store/common/mod.rs new file mode 100644 index 0000000000..45770b569f --- /dev/null +++ b/core-rust/state-manager/src/store/common/mod.rs @@ -0,0 +1 @@ +pub mod rocks_db; \ No newline at end of file diff --git a/core-rust/state-manager/src/store/common/rocks_db.rs b/core-rust/state-manager/src/store/common/rocks_db.rs new file mode 100644 index 0000000000..af94113182 --- /dev/null +++ b/core-rust/state-manager/src/store/common/rocks_db.rs @@ -0,0 +1,268 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; +use rocksdb::checkpoint::Checkpoint; +use rocksdb::{ + AsColumnFamilyRef, ColumnFamily, DBPinnableSlice, + IteratorMode, Snapshot, WriteBatch, DB, +}; + +use std::path::PathBuf; + +/// A redefined RocksDB's "key and value bytes" tuple (the original one lives in a private module). +pub type KVBytes = (Box<[u8]>, Box<[u8]>); + +/// A trait capturing the common read methods present both in a "direct" RocksDB instance and in its +/// snapshots. +/// +/// The library we use (a thin C wrapper, really) does not introduce this trivial and natural trait +/// itself, while we desperately need it to abstract the DB-reading code from the actual source of +/// data. +/// +/// A note on changed error handling: +/// The original methods typically return [`Result`]s. Our trait assumes panics instead, since we +/// treat all database access errors as fatal anyways. +pub trait ReadableRocks { + /// Resolves the column family by name. + fn cf_handle(&self, name: &str) -> &ColumnFamily; + + /// Starts iteration over key-value pairs, according to the given [`IteratorMode`]. + fn iterator_cf( + &self, + cf: &impl AsColumnFamilyRef, + mode: IteratorMode, + ) -> Box + '_>; + + /// Gets a single value by key. + fn get_pinned_cf( + &self, + cf: &impl AsColumnFamilyRef, + key: impl AsRef<[u8]>, + ) -> Option; + + /// Gets multiple values by keys. + /// + /// Syntax note: + /// The `<'a>` here is not special at all: it could technically be 100% inferred. Just the + /// compiler feature allowing to skip it from within the `` is not yet stable. + /// TODO(when the rustc feature mentioned above becomes stable): get rid of the `<'a>`. + fn multi_get_cf<'a>( + &'a self, + keys: impl IntoIterator)>, + ) -> Vec>>; +} + +/// A write-supporting extension of the [`ReadableRocks`]. +/// +/// Naturally, it is expected that only a "direct" RocksDB instance can implement this one. +pub trait WriteableRocks: ReadableRocks { + /// Atomically writes the given batch of updates. + fn write(&self, batch: WriteBatch); + + /// Returns a snapshot of the current state. + fn snapshot(&self) -> SnapshotRocks; +} + +/// A [`ReadableRocks`] instance opened as secondary instance. +pub trait SecondaryRocks: ReadableRocks { + /// Tries to catch up with the primary by reading as much as possible from the + /// log files. + fn try_catchup_with_primary(&self); +} + +/// RocksDB checkpoint support. +pub trait CheckpointableRocks { + fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error>; +} + +/// Direct RocksDB instance. +pub struct DirectRocks { + pub db: DB, +} + +impl ReadableRocks for DirectRocks { + fn cf_handle(&self, name: &str) -> &ColumnFamily { + self.db.cf_handle(name).expect(name) + } + + fn iterator_cf( + &self, + cf: &impl AsColumnFamilyRef, + mode: IteratorMode, + ) -> Box + '_> { + Box::new( + self.db + .iterator_cf(cf, mode) + .map(|result| result.expect("reading from DB iterator")), + ) + } + + fn get_pinned_cf( + &self, + cf: &impl AsColumnFamilyRef, + key: impl AsRef<[u8]>, + ) -> Option { + self.db.get_pinned_cf(cf, key).expect("DB get by key") + } + + fn multi_get_cf<'a>( + &'a self, + keys: impl IntoIterator)>, + ) -> Vec>> { + self.db + .multi_get_cf(keys) + .into_iter() + .map(|result| result.expect("batch DB get by key")) + .collect() + } +} + +impl WriteableRocks for DirectRocks { + fn write(&self, batch: WriteBatch) { + self.db.write(batch).expect("DB write batch"); + } + + fn snapshot(&self) -> SnapshotRocks { + SnapshotRocks { + db: &self.db, + snapshot: self.db.snapshot(), + } + } +} + +impl SecondaryRocks for DirectRocks { + fn try_catchup_with_primary(&self) { + self.db + .try_catch_up_with_primary() + .expect("secondary DB catchup"); + } +} + +impl CheckpointableRocks for DirectRocks { + fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { + create_checkpoint(&self.db, checkpoint_path) + } +} + +impl<'db> CheckpointableRocks for SnapshotRocks<'db> { + fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { + create_checkpoint(self.db, checkpoint_path) + } +} + +fn create_checkpoint(db: &DB, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { + let checkpoint = Checkpoint::new(db)?; + checkpoint.create_checkpoint(checkpoint_path)?; + Ok(()) +} + +/// Snapshot of RocksDB. +/// +/// Implementation note: +/// The original [`DB`] reference is interestingly kept internally by the [`Snapshot`] as well. +/// However, we need direct access to it for the [`Self::cf_handle()`] reasons. +pub struct SnapshotRocks<'db> { + db: &'db DB, + snapshot: Snapshot<'db>, +} + +impl<'db> ReadableRocks for SnapshotRocks<'db> { + fn cf_handle(&self, name: &str) -> &ColumnFamily { + self.db.cf_handle(name).expect(name) + } + + fn iterator_cf( + &self, + cf: &impl AsColumnFamilyRef, + mode: IteratorMode, + ) -> Box + '_> { + Box::new( + self.snapshot + .iterator_cf(cf, mode) + .map(|result| result.expect("reading from snapshot DB iterator")), + ) + } + + fn get_pinned_cf( + &self, + cf: &impl AsColumnFamilyRef, + key: impl AsRef<[u8]>, + ) -> Option { + self.snapshot + .get_pinned_cf(cf, key) + .expect("snapshot DB get by key") + } + + fn multi_get_cf<'a>( + &'a self, + keys: impl IntoIterator)>, + ) -> Vec>> { + self.snapshot + .multi_get_cf(keys) + .into_iter() + .map(|result| result.expect("batch snapshot DB get by key")) + .collect() + } +} diff --git a/core-rust/state-manager/src/store/consensus/mod.rs b/core-rust/state-manager/src/store/consensus/mod.rs new file mode 100644 index 0000000000..908eeb09c6 --- /dev/null +++ b/core-rust/state-manager/src/store/consensus/mod.rs @@ -0,0 +1,2 @@ +pub mod traits; +pub mod rocks_db; diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/consensus/rocks_db.rs similarity index 92% rename from core-rust/state-manager/src/store/rocks_db.rs rename to core-rust/state-manager/src/store/consensus/rocks_db.rs index 51b5ba6009..668278dbac 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/consensus/rocks_db.rs @@ -64,39 +64,38 @@ use std::collections::HashSet; use crate::engine_prelude::*; -use crate::store::traits::*; +use crate::store::common::rocks_db::*; +use crate::store::consensus::traits::*; use crate::{ BySubstate, CommittedTransactionIdentifiers, LedgerProof, LedgerProofOrigin, LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, ReceiptTreeHash, StateVersion, SubstateChangeAction, TransactionTreeHash, }; use node_common::utils::IsAccountExt; -use rocksdb::checkpoint::Checkpoint; use rocksdb::{ - AsColumnFamilyRef, ColumnFamily, ColumnFamilyDescriptor, DBPinnableSlice, Direction, - IteratorMode, Options, Snapshot, WriteBatch, DB, + ColumnFamilyDescriptor, Direction, IteratorMode, Options, DB, }; use std::path::PathBuf; use node_common::locks::Snapshottable; use tracing::{error, info, warn}; -use super::traits::extensions::*; +use crate::store::consensus::traits::extensions::*; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; -use crate::address_book_components::AddressBookNodeId; +use crate::p2p::address_book_components::AddressBookNodeId; use crate::column_families::*; -use crate::migration::{MigrationId, MigrationStatus}; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; use crate::query::TransactionIdentifierLoader; use crate::store::historical_state::StateTreeBasedSubstateDatabase; -use crate::store::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; -use crate::store::traits::indices::{ +use crate::store::consensus::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; +use crate::store::consensus::traits::indices::{ CreationId, EntityBlueprintId, EntityListingIndex, ObjectBlueprintNameV1, }; -use crate::store::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDatabase}; -use crate::store::traits::scenario::{ +use crate::store::consensus::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDatabase}; +use crate::store::consensus::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; use crate::store::typed_cf_api::*; -use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; +use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; @@ -154,202 +153,6 @@ const ALL_NODE_COLUMN_FAMILIES: [&str; 4] = [ HighPriorityPeersCf::DEFAULT_NAME, ]; -/// A redefined RocksDB's "key and value bytes" tuple (the original one lives in a private module). -pub type KVBytes = (Box<[u8]>, Box<[u8]>); - -/// A trait capturing the common read methods present both in a "direct" RocksDB instance and in its -/// snapshots. -/// -/// The library we use (a thin C wrapper, really) does not introduce this trivial and natural trait -/// itself, while we desperately need it to abstract the DB-reading code from the actual source of -/// data. -/// -/// A note on changed error handling: -/// The original methods typically return [`Result`]s. Our trait assumes panics instead, since we -/// treat all database access errors as fatal anyways. -pub trait ReadableRocks { - /// Resolves the column family by name. - fn cf_handle(&self, name: &str) -> &ColumnFamily; - - /// Starts iteration over key-value pairs, according to the given [`IteratorMode`]. - fn iterator_cf( - &self, - cf: &impl AsColumnFamilyRef, - mode: IteratorMode, - ) -> Box + '_>; - - /// Gets a single value by key. - fn get_pinned_cf( - &self, - cf: &impl AsColumnFamilyRef, - key: impl AsRef<[u8]>, - ) -> Option; - - /// Gets multiple values by keys. - /// - /// Syntax note: - /// The `<'a>` here is not special at all: it could technically be 100% inferred. Just the - /// compiler feature allowing to skip it from within the `` is not yet stable. - /// TODO(when the rustc feature mentioned above becomes stable): get rid of the `<'a>`. - fn multi_get_cf<'a>( - &'a self, - keys: impl IntoIterator)>, - ) -> Vec>>; -} - -/// A write-supporting extension of the [`ReadableRocks`]. -/// -/// Naturally, it is expected that only a "direct" RocksDB instance can implement this one. -pub trait WriteableRocks: ReadableRocks { - /// Atomically writes the given batch of updates. - fn write(&self, batch: WriteBatch); - - /// Returns a snapshot of the current state. - fn snapshot(&self) -> SnapshotRocks; -} - -/// A [`ReadableRocks`] instance opened as secondary instance. -pub trait SecondaryRocks: ReadableRocks { - /// Tries to catch up with the primary by reading as much as possible from the - /// log files. - fn try_catchup_with_primary(&self); -} - -/// RocksDB checkpoint support. -pub trait CheckpointableRocks { - fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error>; -} - -/// Direct RocksDB instance. -pub struct DirectRocks { - db: DB, -} - -impl ReadableRocks for DirectRocks { - fn cf_handle(&self, name: &str) -> &ColumnFamily { - self.db.cf_handle(name).expect(name) - } - - fn iterator_cf( - &self, - cf: &impl AsColumnFamilyRef, - mode: IteratorMode, - ) -> Box + '_> { - Box::new( - self.db - .iterator_cf(cf, mode) - .map(|result| result.expect("reading from DB iterator")), - ) - } - - fn get_pinned_cf( - &self, - cf: &impl AsColumnFamilyRef, - key: impl AsRef<[u8]>, - ) -> Option { - self.db.get_pinned_cf(cf, key).expect("DB get by key") - } - - fn multi_get_cf<'a>( - &'a self, - keys: impl IntoIterator)>, - ) -> Vec>> { - self.db - .multi_get_cf(keys) - .into_iter() - .map(|result| result.expect("batch DB get by key")) - .collect() - } -} - -impl WriteableRocks for DirectRocks { - fn write(&self, batch: WriteBatch) { - self.db.write(batch).expect("DB write batch"); - } - - fn snapshot(&self) -> SnapshotRocks { - SnapshotRocks { - db: &self.db, - snapshot: self.db.snapshot(), - } - } -} - -impl SecondaryRocks for DirectRocks { - fn try_catchup_with_primary(&self) { - self.db - .try_catch_up_with_primary() - .expect("secondary DB catchup"); - } -} - -impl CheckpointableRocks for DirectRocks { - fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { - create_checkpoint(&self.db, checkpoint_path) - } -} - -impl<'db> CheckpointableRocks for SnapshotRocks<'db> { - fn create_checkpoint(&self, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { - create_checkpoint(self.db, checkpoint_path) - } -} - -fn create_checkpoint(db: &DB, checkpoint_path: PathBuf) -> Result<(), rocksdb::Error> { - let checkpoint = Checkpoint::new(db)?; - checkpoint.create_checkpoint(checkpoint_path)?; - Ok(()) -} - -/// Snapshot of RocksDB. -/// -/// Implementation note: -/// The original [`DB`] reference is interestingly kept internally by the [`Snapshot`] as well. -/// However, we need direct access to it for the [`Self::cf_handle()`] reasons. -pub struct SnapshotRocks<'db> { - db: &'db DB, - snapshot: Snapshot<'db>, -} - -impl<'db> ReadableRocks for SnapshotRocks<'db> { - fn cf_handle(&self, name: &str) -> &ColumnFamily { - self.db.cf_handle(name).expect(name) - } - - fn iterator_cf( - &self, - cf: &impl AsColumnFamilyRef, - mode: IteratorMode, - ) -> Box + '_> { - Box::new( - self.snapshot - .iterator_cf(cf, mode) - .map(|result| result.expect("reading from snapshot DB iterator")), - ) - } - - fn get_pinned_cf( - &self, - cf: &impl AsColumnFamilyRef, - key: impl AsRef<[u8]>, - ) -> Option { - self.snapshot - .get_pinned_cf(cf, key) - .expect("snapshot DB get by key") - } - - fn multi_get_cf<'a>( - &'a self, - keys: impl IntoIterator)>, - ) -> Vec>> { - self.snapshot - .multi_get_cf(keys) - .into_iter() - .map(|result| result.expect("batch snapshot DB get by key")) - .collect() - } -} - pub type ActualStateManagerDatabase = StateManagerDatabase; pub type ActualNodeDatabase = NodeDatabase; diff --git a/core-rust/state-manager/src/store/traits.rs b/core-rust/state-manager/src/store/consensus/traits.rs similarity index 97% rename from core-rust/state-manager/src/store/traits.rs rename to core-rust/state-manager/src/store/consensus/traits.rs index c96e3e4df6..44ceb64c29 100644 --- a/core-rust/state-manager/src/store/traits.rs +++ b/core-rust/state-manager/src/store/consensus/traits.rs @@ -169,36 +169,6 @@ pub struct LeafSubstateKeyAssociation { pub substate_value: Vec, } -pub mod node { - use crate::address_book_components::AddressBookNodeId; - use crate::migration::MigrationId; - - use super::*; - - pub trait AddressBookStore { - fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; - fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; - fn reset(&self); - fn get_all(&self, ) -> Vec>; - } - - pub trait HighPriorityPeersStore { - fn upsert_all_peers(&self, peers: &[u8]); - fn get_all_peers(&self) -> Option>; - fn reset_high_priority_peers(&self); - } - - pub trait SafetyStateStore { - fn upsert_safety_state(&self, safety_state: &[u8]); - fn get_safety_state(&self) -> Option>; - } - - pub trait MigrationStore { - fn is_migration_done(&self, store_id: MigrationId) -> bool; - fn migration_done(&self, store_id: MigrationId); - } -} - pub mod vertex { use super::*; @@ -294,7 +264,7 @@ pub mod substate { pub mod transactions { use super::*; - use crate::store::traits::CommittedTransactionBundle; + use crate::store::consensus::traits::CommittedTransactionBundle; use crate::{ CommittedTransactionIdentifiers, LedgerHashes, LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, diff --git a/core-rust/state-manager/src/store/historical_state.rs b/core-rust/state-manager/src/store/historical_state.rs index 977e162bcf..bf42966790 100644 --- a/core-rust/state-manager/src/store/historical_state.rs +++ b/core-rust/state-manager/src/store/historical_state.rs @@ -67,9 +67,10 @@ use std::ops::Deref; use crate::engine_prelude::entity_tier::EntityTier; use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; -use crate::rocks_db::{ReadableRocks, StateManagerDatabase}; -use crate::store::traits::*; -use crate::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; +use crate::store::common::rocks_db::ReadableRocks; +use crate::store::consensus::rocks_db::StateManagerDatabase; +use crate::store::consensus::traits::*; +use crate::consensus::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; use crate::{CommittedTransactionIdentifiers, LedgerStateSummary, StateVersion}; /// An implementation of a [`SubstateDatabase`] viewed at a specific [`StateVersion`]. diff --git a/core-rust/state-manager/src/store/jmt_gc.rs b/core-rust/state-manager/src/store/jmt_gc.rs index 8c618f6023..d93bcb2c5d 100644 --- a/core-rust/state-manager/src/store/jmt_gc.rs +++ b/core-rust/state-manager/src/store/jmt_gc.rs @@ -70,10 +70,10 @@ use std::sync::Arc; use std::time::Duration; use tracing::info; -use crate::rocks_db::ActualStateManagerDatabase; -use crate::store::traits::gc::StateTreeGcStore; -use crate::store::traits::proofs::QueryableProofStore; -use crate::store::traits::StaleTreePartsV1; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::traits::gc::StateTreeGcStore; +use crate::store::consensus::traits::proofs::QueryableProofStore; +use crate::store::consensus::traits::StaleTreePartsV1; use crate::{StateVersion, StateVersionDelta}; /// A maximum number of JMT nodes collected into "batch delete" buffer. diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index 194ae6f773..d1a1ef587d 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -62,25 +62,27 @@ * permissions under this License. */ -pub mod address_book_components; +use std::sync::Arc; + +use prometheus::Registry; +use sbor::{Categorize, Decode, Encode}; + +use node_common::locks::DbLock; +use consensus::rocks_db::ActualStateManagerDatabase; + +use crate::RawDbMetrics; +pub use crate::store::consensus::traits::DatabaseConfig; +use crate::store::consensus::traits::measurement::MeasurableDatabase; + pub mod column_families; pub mod historical_state; pub mod jmt_gc; -pub mod migration; pub mod proofs_gc; -pub mod rocks_db; -pub mod traits; mod codecs; mod typed_cf_api; - -use crate::store::traits::measurement::MeasurableDatabase; -use crate::RawDbMetrics; -use node_common::locks::DbLock; -use prometheus::Registry; -use rocks_db::ActualStateManagerDatabase; -use sbor::{Categorize, Decode, Encode}; -use std::sync::Arc; -pub use traits::DatabaseConfig; +pub mod p2p; +pub mod consensus; +pub mod common; #[derive(Debug, Categorize, Encode, Decode, Clone)] pub struct DatabaseBackendConfig { diff --git a/core-rust/state-manager/src/store/address_book_components.rs b/core-rust/state-manager/src/store/p2p/address_book_components.rs similarity index 100% rename from core-rust/state-manager/src/store/address_book_components.rs rename to core-rust/state-manager/src/store/p2p/address_book_components.rs diff --git a/core-rust/state-manager/src/store/migration.rs b/core-rust/state-manager/src/store/p2p/migration.rs similarity index 100% rename from core-rust/state-manager/src/store/migration.rs rename to core-rust/state-manager/src/store/p2p/migration.rs diff --git a/core-rust/state-manager/src/store/p2p/mod.rs b/core-rust/state-manager/src/store/p2p/mod.rs new file mode 100644 index 0000000000..4394b28dc8 --- /dev/null +++ b/core-rust/state-manager/src/store/p2p/mod.rs @@ -0,0 +1,3 @@ +pub mod address_book_components; +pub mod traits; +pub mod migration; \ No newline at end of file diff --git a/core-rust/state-manager/src/store/p2p/traits.rs b/core-rust/state-manager/src/store/p2p/traits.rs new file mode 100644 index 0000000000..aba42cc1af --- /dev/null +++ b/core-rust/state-manager/src/store/p2p/traits.rs @@ -0,0 +1,95 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::engine_prelude::*; + +pub mod node { + use crate::p2p::address_book_components::AddressBookNodeId; + use crate::p2p::migration::MigrationId; + + use super::*; + + pub trait AddressBookStore { + fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; + fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; + fn reset(&self); + fn get_all(&self, ) -> Vec>; + } + + pub trait HighPriorityPeersStore { + fn upsert_all_peers(&self, peers: &[u8]); + fn get_all_peers(&self) -> Option>; + fn reset_high_priority_peers(&self); + } + + pub trait SafetyStateStore { + fn upsert_safety_state(&self, safety_state: &[u8]); + fn get_safety_state(&self) -> Option>; + } + + pub trait MigrationStore { + fn is_migration_done(&self, store_id: MigrationId) -> bool; + fn migration_done(&self, store_id: MigrationId); + } +} diff --git a/core-rust/state-manager/src/store/proofs_gc.rs b/core-rust/state-manager/src/store/proofs_gc.rs index 7e2d74e160..21f5b71a4e 100644 --- a/core-rust/state-manager/src/store/proofs_gc.rs +++ b/core-rust/state-manager/src/store/proofs_gc.rs @@ -69,15 +69,15 @@ use std::sync::Arc; use std::time::Duration; use tracing::{error, info}; -use crate::store::traits::gc::{ +use crate::store::consensus::traits::gc::{ LedgerProofsGcProgress, LedgerProofsGcProgressV1, LedgerProofsGcStore, }; -use crate::store::traits::proofs::QueryableProofStore; +use crate::store::consensus::traits::proofs::QueryableProofStore; use crate::jni::LedgerSyncLimitsConfig; -use crate::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; -use crate::store::rocks_db::ReadableRocks; -use crate::traits::GetSyncableTxnsAndProofError::{ +use crate::store::consensus::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; +use crate::store::common::rocks_db::ReadableRocks; +use crate::consensus::traits::GetSyncableTxnsAndProofError::{ FailedToPrepareAResponseWithinLimits, NothingToServeAtTheGivenStateVersion, RefusedToServeGenesis, RefusedToServeProtocolUpdate, }; @@ -312,9 +312,9 @@ mod tests { use crate::jni::LedgerSyncLimitsConfig; use crate::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; use crate::protocol::*; - use crate::store::traits::proofs::QueryableProofStore; + use crate::store::consensus::traits::proofs::QueryableProofStore; use crate::test::{commit_round_updates_until_epoch, create_state_manager}; - use crate::traits::GetSyncableTxnsAndProofError; + use crate::consensus::traits::GetSyncableTxnsAndProofError; use crate::{StateManagerConfig, StateVersion}; use std::time::Duration; diff --git a/core-rust/state-manager/src/store/typed_cf_api.rs b/core-rust/state-manager/src/store/typed_cf_api.rs index 18de58727c..1e0ae59bd4 100644 --- a/core-rust/state-manager/src/store/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/typed_cf_api.rs @@ -62,10 +62,10 @@ * permissions under this License. */ -use crate::address_book_components::AddressBookNodeId; +use crate::p2p::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; -use crate::migration::{MigrationId, MigrationStatus}; -use crate::store::rocks_db::{ReadableRocks, WriteableRocks}; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::store::common::rocks_db::{ReadableRocks, WriteableRocks}; use itertools::Itertools; use rocksdb::{ColumnFamily, Direction, IteratorMode, WriteBatch}; use std::ops::Range; diff --git a/core-rust/state-manager/src/system_executor.rs b/core-rust/state-manager/src/system_executor.rs index e8dc192ad6..da26a7307e 100644 --- a/core-rust/state-manager/src/system_executor.rs +++ b/core-rust/state-manager/src/system_executor.rs @@ -64,7 +64,7 @@ use crate::query::*; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::transaction::*; use crate::*; @@ -75,14 +75,14 @@ use node_common::locks::DbLock; use tracing::info; -use crate::store::traits::scenario::{ +use crate::store::consensus::traits::scenario::{ DescribedAddressRendering, ExecutedScenario, ExecutedScenarioStore, ExecutedScenarioTransaction, }; use crate::system_commits::*; use crate::protocol::{ProtocolUpdateNodeBatch, ProtocolVersionName}; -use crate::rocks_db::ActualStateManagerDatabase; -use crate::traits::scenario::ExecutedScenarioV1; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::consensus::traits::scenario::ExecutedScenarioV1; use radix_transaction_scenarios::scenarios::ALL_SCENARIOS; use std::sync::Arc; use std::time::Instant; diff --git a/core-rust/state-manager/src/test/mod.rs b/core-rust/state-manager/src/test/mod.rs index 0d809a9f3e..f8b27cbcb3 100644 --- a/core-rust/state-manager/src/test/mod.rs +++ b/core-rust/state-manager/src/test/mod.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; -use crate::traits::QueryableProofStore; +use crate::store::consensus::traits::QueryableProofStore; use crate::{ CommitRequest, CommitSummary, LedgerHeader, LedgerProof, LedgerProofOrigin, PrepareRequest, PrepareResult, RoundHistory, StateManager, StateManagerConfig, diff --git a/core-rust/state-manager/src/transaction/preparation.rs b/core-rust/state-manager/src/transaction/preparation.rs index edba315b88..f825d4420e 100644 --- a/core-rust/state-manager/src/transaction/preparation.rs +++ b/core-rust/state-manager/src/transaction/preparation.rs @@ -72,7 +72,7 @@ use tracing::{debug, info}; use crate::engine_prelude::*; use crate::limits::*; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::*; use crate::system_commits::*; diff --git a/core-rust/state-manager/src/transaction/preview.rs b/core-rust/state-manager/src/transaction/preview.rs index 0c94867038..c9a9bbc7b1 100644 --- a/core-rust/state-manager/src/transaction/preview.rs +++ b/core-rust/state-manager/src/transaction/preview.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use crate::historical_state::{StateHistoryError, VersionScopingSupport}; -use crate::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::transaction::*; use crate::{ GlobalBalanceSummary, LedgerStateChanges, LedgerStateSummary, PreviewRequest, diff --git a/core-rust/state-manager/src/transaction/series_execution.rs b/core-rust/state-manager/src/transaction/series_execution.rs index 6e1dc88505..2b38ee0d35 100644 --- a/core-rust/state-manager/src/transaction/series_execution.rs +++ b/core-rust/state-manager/src/transaction/series_execution.rs @@ -71,7 +71,7 @@ use crate::commit_bundle::CommitBundleBuilder; use crate::protocol::*; use crate::query::*; use crate::staging::ReadableStore; -use crate::store::traits::*; +use crate::store::consensus::traits::*; use crate::transaction::*; use crate::*; diff --git a/core-rust/state-manager/src/transaction/validation.rs b/core-rust/state-manager/src/transaction/validation.rs index 06e97b3514..0ee5729eab 100644 --- a/core-rust/state-manager/src/transaction/validation.rs +++ b/core-rust/state-manager/src/transaction/validation.rs @@ -7,9 +7,9 @@ use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; -use crate::rocks_db::ActualStateManagerDatabase; -use crate::store::traits::transactions::QueryableTransactionStore; -use crate::store::traits::{QueryableProofStore, TransactionIndex}; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::consensus::traits::transactions::QueryableTransactionStore; +use crate::store::consensus::traits::{QueryableProofStore, TransactionIndex}; use crate::transaction::{ExecutionConfigurator, TransactionLogic}; use crate::{ AlreadyCommittedError, AtSpecificState, AtState, ExecutionRejectionReason, From 7171b4d9a0a141b9b70b9acc144a9455fb824850 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 7 Aug 2024 16:38:45 +0200 Subject: [PATCH 12/40] Separation of DB-related code at Rust side (step #2) --- .../src/jni/node_rust_environment.rs | 3 +- .../store/{ => consensus}/column_families.rs | 97 +++----- .../state-manager/src/store/consensus/mod.rs | 1 + .../src/store/consensus/rocks_db.rs | 144 ++---------- core-rust/state-manager/src/store/mod.rs | 1 - .../src/store/p2p/column_families.rs | 49 +++++ core-rust/state-manager/src/store/p2p/mod.rs | 4 +- .../state-manager/src/store/p2p/rocks_db.rs | 208 ++++++++++++++++++ 8 files changed, 306 insertions(+), 201 deletions(-) rename core-rust/state-manager/src/store/{ => consensus}/column_families.rs (93%) create mode 100644 core-rust/state-manager/src/store/p2p/column_families.rs create mode 100644 core-rust/state-manager/src/store/p2p/rocks_db.rs diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index 9eb868a39d..27eebbe2da 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -87,7 +87,8 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use crate::store::consensus::rocks_db::{ActualNodeDatabase, ActualStateManagerDatabase}; +use crate::store::p2p::rocks_db::ActualNodeDatabase; +use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::transaction::Preparator; use crate::{Committer, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; diff --git a/core-rust/state-manager/src/store/column_families.rs b/core-rust/state-manager/src/store/consensus/column_families.rs similarity index 93% rename from core-rust/state-manager/src/store/column_families.rs rename to core-rust/state-manager/src/store/consensus/column_families.rs index 927b38276b..45cc92043b 100644 --- a/core-rust/state-manager/src/store/column_families.rs +++ b/core-rust/state-manager/src/store/consensus/column_families.rs @@ -1,14 +1,22 @@ -use crate::p2p::address_book_components::AddressBookNodeId; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; -use crate::store::codecs::{ - BlueprintAndCreationIndexKeyDbCodec, EpochDbCodec, HashDbCodec, NodeIdDbCodec, - PrefixGlobalAddressDbCodec, RawLedgerTransactionDbCodec, ScenarioSequenceNumberDbCodec, - StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, - TypeAndCreationIndexKeyDbCodec, +use std::fmt; + +use radix_common::crypto::Hash; +use radix_common::prelude::{EntityType, Epoch, GlobalAddress, NodeId, PackageAddress}; +use radix_substate_store_impls::state_tree::tree_store::{ + StoredTreeNodeKey, TreeNode, VersionedTreeNode, }; -use crate::store::typed_cf_api::{ - AddressBookNodeIdDbCodec, DefaultCf, DirectDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec, - PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, +use radix_substate_store_interface::interface::{DbSubstateKey, DbSubstateValue}; +use radix_transactions::model::{IntentHash, NotarizedTransactionHash}; + +use crate::{ + CommittedTransactionIdentifiers, LedgerProof, LedgerTransactionReceipt, + LocalTransactionExecution, StateVersion, VersionedCommittedTransactionIdentifiers, + VersionedLedgerProof, VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, +}; +use crate::consensus::traits::{ + ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, + VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, + VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, }; use crate::consensus::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; use crate::consensus::traits::indices::{ @@ -18,25 +26,17 @@ use crate::consensus::traits::indices::{ use crate::consensus::traits::scenario::{ ExecutedScenario, ScenarioSequenceNumber, VersionedExecutedScenario, }; -use crate::consensus::traits::{ - ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, - VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, - VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, -}; -use crate::transaction::{LedgerTransactionHash, RawLedgerTransaction}; -use crate::{ - CommittedTransactionIdentifiers, LedgerProof, LedgerTransactionReceipt, - LocalTransactionExecution, StateVersion, VersionedCommittedTransactionIdentifiers, - VersionedLedgerProof, VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, +use crate::store::codecs::{ + BlueprintAndCreationIndexKeyDbCodec, EpochDbCodec, HashDbCodec, NodeIdDbCodec, + PrefixGlobalAddressDbCodec, RawLedgerTransactionDbCodec, ScenarioSequenceNumberDbCodec, + StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, + TypeAndCreationIndexKeyDbCodec, }; -use radix_common::crypto::Hash; -use radix_common::prelude::{EntityType, Epoch, GlobalAddress, NodeId, PackageAddress}; -use radix_substate_store_impls::state_tree::tree_store::{ - StoredTreeNodeKey, TreeNode, VersionedTreeNode, +use crate::store::typed_cf_api::{ + DefaultCf, DirectDbCodec, + PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, }; -use radix_substate_store_interface::interface::{DbSubstateKey, DbSubstateValue}; -use radix_transactions::model::{IntentHash, NotarizedTransactionHash}; -use std::fmt; +use crate::transaction::{LedgerTransactionHash, RawLedgerTransaction}; /// Committed transactions. /// Schema: `StateVersion.to_bytes()` -> `RawLedgerTransaction.as_ref::<[u8]>()` @@ -426,46 +426,3 @@ impl DefaultCf for AssociatedStateTreeValuesCf { type KeyCodec = StoredTreeNodeKeyDbCodec; type ValueCodec = DirectDbCodec; } - -/// Address book and safety state store migration status. Filled once during the migration. -pub struct MigrationStatusCf; -impl DefaultCf for MigrationStatusCf { - type Key = MigrationId; - type Value = MigrationStatus; - - const DEFAULT_NAME: &'static str = "migration_status"; - type KeyCodec = MigrationIdDbCodec; - type ValueCodec = MigrationStatusDbCodec; -} - -/// Address book -pub struct AddressBookCf; -impl DefaultCf for AddressBookCf { - type Key = AddressBookNodeId; - type Value = Vec; - - const DEFAULT_NAME: &'static str = "address_book"; - type KeyCodec = AddressBookNodeIdDbCodec; - type ValueCodec = DirectDbCodec; -} - -/// Safety store -pub struct SafetyStoreCf; -impl DefaultCf for SafetyStoreCf { - type Key = (); - type Value = Vec; - - const DEFAULT_NAME: &'static str = "safety_store"; - type KeyCodec = UnitDbCodec; - type ValueCodec = DirectDbCodec; -} - -pub struct HighPriorityPeersCf; -impl DefaultCf for HighPriorityPeersCf { - type Key = (); - type Value = Vec; - - const DEFAULT_NAME: &'static str = "high_priority_peers"; - type KeyCodec = UnitDbCodec; - type ValueCodec = DirectDbCodec; -} diff --git a/core-rust/state-manager/src/store/consensus/mod.rs b/core-rust/state-manager/src/store/consensus/mod.rs index 908eeb09c6..0c060420bd 100644 --- a/core-rust/state-manager/src/store/consensus/mod.rs +++ b/core-rust/state-manager/src/store/consensus/mod.rs @@ -1,2 +1,3 @@ pub mod traits; pub mod rocks_db; +pub mod column_families; diff --git a/core-rust/state-manager/src/store/consensus/rocks_db.rs b/core-rust/state-manager/src/store/consensus/rocks_db.rs index 668278dbac..372a744999 100644 --- a/core-rust/state-manager/src/store/consensus/rocks_db.rs +++ b/core-rust/state-manager/src/store/consensus/rocks_db.rs @@ -63,29 +63,28 @@ */ use std::collections::HashSet; -use crate::engine_prelude::*; -use crate::store::common::rocks_db::*; -use crate::store::consensus::traits::*; +use std::path::PathBuf; + +use rocksdb::{ + ColumnFamilyDescriptor, DB, Direction, IteratorMode, Options, +}; +use tracing::{error, info, warn}; + +use node_common::locks::Snapshottable; +use node_common::utils::IsAccountExt; + use crate::{ BySubstate, CommittedTransactionIdentifiers, LedgerProof, LedgerProofOrigin, LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, ReceiptTreeHash, StateVersion, SubstateChangeAction, TransactionTreeHash, }; -use node_common::utils::IsAccountExt; -use rocksdb::{ - ColumnFamilyDescriptor, Direction, IteratorMode, Options, DB, -}; - -use std::path::PathBuf; -use node_common::locks::Snapshottable; -use tracing::{error, info, warn}; -use crate::store::consensus::traits::extensions::*; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; -use crate::p2p::address_book_components::AddressBookNodeId; -use crate::column_families::*; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::consensus::column_families::*; +use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; -use crate::store::historical_state::StateTreeBasedSubstateDatabase; +use crate::store::common::rocks_db::*; +use crate::store::consensus::traits::*; +use crate::store::consensus::traits::extensions::*; use crate::store::consensus::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; use crate::store::consensus::traits::indices::{ CreationId, EntityBlueprintId, EntityListingIndex, ObjectBlueprintNameV1, @@ -94,8 +93,8 @@ use crate::store::consensus::traits::measurement::{CategoryDbVolumeStatistic, Me use crate::store::consensus::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; +use crate::store::historical_state::StateTreeBasedSubstateDatabase; use crate::store::typed_cf_api::*; -use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; @@ -146,15 +145,7 @@ const ALL_STATE_MANAGER_COLUMN_FAMILIES: [&str; 25] = [ BlueprintAndCreationIndexedObjectsCf::VERSIONED_NAME, ]; -const ALL_NODE_COLUMN_FAMILIES: [&str; 4] = [ - MigrationStatusCf::DEFAULT_NAME, - AddressBookCf::DEFAULT_NAME, - SafetyStoreCf::DEFAULT_NAME, - HighPriorityPeersCf::DEFAULT_NAME, -]; - pub type ActualStateManagerDatabase = StateManagerDatabase; -pub type ActualNodeDatabase = NodeDatabase; impl<'db> Snapshottable<'db> for StateManagerDatabase { type Snapshot = StateManagerDatabase>; @@ -175,109 +166,6 @@ impl<'db> Snapshottable<'db> for StateManagerDatabase { } } -/// A RocksDB-backed persistence layer for node-specific address book and safety store. -pub struct NodeDatabase { - /// Underlying RocksDB instance. - rocks: R, -} - -impl ActualNodeDatabase { - pub fn new( - root_path: PathBuf, - ) -> ActualNodeDatabase { - let mut db_opts = Options::default(); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); - - let column_families: Vec = ALL_NODE_COLUMN_FAMILIES - .iter() - .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) - .collect(); - - let db = DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap(); - - NodeDatabase { - rocks: DirectRocks { db }, - } - } -} - -impl NodeDatabase { - fn open_rw_context(&self) -> TypedDbContext> { - TypedDbContext::new(&self.rocks, BufferedWriteSupport::new(&self.rocks)) - } -} - -impl AddressBookStore for NodeDatabase { - fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { - let binding = self.open_rw_context(); - let context = binding.cf(AddressBookCf); - - if context.get(node_id).is_some() { - context.delete(node_id); - } - - true - } - - fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { - let binding = self.open_rw_context(); - let context = binding.cf(AddressBookCf); - - context.put(node_id, &entry.to_vec()); - - true - } - - fn reset(&self) { - self.open_rw_context().cf(AddressBookCf).delete_all(); - } - - fn get_all(&self) -> Vec> { - self.open_rw_context() - .cf(AddressBookCf) - .get_all() - } -} - -impl HighPriorityPeersStore for NodeDatabase { - fn upsert_all_peers(&self, peers: &[u8]) { - self.open_rw_context().cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); - } - - fn get_all_peers(&self) -> Option> { - self.open_rw_context().cf(HighPriorityPeersCf) - .get(&()) - } - - fn reset_high_priority_peers(&self) { - self.open_rw_context().cf(HighPriorityPeersCf).delete(&()); - } -} - -impl SafetyStateStore for NodeDatabase { - fn upsert_safety_state(&self, safety_state: &[u8]) { - self.open_rw_context().cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); - } - - fn get_safety_state(&self) -> Option> { - self.open_rw_context().cf(SafetyStoreCf) - .get(&()) - } -} - -impl MigrationStore for NodeDatabase { - fn is_migration_done(&self, store_id: MigrationId) -> bool { - self.open_rw_context().cf(MigrationStatusCf) - .get(&store_id).is_some() - } - - fn migration_done(&self, store_id: MigrationId) { - self.open_rw_context().cf(MigrationStatusCf) - .put(&store_id, &MigrationStatus::Completed) - } -} - /// A RocksDB-backed persistence layer for state manager. pub struct StateManagerDatabase { diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index d1a1ef587d..b988d5c308 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -74,7 +74,6 @@ use crate::RawDbMetrics; pub use crate::store::consensus::traits::DatabaseConfig; use crate::store::consensus::traits::measurement::MeasurableDatabase; -pub mod column_families; pub mod historical_state; pub mod jmt_gc; pub mod proofs_gc; diff --git a/core-rust/state-manager/src/store/p2p/column_families.rs b/core-rust/state-manager/src/store/p2p/column_families.rs new file mode 100644 index 0000000000..50f4566e65 --- /dev/null +++ b/core-rust/state-manager/src/store/p2p/column_families.rs @@ -0,0 +1,49 @@ +use crate::store::p2p::address_book_components::AddressBookNodeId; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::store::typed_cf_api::{ + AddressBookNodeIdDbCodec, DefaultCf, DirectDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec + , UnitDbCodec, +}; + +/// Address book and safety state store migration status. Filled once during the migration. +pub struct MigrationStatusCf; +impl DefaultCf for MigrationStatusCf { + type Key = MigrationId; + type Value = MigrationStatus; + + const DEFAULT_NAME: &'static str = "migration_status"; + type KeyCodec = MigrationIdDbCodec; + type ValueCodec = MigrationStatusDbCodec; +} + +/// Address book +pub struct AddressBookCf; +impl DefaultCf for AddressBookCf { + type Key = AddressBookNodeId; + type Value = Vec; + + const DEFAULT_NAME: &'static str = "address_book"; + type KeyCodec = AddressBookNodeIdDbCodec; + type ValueCodec = DirectDbCodec; +} + +/// Safety store +pub struct SafetyStoreCf; +impl DefaultCf for SafetyStoreCf { + type Key = (); + type Value = Vec; + + const DEFAULT_NAME: &'static str = "safety_store"; + type KeyCodec = UnitDbCodec; + type ValueCodec = DirectDbCodec; +} + +pub struct HighPriorityPeersCf; +impl DefaultCf for HighPriorityPeersCf { + type Key = (); + type Value = Vec; + + const DEFAULT_NAME: &'static str = "high_priority_peers"; + type KeyCodec = UnitDbCodec; + type ValueCodec = DirectDbCodec; +} diff --git a/core-rust/state-manager/src/store/p2p/mod.rs b/core-rust/state-manager/src/store/p2p/mod.rs index 4394b28dc8..d5df6f1718 100644 --- a/core-rust/state-manager/src/store/p2p/mod.rs +++ b/core-rust/state-manager/src/store/p2p/mod.rs @@ -1,3 +1,5 @@ pub mod address_book_components; pub mod traits; -pub mod migration; \ No newline at end of file +pub mod migration; +pub mod column_families; +pub mod rocks_db; diff --git a/core-rust/state-manager/src/store/p2p/rocks_db.rs b/core-rust/state-manager/src/store/p2p/rocks_db.rs new file mode 100644 index 0000000000..6cd3ff50e8 --- /dev/null +++ b/core-rust/state-manager/src/store/p2p/rocks_db.rs @@ -0,0 +1,208 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use std::path::PathBuf; + +use rocksdb::{ + ColumnFamilyDescriptor, DB, Options, +}; + +use crate::engine_prelude::*; +use crate::p2p::address_book_components::AddressBookNodeId; +use crate::p2p::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; +use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; +use crate::store::common::rocks_db::*; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::store::typed_cf_api::*; + +/// A listing of all column family names used by the Node. +/// +/// This is directly needed to initialize the column families within the DB, but is also a nice +/// place to link to all of them (please see the documentation of each CF to learn about its +/// business purpose and DB schema) and to put the important general notes regarding all of them +/// (see below). +/// +/// **Note on the key encoding used throughout all column families:** +/// We often rely on the RocksDB's unsurprising ability to efficiently list entries sorted +/// lexicographically by key. For this reason, our byte-level encoding of certain keys (e.g. +/// [`StateVersion`]) needs to reflect the business-level ordering of the represented concept (i.e. +/// since state versions grow, the "last" state version must have a lexicographically greatest key, +/// which means that we need to use a constant-length big-endian integer encoding). +/// +/// **Note on the name strings:** +/// The `NAME` constants defined by `*Cf` structs (and referenced below) are used as database column +/// family names. Any change would effectively mean a ledger wipe. For this reason, we choose to +/// define them manually (rather than using the `Into`, which is refactor-sensitive). + +const ALL_NODE_COLUMN_FAMILIES: [&str; 4] = [ + MigrationStatusCf::DEFAULT_NAME, + AddressBookCf::DEFAULT_NAME, + SafetyStoreCf::DEFAULT_NAME, + HighPriorityPeersCf::DEFAULT_NAME, +]; + +pub type ActualNodeDatabase = NodeDatabase; + +/// A RocksDB-backed persistence layer for node-specific address book and safety store. +pub struct NodeDatabase { + /// Underlying RocksDB instance. + rocks: R, +} + +impl ActualNodeDatabase { + pub fn new( + root_path: PathBuf, + ) -> ActualNodeDatabase { + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + + let column_families: Vec = ALL_NODE_COLUMN_FAMILIES + .iter() + .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) + .collect(); + + let db = DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap(); + + NodeDatabase { + rocks: DirectRocks { db }, + } + } +} + +impl NodeDatabase { + fn open_rw_context(&self) -> TypedDbContext> { + TypedDbContext::new(&self.rocks, BufferedWriteSupport::new(&self.rocks)) + } +} + +impl AddressBookStore for NodeDatabase { + fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { + let binding = self.open_rw_context(); + let context = binding.cf(AddressBookCf); + + if context.get(node_id).is_some() { + context.delete(node_id); + } + + true + } + + fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { + let binding = self.open_rw_context(); + let context = binding.cf(AddressBookCf); + + context.put(node_id, &entry.to_vec()); + + true + } + + fn reset(&self) { + self.open_rw_context().cf(AddressBookCf).delete_all(); + } + + fn get_all(&self) -> Vec> { + self.open_rw_context() + .cf(AddressBookCf) + .get_all() + } +} + +impl HighPriorityPeersStore for NodeDatabase { + fn upsert_all_peers(&self, peers: &[u8]) { + self.open_rw_context().cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); + } + + fn get_all_peers(&self) -> Option> { + self.open_rw_context().cf(HighPriorityPeersCf) + .get(&()) + } + + fn reset_high_priority_peers(&self) { + self.open_rw_context().cf(HighPriorityPeersCf).delete(&()); + } +} + +impl SafetyStateStore for NodeDatabase { + fn upsert_safety_state(&self, safety_state: &[u8]) { + self.open_rw_context().cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); + } + + fn get_safety_state(&self) -> Option> { + self.open_rw_context().cf(SafetyStoreCf) + .get(&()) + } +} + +impl MigrationStore for NodeDatabase { + fn is_migration_done(&self, store_id: MigrationId) -> bool { + self.open_rw_context().cf(MigrationStatusCf) + .get(&store_id).is_some() + } + + fn migration_done(&self, store_id: MigrationId) { + self.open_rw_context().cf(MigrationStatusCf) + .put(&store_id, &MigrationStatus::Completed) + } +} From 1f9b18e87f530ff8c4add34fb60ec21253d6aa39 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 12 Aug 2024 11:00:00 +0200 Subject: [PATCH 13/40] Separation of DB-related code at Rust side (step #3) --- core-rust/state-manager/src/store/codecs.rs | 2 +- .../state-manager/src/store/common/mod.rs | 3 +- .../src/store/{ => common}/typed_cf_api.rs | 56 -------- .../src/store/consensus/column_families.rs | 2 +- .../src/store/consensus/rocks_db.rs | 2 +- core-rust/state-manager/src/store/mod.rs | 1 - .../src/store/p2p/column_families.rs | 10 +- core-rust/state-manager/src/store/p2p/mod.rs | 1 + .../state-manager/src/store/p2p/rocks_db.rs | 2 +- .../src/store/p2p/typed_cf_api.rs | 122 ++++++++++++++++++ 10 files changed, 135 insertions(+), 66 deletions(-) rename core-rust/state-manager/src/store/{ => common}/typed_cf_api.rs (95%) create mode 100644 core-rust/state-manager/src/store/p2p/typed_cf_api.rs diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index bd26a13d5a..10aa6e9745 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -69,7 +69,7 @@ use crate::engine_prelude::*; use crate::store::consensus::traits::indices::CreationId; use crate::store::consensus::traits::scenario::ScenarioSequenceNumber; -use crate::store::typed_cf_api::*; +use crate::store::common::typed_cf_api::*; use crate::transaction::RawLedgerTransaction; use crate::StateVersion; diff --git a/core-rust/state-manager/src/store/common/mod.rs b/core-rust/state-manager/src/store/common/mod.rs index 45770b569f..788d25fb73 100644 --- a/core-rust/state-manager/src/store/common/mod.rs +++ b/core-rust/state-manager/src/store/common/mod.rs @@ -1 +1,2 @@ -pub mod rocks_db; \ No newline at end of file +pub mod rocks_db; +pub mod typed_cf_api; \ No newline at end of file diff --git a/core-rust/state-manager/src/store/typed_cf_api.rs b/core-rust/state-manager/src/store/common/typed_cf_api.rs similarity index 95% rename from core-rust/state-manager/src/store/typed_cf_api.rs rename to core-rust/state-manager/src/store/common/typed_cf_api.rs index 1e0ae59bd4..4ffdd6d79c 100644 --- a/core-rust/state-manager/src/store/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/common/typed_cf_api.rs @@ -62,9 +62,7 @@ * permissions under this License. */ -use crate::p2p::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; use crate::store::common::rocks_db::{ReadableRocks, WriteableRocks}; use itertools::Itertools; use rocksdb::{ColumnFamily, Direction, IteratorMode, WriteBatch}; @@ -732,25 +730,6 @@ impl DbCodec> for DirectDbCodec { } } -#[derive(Clone, Default)] -pub struct AddressBookNodeIdDbCodec {} - -impl DbCodec for AddressBookNodeIdDbCodec { - fn encode(&self, value: &AddressBookNodeId) -> Vec { - value.as_bytes().to_vec() - } - - fn decode(&self, bytes: &[u8]) -> AddressBookNodeId { - AddressBookNodeId::new(bytes.try_into().expect("Invalid NodeId")) - } -} - -impl BoundedDbCodec for AddressBookNodeIdDbCodec { - fn upper_bound_encoding(&self) -> Vec { - vec![0xFF; AddressBookNodeId::LENGTH] - } -} - /// A [`DbCodec]` capable of representing only a unit `()` (as an empty array). /// This is useful e.g. for "single-row" column families (which do not need keys), or "key-only" /// column families (which do not need values). @@ -767,41 +746,6 @@ impl DbCodec<()> for UnitDbCodec { } } -/// A [`DbCodec]` capable of representing [`MigrationId]`. -#[derive(Clone, Default)] -pub struct MigrationIdDbCodec; - -impl DbCodec for MigrationIdDbCodec { - fn encode(&self, key: &MigrationId) -> Vec { - vec![*key as u8] - } - - fn decode(&self, bytes: &[u8]) -> MigrationId { - match bytes[0] { - 0 => MigrationId::AddressBook, - 1 => MigrationId::SafetyState, - _ => panic!("Invalid MigrationId"), - } - } -} - -/// A [`DbCodec]` capable of representing [`MigrationStatus]`. -#[derive(Clone, Default)] -pub struct MigrationStatusDbCodec; - -impl DbCodec for MigrationStatusDbCodec { - fn encode(&self, key: &MigrationStatus) -> Vec { - vec![*key as u8] - } - - fn decode(&self, bytes: &[u8]) -> MigrationStatus { - match bytes[0] { - 0 => MigrationStatus::Completed, - _ => panic!("Invalid MigrationId"), - } - } -} - /// A [`DbCodec]` based on a predefined set of mappings. #[derive(Default)] pub struct PredefinedDbCodec { diff --git a/core-rust/state-manager/src/store/consensus/column_families.rs b/core-rust/state-manager/src/store/consensus/column_families.rs index 45cc92043b..72a9524eb8 100644 --- a/core-rust/state-manager/src/store/consensus/column_families.rs +++ b/core-rust/state-manager/src/store/consensus/column_families.rs @@ -32,7 +32,7 @@ use crate::store::codecs::{ StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, TypeAndCreationIndexKeyDbCodec, }; -use crate::store::typed_cf_api::{ +use crate::store::common::typed_cf_api::{ DefaultCf, DirectDbCodec, PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, }; diff --git a/core-rust/state-manager/src/store/consensus/rocks_db.rs b/core-rust/state-manager/src/store/consensus/rocks_db.rs index 372a744999..347beb7555 100644 --- a/core-rust/state-manager/src/store/consensus/rocks_db.rs +++ b/core-rust/state-manager/src/store/consensus/rocks_db.rs @@ -94,7 +94,7 @@ use crate::store::consensus::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; use crate::store::historical_state::StateTreeBasedSubstateDatabase; -use crate::store::typed_cf_api::*; +use crate::store::common::typed_cf_api::*; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index b988d5c308..c9d26dea1d 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -78,7 +78,6 @@ pub mod historical_state; pub mod jmt_gc; pub mod proofs_gc; mod codecs; -mod typed_cf_api; pub mod p2p; pub mod consensus; pub mod common; diff --git a/core-rust/state-manager/src/store/p2p/column_families.rs b/core-rust/state-manager/src/store/p2p/column_families.rs index 50f4566e65..3b02d3fa32 100644 --- a/core-rust/state-manager/src/store/p2p/column_families.rs +++ b/core-rust/state-manager/src/store/p2p/column_families.rs @@ -1,9 +1,11 @@ +use crate::store::common::typed_cf_api::{ + DefaultCf, DirectDbCodec, UnitDbCodec, +}; +use crate::store::p2p::typed_cf_api::{ + AddressBookNodeIdDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec, +}; use crate::store::p2p::address_book_components::AddressBookNodeId; use crate::store::p2p::migration::{MigrationId, MigrationStatus}; -use crate::store::typed_cf_api::{ - AddressBookNodeIdDbCodec, DefaultCf, DirectDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec - , UnitDbCodec, -}; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; diff --git a/core-rust/state-manager/src/store/p2p/mod.rs b/core-rust/state-manager/src/store/p2p/mod.rs index d5df6f1718..21d63c8211 100644 --- a/core-rust/state-manager/src/store/p2p/mod.rs +++ b/core-rust/state-manager/src/store/p2p/mod.rs @@ -3,3 +3,4 @@ pub mod traits; pub mod migration; pub mod column_families; pub mod rocks_db; +pub mod typed_cf_api; diff --git a/core-rust/state-manager/src/store/p2p/rocks_db.rs b/core-rust/state-manager/src/store/p2p/rocks_db.rs index 6cd3ff50e8..53201c1f74 100644 --- a/core-rust/state-manager/src/store/p2p/rocks_db.rs +++ b/core-rust/state-manager/src/store/p2p/rocks_db.rs @@ -74,7 +74,7 @@ use crate::p2p::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationS use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; use crate::store::common::rocks_db::*; use crate::store::p2p::migration::{MigrationId, MigrationStatus}; -use crate::store::typed_cf_api::*; +use crate::store::common::typed_cf_api::*; /// A listing of all column family names used by the Node. /// diff --git a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs new file mode 100644 index 0000000000..909b970735 --- /dev/null +++ b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs @@ -0,0 +1,122 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::p2p::address_book_components::AddressBookNodeId; +use crate::engine_prelude::*; +use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::store::common::typed_cf_api::{ DbCodec, BoundedDbCodec}; + +#[derive(Clone, Default)] +pub struct AddressBookNodeIdDbCodec {} + +impl DbCodec for AddressBookNodeIdDbCodec { + fn encode(&self, value: &AddressBookNodeId) -> Vec { + value.as_bytes().to_vec() + } + + fn decode(&self, bytes: &[u8]) -> AddressBookNodeId { + AddressBookNodeId::new(bytes.try_into().expect("Invalid NodeId")) + } +} + +impl BoundedDbCodec for AddressBookNodeIdDbCodec { + fn upper_bound_encoding(&self) -> Vec { + vec![0xFF; AddressBookNodeId::LENGTH] + } +} + +/// A [`DbCodec]` capable of representing [`MigrationId]`. +#[derive(Clone, Default)] +pub struct MigrationIdDbCodec; + +impl DbCodec for MigrationIdDbCodec { + fn encode(&self, key: &MigrationId) -> Vec { + vec![*key as u8] + } + + fn decode(&self, bytes: &[u8]) -> MigrationId { + match bytes[0] { + 0 => MigrationId::AddressBook, + 1 => MigrationId::SafetyState, + _ => panic!("Invalid MigrationId"), + } + } +} + +/// A [`DbCodec]` capable of representing [`MigrationStatus]`. +#[derive(Clone, Default)] +pub struct MigrationStatusDbCodec; + +impl DbCodec for MigrationStatusDbCodec { + fn encode(&self, key: &MigrationStatus) -> Vec { + vec![*key as u8] + } + + fn decode(&self, bytes: &[u8]) -> MigrationStatus { + match bytes[0] { + 0 => MigrationStatus::Completed, + _ => panic!("Invalid MigrationId"), + } + } +} From 6fcc8597b96a4d2d9211d4b648b57254308cf87c Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 12 Aug 2024 11:29:32 +0200 Subject: [PATCH 14/40] Separation of DB-related code at Rust side (step #4) --- core-rust/state-manager/src/jni/mod.rs | 5 +---- core-rust/state-manager/src/jni/{ => p2p}/address_book.rs | 0 .../state-manager/src/jni/{ => p2p}/high_priority_peers.rs | 0 core-rust/state-manager/src/jni/{ => p2p}/migration_store.rs | 0 core-rust/state-manager/src/jni/p2p/mod.rs | 4 ++++ core-rust/state-manager/src/jni/{ => p2p}/safety_store.rs | 0 6 files changed, 5 insertions(+), 4 deletions(-) rename core-rust/state-manager/src/jni/{ => p2p}/address_book.rs (100%) rename core-rust/state-manager/src/jni/{ => p2p}/high_priority_peers.rs (100%) rename core-rust/state-manager/src/jni/{ => p2p}/migration_store.rs (100%) create mode 100644 core-rust/state-manager/src/jni/p2p/mod.rs rename core-rust/state-manager/src/jni/{ => p2p}/safety_store.rs (100%) diff --git a/core-rust/state-manager/src/jni/mod.rs b/core-rust/state-manager/src/jni/mod.rs index 1b95eae905..e018db38fc 100644 --- a/core-rust/state-manager/src/jni/mod.rs +++ b/core-rust/state-manager/src/jni/mod.rs @@ -62,22 +62,19 @@ * permissions under this License. */ -pub mod address_book; pub mod db_checkpoints; pub mod fatal_panic_handler; -pub mod high_priority_peers; pub mod mempool; -pub mod migration_store; pub mod node_rust_environment; pub mod prometheus; pub mod protocol_update; -pub mod safety_store; pub mod state_computer; pub mod state_reader; pub mod test_state_reader; pub mod transaction_preparer; pub mod transaction_store; pub mod vertex_store_recovery; +pub mod p2p; use crate::engine_prelude::*; diff --git a/core-rust/state-manager/src/jni/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs similarity index 100% rename from core-rust/state-manager/src/jni/address_book.rs rename to core-rust/state-manager/src/jni/p2p/address_book.rs diff --git a/core-rust/state-manager/src/jni/high_priority_peers.rs b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs similarity index 100% rename from core-rust/state-manager/src/jni/high_priority_peers.rs rename to core-rust/state-manager/src/jni/p2p/high_priority_peers.rs diff --git a/core-rust/state-manager/src/jni/migration_store.rs b/core-rust/state-manager/src/jni/p2p/migration_store.rs similarity index 100% rename from core-rust/state-manager/src/jni/migration_store.rs rename to core-rust/state-manager/src/jni/p2p/migration_store.rs diff --git a/core-rust/state-manager/src/jni/p2p/mod.rs b/core-rust/state-manager/src/jni/p2p/mod.rs new file mode 100644 index 0000000000..97b8a8e6b4 --- /dev/null +++ b/core-rust/state-manager/src/jni/p2p/mod.rs @@ -0,0 +1,4 @@ +pub mod address_book; +pub mod high_priority_peers; +pub mod migration_store; +pub mod safety_store; \ No newline at end of file diff --git a/core-rust/state-manager/src/jni/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs similarity index 100% rename from core-rust/state-manager/src/jni/safety_store.rs rename to core-rust/state-manager/src/jni/p2p/safety_store.rs From 0a311800f60f0aa33b6a5d81f6473e1f78120aec Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Tue, 13 Aug 2024 14:10:53 +0200 Subject: [PATCH 15/40] Make separate address book, safety store and migration store. --- .../java/com/radixdlt/shell/RadixShell.java | 1 - .../com/radixdlt/environment/NodeConfig.java | 75 ------------ .../environment/NodeRustEnvironment.java | 10 +- .../com/radixdlt/sbor/NodeSborCodecs.java | 1 - .../db/RocksDbMigrationStoreTest.java | 4 +- .../p2p/RocksDbAddressBookStoreTest.java | 4 +- .../RocksDbHighPriorityPeersStoreTest.java | 4 +- .../safety/RocksDbSafetyStoreTest.java | 4 +- .../src/jni/node_rust_environment.rs | 91 +++++++++----- .../state-manager/src/jni/p2p/address_book.rs | 12 +- .../src/jni/p2p/high_priority_peers.rs | 9 +- .../src/jni/p2p/migration_store.rs | 6 +- .../state-manager/src/jni/p2p/safety_store.rs | 6 +- core-rust/state-manager/src/lib.rs | 1 - core-rust/state-manager/src/node.rs | 71 ----------- .../state-manager/src/store/p2p/rocks_db.rs | 115 ++++++++++++------ core/mainnet.config | 1 - .../radixdlt/api/system/SystemApiModule.java | 5 +- .../bootstrap/RadixNodeBootstrapper.java | 6 +- .../RadixNodeBootstrapperModule.java | 4 +- .../safety/BerkeleySafetyStateStore.java | 4 +- .../addressbook/BerkeleyAddressBookStore.java | 8 +- .../rev2/modules/NodePersistenceModule.java | 6 +- .../rev2/modules/REv2StateManagerModule.java | 21 +--- .../store/StateManagerStorageLocation.java | 80 ------------ .../StorageLocationFromPropertiesModule.java | 9 +- core/src/main/resources/default.config | 4 - .../modules/FunctionalRadixNodeModule.java | 4 +- .../PrefixedNodeStorageLocationModule.java | 10 +- .../safety/RocksSafetyStateStoreTest.java | 7 +- .../RocksAddressBookStoreTest.java | 4 +- .../p2p/test/P2PTestNetworkRunner.java | 6 +- .../radixdlt/rev2/REv2StateComputerTest.java | 6 +- .../com/radixdlt/rev2/RustMempoolTest.java | 4 +- core/stokenet.config | 1 - 35 files changed, 186 insertions(+), 418 deletions(-) delete mode 100644 core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java delete mode 100644 core-rust/state-manager/src/node.rs delete mode 100644 core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java diff --git a/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java b/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java index c3160692d2..6f1acdddba 100644 --- a/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java +++ b/cli-tools/src/main/java/com/radixdlt/shell/RadixShell.java @@ -181,7 +181,6 @@ public Node build() throws Exception { customProperties.build().forEach(properties::set); properties.set("db.location", dataDir.toString()); - properties.set("db.location.node", new File(dataDir, "node").toString()); if (properties.get("node.key.path", "").isEmpty()) { properties.set("node.key.path", new File(dataDir, "node-keystore.ks").getAbsolutePath()); diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java deleted file mode 100644 index 79f2a74328..0000000000 --- a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeConfig.java +++ /dev/null @@ -1,75 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.environment; - -import com.radixdlt.sbor.codec.CodecMap; -import com.radixdlt.sbor.codec.StructCodec; - -public record NodeConfig(DatabaseBackendConfig databaseBackendConfig) { - public static void registerCodec(CodecMap codecMap) { - codecMap.register( - NodeConfig.class, codecs -> StructCodec.fromRecordComponents(NodeConfig.class, codecs)); - } -} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java index 996f2e98d0..4e394b651e 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java @@ -90,15 +90,12 @@ public final class NodeRustEnvironment implements AutoCloseable { public NodeRustEnvironment( MempoolRelayDispatcher mempoolRelayDispatcher, FatalPanicHandler fatalPanicHandler, - StateManagerConfig config, - NodeConfig nodeConfig) { + StateManagerConfig config) { this.mempoolRelayDispatcher = mempoolRelayDispatcher; this.fatalPanicHandler = fatalPanicHandler; final var encodedConfig = NodeSborCodecs.encode(config, NodeSborCodecs.resolveCodec(new TypeToken<>() {})); - final var encodedNodeConfig = - NodeSborCodecs.encode(nodeConfig, NodeSborCodecs.resolveCodec(new TypeToken<>() {})); - init(this, encodedConfig, encodedNodeConfig); + init(this, encodedConfig); } @Override @@ -131,8 +128,7 @@ public void handleFatalPanic() { this.fatalPanicHandler.handleFatalPanic(); } - private static native void init( - NodeRustEnvironment nodeRustEnvironment, byte[] config, byte[] nodeConfig); + private static native void init(NodeRustEnvironment nodeRustEnvironment, byte[] config); private static native void cleanup(NodeRustEnvironment nodeRustEnvironment); } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index 556b2b045d..ff5534fcd2 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -218,7 +218,6 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { StateTreeGcConfig.registerCodec(codecMap); LedgerProofsGcConfig.registerCodec(codecMap); LedgerSyncLimitsConfig.registerCodec(codecMap); - NodeConfig.registerCodec(codecMap); BFTHeaderDTO.registerCodec(codecMap); BFTValidatorDTO.registerCodec(codecMap); diff --git a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java index d9c54c0ffa..2a6444b483 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java @@ -102,7 +102,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -127,7 +126,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index 03b6b6561b..b563566a64 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -197,7 +197,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -222,8 +221,7 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } private static AddressBookEntryDTO newAddressBookEntry(int id) { diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java index 3031ed070f..74d743ae00 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java @@ -133,7 +133,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -158,7 +157,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 958a34c15c..19936a4c87 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -118,7 +118,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -143,7 +142,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } } diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index 27eebbe2da..bb9960d0dd 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -61,9 +61,8 @@ * Work. You assume all risks associated with Your use of the Work and the exercise of * permissions under this License. */ - use std::ops::Deref; -use std::path::PathBuf; +use std::path::{MAIN_SEPARATOR, PathBuf}; use std::str::FromStr; use std::sync::Arc; @@ -72,7 +71,7 @@ use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::environment::setup_tracing; -use node_common::java::{jni_call, jni_jbytearray_to_vector, StructFromJava}; +use node_common::java::{jni_call, jni_jbytearray_to_vector}; use node_common::locks::*; use prometheus::Registry; @@ -87,12 +86,12 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use crate::store::p2p::rocks_db::ActualNodeDatabase; +use crate::store::p2p::rocks_db::ActualAddressBookDatabase; use crate::store::consensus::rocks_db::ActualStateManagerDatabase; use crate::transaction::Preparator; -use crate::{Committer, LedgerMetrics, SystemExecutor}; +use crate::{Committer, DatabaseBackendConfig, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; -use crate::node::NodeConfig; +use crate::p2p::rocks_db::{ActualMigrationDatabase, ActualSafetyStoreDatabase}; const POINTER_JNI_FIELD_NAME: &str = "rustNodeRustEnvironmentPointer"; @@ -102,10 +101,9 @@ extern "system" fn Java_com_radixdlt_environment_NodeRustEnvironment_init( _class: JClass, j_node_rust_env: JObject, j_config: jbyteArray, - j_node_config: jbyteArray, ) { jni_call(&env, || { - JNINodeRustEnvironment::init(&env, j_node_rust_env, j_config, j_node_config) + JNINodeRustEnvironment::init(&env, j_node_rust_env, j_config) }); } @@ -126,18 +124,15 @@ pub struct JNINodeRustEnvironment { pub state_manager: StateManager, pub metric_registry: Arc, pub running_task_tracker: UntilDropTracker, - pub node_store: Arc>, + pub address_book_store: Arc, + pub safety_store_store: Arc, + pub migration_store: Arc, } impl JNINodeRustEnvironment { - pub fn init(env: &JNIEnv, j_node_rust_env: JObject, j_config: jbyteArray, j_node_config: jbyteArray) { - let config_bytes: Vec = jni_jbytearray_to_vector(env, j_config).unwrap(); - let config = StateManagerConfig::valid_from_java(&config_bytes).unwrap(); - - let db_config_bytes: Vec = jni_jbytearray_to_vector(env, j_node_config).unwrap(); - let node_config = NodeConfig::from_java(&db_config_bytes).unwrap(); + pub fn init(env: &JNIEnv, j_node_rust_env: JObject, j_config: jbyteArray) { + let (base_path, config) = Self::prepare_config(&jni_jbytearray_to_vector(env, j_config).unwrap()); let network = config.network_definition.clone(); - let runtime = Arc::new(Runtime::new().unwrap()); setup_tracing( @@ -159,7 +154,7 @@ impl JNINodeRustEnvironment { .use_tokio(runtime.deref()) .track_running_tasks() .measured(metric_registry.deref()); - + let state_manager = StateManager::new( config, Some(MempoolRelayDispatcher::new(env, j_node_rust_env).unwrap()), @@ -169,10 +164,10 @@ impl JNINodeRustEnvironment { ); let running_task_tracker = scheduler.into_task_tracker(); - - let node_db_path = PathBuf::from(node_config.database_backend_config.rocks_db_path); - let raw_node_db = ActualNodeDatabase::new(node_db_path); - let node_store = Arc::new(lock_factory.named("node_database").new_db_lock(raw_node_db)); + + let address_book_db_path = Self::combine(&base_path, "address_book"); + let safety_store_db_path = Self::combine(&base_path, "safety_store"); + let migration_store_db_path = Self::combine(&base_path, "migration_store"); let jni_node_rust_env = JNINodeRustEnvironment { runtime, @@ -180,13 +175,41 @@ impl JNINodeRustEnvironment { state_manager, metric_registry, running_task_tracker, - node_store, + address_book_store: Arc::new(ActualAddressBookDatabase::new(address_book_db_path)), + safety_store_store: Arc::new(ActualSafetyStoreDatabase::new(safety_store_db_path)), + migration_store: Arc::new(ActualMigrationDatabase::new(migration_store_db_path)), }; env.set_rust_field(j_node_rust_env, POINTER_JNI_FIELD_NAME, jni_node_rust_env) .unwrap(); } + fn prepare_config(config_bytes: &[u8]) -> (String, StateManagerConfig) { + let config = StateManagerConfig::valid_from_java(config_bytes).unwrap(); + let base_path = config.database_backend_config.rocks_db_path.clone(); + let mut state_manager_db_path = config.database_backend_config.rocks_db_path.clone(); + + state_manager_db_path.push(MAIN_SEPARATOR); + state_manager_db_path.push_str("state_manager"); + + let config = StateManagerConfig { + database_backend_config: DatabaseBackendConfig { + rocks_db_path: state_manager_db_path + }, + ..config + }; + + (base_path, config) + } + + fn combine(base: &String, ext: &str) -> PathBuf { + let mut path = base.clone(); + path.push(MAIN_SEPARATOR); + path.push_str(ext); + + PathBuf::from(path) + } + pub fn cleanup(env: &JNIEnv, j_node_rust_env: JObject) { let jni_node_rust_env: JNINodeRustEnvironment = env .take_rust_field(j_node_rust_env, POINTER_JNI_FIELD_NAME) @@ -219,16 +242,28 @@ impl JNINodeRustEnvironment { .database .clone() } - - pub fn get_node_database( + + pub fn get_address_book_database( env: &JNIEnv, j_node_rust_env: JObject, - ) -> Arc> { - Self::get(env, j_node_rust_env) - .node_store - .clone() + ) -> Arc { + Self::get(env, j_node_rust_env).address_book_store.clone() + } + + pub fn get_safety_store_database( + env: &JNIEnv, + j_node_rust_env: JObject, + ) -> Arc { + Self::get(env, j_node_rust_env).safety_store_store.clone() } + pub fn get_migration_database( + env: &JNIEnv, + j_node_rust_env: JObject, + ) -> Arc { + Self::get(env, j_node_rust_env).migration_store.clone() + } + pub fn get_mempool(env: &JNIEnv, j_node_rust_env: JObject) -> Arc> { Self::get(env, j_node_rust_env) .state_manager diff --git a/core-rust/state-manager/src/jni/p2p/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs index 8df2dd5761..c71374f7c0 100644 --- a/core-rust/state-manager/src/jni/p2p/address_book.rs +++ b/core-rust/state-manager/src/jni/p2p/address_book.rs @@ -79,8 +79,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |request: AddressBookNodeId| -> bool { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .remove_one(&request) }) } @@ -97,8 +96,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_upsertOne( // jni_sbor_coded_call. let entity = jni_jbytearray_to_vector(&env, address_entry).unwrap(); - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .upsert_one(&request.node_id, &entity) }) } @@ -111,8 +109,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_reset( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .reset(); }) } @@ -125,8 +122,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec> { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .get_all() }) } diff --git a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs index 989636d43b..942c9ea016 100644 --- a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs @@ -80,8 +80,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAll payload: jbyteArray, ) -> jbyteArray { jni_raw_sbor_fallible_call(&env, payload, |bytes| { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .upsert_all_peers(&bytes); Ok(()) }) @@ -95,8 +94,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHig payload: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, payload, |_: ()| -> Option> { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .get_all_peers() }) } @@ -109,8 +107,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_resetHigh node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .reset_high_priority_peers(); }) } diff --git a/core-rust/state-manager/src/jni/p2p/migration_store.rs b/core-rust/state-manager/src/jni/p2p/migration_store.rs index 2a29e6a628..9f1e23a4f1 100644 --- a/core-rust/state-manager/src/jni/p2p/migration_store.rs +++ b/core-rust/state-manager/src/jni/p2p/migration_store.rs @@ -80,8 +80,7 @@ extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_migrationDone( migration_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_migration_database(&env, j_rust_global_context) .migration_done(migration_store_id); }) } @@ -94,8 +93,7 @@ extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_isMigrated( migration_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| -> bool { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_migration_database(&env, j_rust_global_context) .is_migration_done(migration_store_id) }) } diff --git a/core-rust/state-manager/src/jni/p2p/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs index d150e9fc24..1bb142bad2 100644 --- a/core-rust/state-manager/src/jni/p2p/safety_store.rs +++ b/core-rust/state-manager/src/jni/p2p/safety_store.rs @@ -78,8 +78,7 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_upsert( request: jbyteArray, ) -> jbyteArray { jni_raw_sbor_fallible_call(&env, request, |bytes| { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) .upsert_safety_state(&bytes); Ok(()) }) @@ -93,8 +92,7 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_get( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| -> Option> { - JNINodeRustEnvironment::get_node_database(&env, j_rust_global_context) - .lock() + JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) .get_safety_state() }) } diff --git a/core-rust/state-manager/src/lib.rs b/core-rust/state-manager/src/lib.rs index c9bc8dc4a0..694c01fdb5 100644 --- a/core-rust/state-manager/src/lib.rs +++ b/core-rust/state-manager/src/lib.rs @@ -73,7 +73,6 @@ pub mod mempool; pub mod metrics; pub mod protocol; pub mod query; -mod node; mod receipt; mod staging; mod state_manager; diff --git a/core-rust/state-manager/src/node.rs b/core-rust/state-manager/src/node.rs deleted file mode 100644 index 1be1d5b661..0000000000 --- a/core-rust/state-manager/src/node.rs +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -use crate::engine_prelude::*; -use crate::store::DatabaseBackendConfig; - -#[derive(Clone, Debug, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] -pub struct NodeConfig { - pub database_backend_config: DatabaseBackendConfig, -} diff --git a/core-rust/state-manager/src/store/p2p/rocks_db.rs b/core-rust/state-manager/src/store/p2p/rocks_db.rs index 53201c1f74..596a58f247 100644 --- a/core-rust/state-manager/src/store/p2p/rocks_db.rs +++ b/core-rust/state-manager/src/store/p2p/rocks_db.rs @@ -95,51 +95,89 @@ use crate::store::common::typed_cf_api::*; /// family names. Any change would effectively mean a ledger wipe. For this reason, we choose to /// define them manually (rather than using the `Into`, which is refactor-sensitive). -const ALL_NODE_COLUMN_FAMILIES: [&str; 4] = [ - MigrationStatusCf::DEFAULT_NAME, +const ALL_ADDRESS_BOOK_COLUMN_FAMILIES: [&str; 2] = [ AddressBookCf::DEFAULT_NAME, - SafetyStoreCf::DEFAULT_NAME, HighPriorityPeersCf::DEFAULT_NAME, ]; +const ALL_SAFETY_STORE_COLUMN_FAMILIES: [&str; 1] = [ + SafetyStoreCf::DEFAULT_NAME, +]; +const ALL_MIGRATION_STORE_COLUMN_FAMILIES: [&str; 1] = [ + MigrationStatusCf::DEFAULT_NAME, +]; -pub type ActualNodeDatabase = NodeDatabase; +pub type ActualAddressBookDatabase = AddressBookDatabase; +pub type ActualSafetyStoreDatabase = SafetyStoreDatabase; +pub type ActualMigrationDatabase = MigrationDatabase; -/// A RocksDB-backed persistence layer for node-specific address book and safety store. -pub struct NodeDatabase { +/// A RocksDB-backed persistence layer for address book. +pub struct AddressBookDatabase { /// Underlying RocksDB instance. rocks: R, } -impl ActualNodeDatabase { - pub fn new( - root_path: PathBuf, - ) -> ActualNodeDatabase { - let mut db_opts = Options::default(); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); +/// A RocksDB-backed persistence layer for safety store. +pub struct SafetyStoreDatabase { + /// Underlying RocksDB instance. + rocks: R, +} + +/// A RocksDB-backed persistence layer for migration database. +pub struct MigrationDatabase { + /// Underlying RocksDB instance. + rocks: R, +} - let column_families: Vec = ALL_NODE_COLUMN_FAMILIES - .iter() - .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) - .collect(); +fn new_rocks_db(root_path: PathBuf, column_families: &[&str]) -> DB { + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); - let db = DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap(); + let column_families: Vec = column_families + .iter() + .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), Options::default())) + .collect(); - NodeDatabase { - rocks: DirectRocks { db }, + DB::open_cf_descriptors(&db_opts, root_path.as_path(), column_families).unwrap() +} + +fn open_rw_context(db: &R) -> TypedDbContext> { + TypedDbContext::new(db, BufferedWriteSupport::new(db)) +} + +impl ActualAddressBookDatabase { + pub fn new( + root_path: PathBuf, + ) -> ActualAddressBookDatabase { + AddressBookDatabase { + rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_ADDRESS_BOOK_COLUMN_FAMILIES) }, } } } -impl NodeDatabase { - fn open_rw_context(&self) -> TypedDbContext> { - TypedDbContext::new(&self.rocks, BufferedWriteSupport::new(&self.rocks)) +impl ActualSafetyStoreDatabase { + pub fn new( + root_path: PathBuf, + ) -> ActualSafetyStoreDatabase { + ActualSafetyStoreDatabase { + rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_SAFETY_STORE_COLUMN_FAMILIES) }, + } + } +} + +impl ActualMigrationDatabase { + pub fn new( + root_path: PathBuf, + ) -> ActualMigrationDatabase { + ActualMigrationDatabase { + rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_MIGRATION_STORE_COLUMN_FAMILIES) }, + } } } -impl AddressBookStore for NodeDatabase { +impl AddressBookStore for AddressBookDatabase { fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { - let binding = self.open_rw_context(); + let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); if context.get(node_id).is_some() { @@ -150,7 +188,7 @@ impl AddressBookStore for NodeDatabase { } fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { - let binding = self.open_rw_context(); + let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); context.put(node_id, &entry.to_vec()); @@ -159,50 +197,49 @@ impl AddressBookStore for NodeDatabase { } fn reset(&self) { - self.open_rw_context().cf(AddressBookCf).delete_all(); + open_rw_context(&self.rocks).cf(AddressBookCf).delete_all(); } fn get_all(&self) -> Vec> { - self.open_rw_context() - .cf(AddressBookCf) + open_rw_context(&self.rocks).cf(AddressBookCf) .get_all() } } -impl HighPriorityPeersStore for NodeDatabase { +impl HighPriorityPeersStore for AddressBookDatabase { fn upsert_all_peers(&self, peers: &[u8]) { - self.open_rw_context().cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); + open_rw_context(&self.rocks).cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); } fn get_all_peers(&self) -> Option> { - self.open_rw_context().cf(HighPriorityPeersCf) + open_rw_context(&self.rocks).cf(HighPriorityPeersCf) .get(&()) } fn reset_high_priority_peers(&self) { - self.open_rw_context().cf(HighPriorityPeersCf).delete(&()); + open_rw_context(&self.rocks).cf(HighPriorityPeersCf).delete(&()); } } -impl SafetyStateStore for NodeDatabase { +impl SafetyStateStore for SafetyStoreDatabase { fn upsert_safety_state(&self, safety_state: &[u8]) { - self.open_rw_context().cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); + open_rw_context(&self.rocks).cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); } fn get_safety_state(&self) -> Option> { - self.open_rw_context().cf(SafetyStoreCf) + open_rw_context(&self.rocks).cf(SafetyStoreCf) .get(&()) } } -impl MigrationStore for NodeDatabase { +impl MigrationStore for MigrationDatabase { fn is_migration_done(&self, store_id: MigrationId) -> bool { - self.open_rw_context().cf(MigrationStatusCf) + open_rw_context(&self.rocks).cf(MigrationStatusCf) .get(&store_id).is_some() } fn migration_done(&self, store_id: MigrationId) { - self.open_rw_context().cf(MigrationStatusCf) + open_rw_context(&self.rocks).cf(MigrationStatusCf) .put(&store_id, &MigrationStatus::Completed) } } diff --git a/core/mainnet.config b/core/mainnet.config index f4e30dab46..a21fed8fdb 100644 --- a/core/mainnet.config +++ b/core/mainnet.config @@ -8,7 +8,6 @@ node.key.path=.//NODEMOUNT/mainnet/key/node.ks node.key.create_if_missing=true db.location=.//NODEMOUNT/mainnet/ledger -db.location.node=.//NODEMOUNT/mainnet/node network.p2p.seed_nodes=radix://node_rdx1qf2x63qx4jdaxj83kkw2yytehvvmu6r2xll5gcp6c9rancmrfsgfw0vnc65@babylon-mainnet-eu-west-1-node0.radixdlt.com,radix://node_rdx1qgxn3eeldj33kd98ha6wkjgk4k77z6xm0dv7mwnrkefknjcqsvhuu4gc609@babylon-mainnet-ap-southeast-2-node0.radixdlt.com,radix://node_rdx1qwrrnhzfu99fg3yqgk3ut9vev2pdssv7hxhff80msjmmcj968487uugc0t2@babylon-mainnet-ap-south-1-node0.radixdlt.com,radix://node_rdx1q0gnmwv0fmcp7ecq0znff7yzrt7ggwrp47sa9pssgyvrnl75tvxmvj78u7t@babylon-mainnet-us-east-1-node0.radixdlt.com diff --git a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java index e7cb60fcab..a973196386 100644 --- a/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java +++ b/core/src/main/java/com/radixdlt/api/system/SystemApiModule.java @@ -70,7 +70,7 @@ import com.radixdlt.api.system.health.HealthInfoService; import com.radixdlt.api.system.health.HealthInfoServiceImpl; import com.radixdlt.api.system.routes.*; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import io.undertow.server.HttpHandler; import java.nio.file.Path; @@ -116,8 +116,7 @@ public SystemApi systemApi(@SystemApiEndpoints Map ha @Provides @Singleton public SystemApiConfig systemApiConfig( - RuntimeProperties runtimeProperties, - @StateManagerStorageLocation String nodeStorageLocation) { + RuntimeProperties runtimeProperties, @NodeStorageLocation String nodeStorageLocation) { final var dbCheckpointEnabled = runtimeProperties.get("api.system.enable_db_checkpoint", false); final var dbCheckpointsPath = runtimeProperties.get( diff --git a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java index d43d96df96..477b2e7c73 100644 --- a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java +++ b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapper.java @@ -79,7 +79,7 @@ import com.radixdlt.lang.Unit; import com.radixdlt.networks.FixedNetworkGenesis; import com.radixdlt.networks.Network; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; import java.io.IOException; @@ -111,7 +111,7 @@ public static RadixNode runNewNode(RuntimeProperties properties) { RuntimeProperties properties, GenesisFromPropertiesLoader genesisFromPropertiesLoader, GenesisStore genesisStore, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { this.network = network; this.hasher = hasher; this.properties = properties; @@ -149,7 +149,7 @@ private RadixNode bootstrapRadixNode() { .map(Optional::get) .collect(ImmutableSet.toImmutableSet()); - if (distinctGenesisHashes.size() == 0) { + if (distinctGenesisHashes.isEmpty()) { // No genesis was configured throw new RuntimeException( """ diff --git a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java index 54d3d483b5..d08a05c9b1 100644 --- a/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java +++ b/core/src/main/java/com/radixdlt/bootstrap/RadixNodeBootstrapperModule.java @@ -74,7 +74,7 @@ import com.radixdlt.modules.CryptoModule; import com.radixdlt.modules.MetricsModule; import com.radixdlt.networks.Network; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.store.StorageLocationFromPropertiesModule; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; @@ -103,7 +103,7 @@ public void configure() { @Provides @Singleton - GenesisStore genesisStore(@StateManagerStorageLocation String nodeStorageLocation) { + GenesisStore genesisStore(@NodeStorageLocation String nodeStorageLocation) { return new GenesisFileStore(new File(nodeStorageLocation, "executed_genesis")); } diff --git a/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java index 3c57a6d906..d66f1260c8 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/BerkeleySafetyStateStore.java @@ -71,7 +71,7 @@ import com.radixdlt.serialization.DeserializeException; import com.radixdlt.serialization.DsonOutput; import com.radixdlt.serialization.Serialization; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.Longs; import com.sleepycat.je.*; import java.io.File; @@ -101,7 +101,7 @@ public final class BerkeleySafetyStateStore implements PersistentSafetyStateStor public BerkeleySafetyStateStore( Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation, + @NodeStorageLocation String nodeStorageLocation, EnvironmentConfig envConfig) { this.serialization = Objects.requireNonNull(serialization); this.metrics = Objects.requireNonNull(metrics); diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java b/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java index 286e86ca5c..07a08ff37d 100644 --- a/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/BerkeleyAddressBookStore.java @@ -72,7 +72,7 @@ import com.radixdlt.p2p.NodeId; import com.radixdlt.serialization.DsonOutput.Output; import com.radixdlt.serialization.Serialization; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.sleepycat.je.*; import java.io.File; import java.io.IOException; @@ -101,7 +101,7 @@ public final class BerkeleyAddressBookStore implements AddressBookPersistence { public BerkeleyAddressBookStore( Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation, + @NodeStorageLocation String nodeStorageLocation, EnvironmentConfig envConfig) { this.serialization = Objects.requireNonNull(serialization); this.metrics = Objects.requireNonNull(metrics); @@ -300,9 +300,5 @@ public static final class BerkeleyAddressBookStoreException extends RuntimeExcep public BerkeleyAddressBookStoreException(String message) { super(message); } - - public BerkeleyAddressBookStoreException(String message, Throwable cause) { - super(message, cause); - } } } diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java index 7dfae3c494..602a5cc59a 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java @@ -82,7 +82,7 @@ import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.serialization.Serialization; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; public class NodePersistenceModule extends AbstractModule { @@ -95,7 +95,7 @@ BerkeleySafetyStateStore bdbSafetyStateStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { return new BerkeleySafetyStateStore( serialization, metrics, @@ -109,7 +109,7 @@ BerkeleyAddressBookStore bdbAddressBookStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { return new BerkeleyAddressBookStore( serialization, metrics, diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index 4d20d8049f..47e08b2ceb 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -94,7 +94,6 @@ import com.radixdlt.state.RustStateReader; import com.radixdlt.statecomputer.RustStateComputer; import com.radixdlt.store.NodeStorageLocation; -import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.sync.TransactionsAndProofReader; import com.radixdlt.testutil.TestStateReader; import com.radixdlt.transaction.LedgerSyncLimitsConfig; @@ -102,7 +101,6 @@ import com.radixdlt.transactions.NotarizedTransactionHash; import com.radixdlt.transactions.PreparedNotarizedTransaction; import com.radixdlt.transactions.RawNotarizedTransaction; -import java.io.File; public final class REv2StateManagerModule extends AbstractModule { @@ -209,21 +207,12 @@ public void configure() { install( new AbstractModule() { - @Provides - @Singleton - @StateManagerStorageLocation - DatabaseBackendConfig stateManagerDatabaseBackendConfig( - @StateManagerStorageLocation String nodeStorageLocation) { - return new DatabaseBackendConfig( - new File(nodeStorageLocation, "state_manager").getPath()); - } - @Provides @Singleton @NodeStorageLocation - DatabaseBackendConfig NodeDatabaseBackendConfig( + DatabaseBackendConfig stateManagerDatabaseBackendConfig( @NodeStorageLocation String nodeStorageLocation) { - return new DatabaseBackendConfig(new File(nodeStorageLocation, "node").getPath()); + return new DatabaseBackendConfig(nodeStorageLocation); } @Provides @@ -232,7 +221,6 @@ private NodeRustEnvironment stateManager( MempoolRelayDispatcher mempoolRelayDispatcher, FatalPanicHandler fatalPanicHandler, Network network, - @StateManagerStorageLocation DatabaseBackendConfig stateManagerDatabaseBackendConfig, @NodeStorageLocation DatabaseBackendConfig nodeDatabaseBackendConfig, DatabaseConfig databaseConfig) { return new NodeRustEnvironment( @@ -242,7 +230,7 @@ private NodeRustEnvironment stateManager( NetworkDefinition.from(network), mempoolConfig, vertexLimitsConfigOpt, - stateManagerDatabaseBackendConfig, + nodeDatabaseBackendConfig, databaseConfig, getLoggingConfig(), stateTreeGcConfig, @@ -250,8 +238,7 @@ private NodeRustEnvironment stateManager( ledgerSyncLimitsConfig, protocolConfig, noFees, - scenariosExecutionConfig), - new NodeConfig(nodeDatabaseBackendConfig)); + scenariosExecutionConfig)); } @Provides diff --git a/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java b/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java deleted file mode 100644 index f06ff9c29a..0000000000 --- a/core/src/main/java/com/radixdlt/store/StateManagerStorageLocation.java +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.store; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; -import javax.inject.Qualifier; - -/** Specifies a folder where the state manager can store its files */ -@Qualifier -@Target({FIELD, PARAMETER, METHOD}) -@Retention(RUNTIME) -public @interface StateManagerStorageLocation {} diff --git a/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java b/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java index ebcdec71e0..e24d33893b 100644 --- a/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java +++ b/core/src/main/java/com/radixdlt/store/StorageLocationFromPropertiesModule.java @@ -71,17 +71,10 @@ /** Reads node's storage location from properties */ public final class StorageLocationFromPropertiesModule extends AbstractModule { - @Provides - @Singleton - @StateManagerStorageLocation - private String stateManagerStorageLocation(RuntimeProperties properties) { - return properties.get("db.location", ".//RADIXDB"); - } - @Provides @Singleton @NodeStorageLocation private String nodeStorageLocation(RuntimeProperties properties) { - return properties.get("db.location.node", ".//RADIXDB/node"); + return properties.get("db.location", ".//RADIXDB"); } } diff --git a/core/src/main/resources/default.config b/core/src/main/resources/default.config index ebe4d18072..ea0d8249e4 100755 --- a/core/src/main/resources/default.config +++ b/core/src/main/resources/default.config @@ -143,10 +143,6 @@ cp.port=8080 # Default: ./RADIXDB # db.location=./RADIXDB -# Location to create/find address book and safety store -# Default: ./RADIXDB/node -# db.location.node=./RADIXDB/node - #### ## Debug configuration #### diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index 1d5975bf43..a3b9dbcfc1 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -96,7 +96,7 @@ import com.radixdlt.statecomputer.RandomTransactionGenerator; import com.radixdlt.store.BerkeleyDbDefaults; import com.radixdlt.store.InMemoryCommittedReaderModule; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.sync.SyncRelayConfig; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; @@ -341,7 +341,7 @@ BerkeleySafetyStateStore safetyStateStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { return new BerkeleySafetyStateStore( serialization, metrics, diff --git a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java index 2f4c6211b7..f089841f24 100644 --- a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java @@ -70,7 +70,6 @@ import com.radixdlt.consensus.bft.Self; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; import com.radixdlt.store.NodeStorageLocation; -import com.radixdlt.store.StateManagerStorageLocation; import java.io.File; /** @@ -86,15 +85,8 @@ public PrefixedNodeStorageLocationModule(String baseLocation) { @Provides @Singleton - @StateManagerStorageLocation + @NodeStorageLocation private String stateManagerStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { return new File(baseLocation, publicKey.toHex()).getPath(); } - - @Provides - @Singleton - @NodeStorageLocation - private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { - return new File(new File(baseLocation, publicKey.toHex()), "node").getPath(); - } } diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index f98c382a11..11de83c2e0 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -78,15 +78,12 @@ import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import java.io.IOException; -import java.util.Random; import java.util.function.Consumer; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; public class RocksSafetyStateStoreTest { - private static final Random RANDOM = new Random(); - @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test @@ -126,7 +123,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -151,7 +147,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } } diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index 8b06949d9e..95b1f41a38 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -157,7 +157,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -182,8 +181,7 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } private static AddressBookEntry newAddressBookEntry(int id) { diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index 86330be65b..df9e129191 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -106,7 +106,7 @@ import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.ProtocolState; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.StateManagerStorageLocation; +import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.IOException; import java.util.Objects; @@ -210,7 +210,7 @@ BerkeleySafetyStateStore safetyStateStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { return new BerkeleySafetyStateStore( serialization, metrics, @@ -224,7 +224,7 @@ BerkeleyAddressBookStore bdbAddressBookStore( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @StateManagerStorageLocation String nodeStorageLocation) { + @NodeStorageLocation String nodeStorageLocation) { return new BerkeleyAddressBookStore( serialization, metrics, diff --git a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java index edf7bdef62..9a8581252c 100644 --- a/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java +++ b/core/src/test/java/com/radixdlt/rev2/REv2StateComputerTest.java @@ -98,7 +98,6 @@ import com.radixdlt.statecomputer.commit.ActiveValidatorInfo; import com.radixdlt.statecomputer.commit.LedgerHeader; import com.radixdlt.store.NodeStorageLocation; -import com.radixdlt.store.StateManagerStorageLocation; import com.radixdlt.transaction.LedgerSyncLimitsConfig; import com.radixdlt.transaction.REv2TransactionAndProofStore; import com.radixdlt.transactions.RawNotarizedTransaction; @@ -162,12 +161,9 @@ protected void configure() { .toInstance( new SelfValidatorInfo( ONLY_VALIDATOR_ID.getKey(), Optional.of(ONLY_VALIDATOR_ID))); - bind(String.class) - .annotatedWith(StateManagerStorageLocation.class) - .toInstance(folder.getRoot().getAbsolutePath()); bind(String.class) .annotatedWith(NodeStorageLocation.class) - .toInstance(folder.getRoot().getAbsolutePath() + "/NODE"); + .toInstance(folder.getRoot().getAbsolutePath()); bind(FatalPanicHandler.class).toInstance(() -> {}); } diff --git a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java index b937c49855..28329b210b 100644 --- a/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java +++ b/core/src/test/java/com/radixdlt/rev2/RustMempoolTest.java @@ -306,7 +306,6 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - final var nodeDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); final var config = new StateManagerConfig( @@ -331,8 +330,7 @@ private NodeRustEnvironment createNodeRustEnvironment() throws IOException { // panics as runtime exceptions propagated up the stack (through JNI), which will fail the // test // gracefully anyway. - config, - new NodeConfig(nodeDbConfig)); + config); } private static void initStateComputer(NodeRustEnvironment nodeRustEnvironment) { diff --git a/core/stokenet.config b/core/stokenet.config index 7689e70655..527d3ab41a 100644 --- a/core/stokenet.config +++ b/core/stokenet.config @@ -8,7 +8,6 @@ node.key.path=.//NODEMOUNT/stokenet/key/node.ks node.key.create_if_missing=true db.location=.//NODEMOUNT/stokenet/ledger -db.location.node=.//NODEMOUNT/stokenet/node network.p2p.seed_nodes=radix://node_tdx_2_1qv89yg0la2jt429vqp8sxtpg95hj637gards67gpgqy2vuvwe4s5ss0va2y@13.126.248.88,radix://node_tdx_2_1qvtd9ffdhxyg7meqggr2ezsdfgjre5aqs6jwk5amdhjg86xhurgn5c79t9t@13.210.209.103,radix://node_tdx_2_1qwfh2nn0zx8cut5fqfz6n7pau2f7vdyl89mypldnn4fwlhaeg2tvunp8s8h@54.229.126.97,radix://node_tdx_2_1qwz237kqdpct5l3yjhmna66uxja2ymrf3x6hh528ng3gtvnwndtn5rsrad4@3.210.187.161 From 1d988ad16ee413267d599e4cb2e0cd151d514b20 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 14 Aug 2024 12:43:09 +0200 Subject: [PATCH 16/40] Use rocks db for tests. --- .../src/store/p2p/address_book_components.rs | 15 ++++++-- .../src/store/p2p/column_families.rs | 9 +++-- .../src/store/p2p/typed_cf_api.rs | 36 ------------------- .../modules/FunctionalRadixNodeModule.java | 29 +++++---------- 4 files changed, 25 insertions(+), 64 deletions(-) diff --git a/core-rust/state-manager/src/store/p2p/address_book_components.rs b/core-rust/state-manager/src/store/p2p/address_book_components.rs index 946202bef3..5ad18f0e49 100644 --- a/core-rust/state-manager/src/store/p2p/address_book_components.rs +++ b/core-rust/state-manager/src/store/p2p/address_book_components.rs @@ -17,12 +17,23 @@ impl AddressBookNodeId { } } +/// Timestamp of the various peer-related events +// At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and +// do something like shown below to transparently convert to/from internal representation +// (once there will be real usage at Rust side). +// #[sbor( +// as_type = "i64", +// as_ref = "self.timestamp()", +// from_value = "Self(DateTime::from_timestamp(value, 0))" +// )] +type PeerTimestamp = i64; + /// Peer address entry with all components #[derive(Clone, Sbor)] pub struct PeerAddress { pub encoded_uri: Vec, pub latest_connection_status: Option, - pub last_seen: Option, + pub last_seen: Option, } #[derive(Clone, Copy, Sbor)] @@ -35,6 +46,6 @@ pub enum ConnectionStatus { #[derive(Clone, Sbor)] pub struct AddressBookEntry { pub node_id: AddressBookNodeId, - pub banned_until: Option, + pub banned_until: Option, pub known_addresses: Vec, } diff --git a/core-rust/state-manager/src/store/p2p/column_families.rs b/core-rust/state-manager/src/store/p2p/column_families.rs index 3b02d3fa32..5bc19a50a3 100644 --- a/core-rust/state-manager/src/store/p2p/column_families.rs +++ b/core-rust/state-manager/src/store/p2p/column_families.rs @@ -1,11 +1,10 @@ +use crate::common::typed_cf_api::SborDbCodec; use crate::store::common::typed_cf_api::{ DefaultCf, DirectDbCodec, UnitDbCodec, }; -use crate::store::p2p::typed_cf_api::{ - AddressBookNodeIdDbCodec, MigrationIdDbCodec, MigrationStatusDbCodec, -}; use crate::store::p2p::address_book_components::AddressBookNodeId; use crate::store::p2p::migration::{MigrationId, MigrationStatus}; +use crate::store::p2p::typed_cf_api::AddressBookNodeIdDbCodec; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; @@ -14,8 +13,8 @@ impl DefaultCf for MigrationStatusCf { type Value = MigrationStatus; const DEFAULT_NAME: &'static str = "migration_status"; - type KeyCodec = MigrationIdDbCodec; - type ValueCodec = MigrationStatusDbCodec; + type KeyCodec = SborDbCodec; + type ValueCodec = SborDbCodec; } /// Address book diff --git a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs index 909b970735..8710329403 100644 --- a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs @@ -64,7 +64,6 @@ use crate::p2p::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; use crate::store::common::typed_cf_api::{ DbCodec, BoundedDbCodec}; #[derive(Clone, Default)] @@ -85,38 +84,3 @@ impl BoundedDbCodec for AddressBookNodeIdDbCodec { vec![0xFF; AddressBookNodeId::LENGTH] } } - -/// A [`DbCodec]` capable of representing [`MigrationId]`. -#[derive(Clone, Default)] -pub struct MigrationIdDbCodec; - -impl DbCodec for MigrationIdDbCodec { - fn encode(&self, key: &MigrationId) -> Vec { - vec![*key as u8] - } - - fn decode(&self, bytes: &[u8]) -> MigrationId { - match bytes[0] { - 0 => MigrationId::AddressBook, - 1 => MigrationId::SafetyState, - _ => panic!("Invalid MigrationId"), - } - } -} - -/// A [`DbCodec]` capable of representing [`MigrationStatus]`. -#[derive(Clone, Default)] -pub struct MigrationStatusDbCodec; - -impl DbCodec for MigrationStatusDbCodec { - fn encode(&self, key: &MigrationStatus) -> Vec { - vec![*key as u8] - } - - fn decode(&self, bytes: &[u8]) -> MigrationStatus { - match bytes[0] { - 0 => MigrationStatus::Completed, - _ => panic!("Invalid MigrationId"), - } - } -} diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index a3b9dbcfc1..d4c05ff2e2 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -74,13 +74,10 @@ import com.radixdlt.consensus.bft.*; import com.radixdlt.consensus.epoch.EpochsConsensusModule; import com.radixdlt.consensus.liveness.ProposalGenerator; -import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; import com.radixdlt.consensus.safety.PersistentSafetyStateStore; +import com.radixdlt.consensus.safety.RocksSafetyStateStore; import com.radixdlt.consensus.sync.BFTSyncPatienceMillis; -import com.radixdlt.environment.NoEpochsConsensusModule; -import com.radixdlt.environment.NoEpochsSyncModule; -import com.radixdlt.environment.NodeAutoCloseable; -import com.radixdlt.environment.ScenariosExecutionConfig; +import com.radixdlt.environment.*; import com.radixdlt.genesis.RawGenesisDataWithHash; import com.radixdlt.lang.Option; import com.radixdlt.ledger.MockedLedgerModule; @@ -89,16 +86,13 @@ import com.radixdlt.modules.StateComputerConfig.*; import com.radixdlt.monitoring.Metrics; import com.radixdlt.rev2.modules.*; -import com.radixdlt.serialization.Serialization; +import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.statecomputer.MockedMempoolStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerWithEpochsModule; import com.radixdlt.statecomputer.RandomTransactionGenerator; -import com.radixdlt.store.BerkeleyDbDefaults; import com.radixdlt.store.InMemoryCommittedReaderModule; -import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.sync.SyncRelayConfig; -import com.radixdlt.utils.properties.RuntimeProperties; import java.io.File; import java.time.Duration; import org.junit.rules.TemporaryFolder; @@ -329,24 +323,17 @@ public void configure() { new AbstractModule() { @Override protected void configure() { - bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); + bind(PersistentSafetyStateStore.class).to(RocksSafetyStateStore.class); Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) .addBinding() - .to(BerkeleySafetyStateStore.class); + .to(RocksSafetyStateStore.class); } @Provides @Singleton - BerkeleySafetyStateStore safetyStateStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + RocksDbSafetyStore rocksDbSafetyStore( + Metrics metrics, NodeRustEnvironment environment) { + return RocksDbSafetyStore.create(metrics, environment); } }); } From 328039af2be4a2f697b75401cfb0259adec5533a Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 14 Aug 2024 13:07:41 +0200 Subject: [PATCH 17/40] Move common rocks db code to node-common --- core-rust/Cargo.lock | 3 ++- core-rust/core-api-server/src/core_api/conversions/lts.rs | 2 +- .../core-api-server/src/core_api/conversions/receipt.rs | 2 +- .../src/core_api/handlers/state_consensus_manager.rs | 2 +- .../core-api-server/src/core_api/handlers/stream_proofs.rs | 2 +- .../src/core_api/handlers/stream_transactions.rs | 2 +- core-rust/core-api-server/src/core_api/helpers.rs | 2 +- core-rust/node-common/Cargo.toml | 2 ++ core-rust/node-common/src/lib.rs | 2 ++ .../src/store/common => node-common/src/store}/mod.rs | 0 .../src/store/common => node-common/src/store}/rocks_db.rs | 0 .../store/common => node-common/src/store}/typed_cf_api.rs | 2 +- core-rust/state-manager/Cargo.toml | 2 +- core-rust/state-manager/src/store/codecs.rs | 2 +- .../state-manager/src/store/consensus/column_families.rs | 2 +- core-rust/state-manager/src/store/consensus/rocks_db.rs | 6 +++--- core-rust/state-manager/src/store/historical_state.rs | 2 +- core-rust/state-manager/src/store/mod.rs | 1 - core-rust/state-manager/src/store/p2p/column_families.rs | 4 ++-- core-rust/state-manager/src/store/p2p/rocks_db.rs | 6 +++--- core-rust/state-manager/src/store/p2p/typed_cf_api.rs | 2 +- core-rust/state-manager/src/store/proofs_gc.rs | 2 +- 22 files changed, 27 insertions(+), 23 deletions(-) rename core-rust/{state-manager/src/store/common => node-common/src/store}/mod.rs (100%) rename core-rust/{state-manager/src/store/common => node-common/src/store}/rocks_db.rs (100%) rename core-rust/{state-manager/src/store/common => node-common/src/store}/typed_cf_api.rs (99%) diff --git a/core-rust/Cargo.lock b/core-rust/Cargo.lock index 5a34f9076b..31dba83529 100644 --- a/core-rust/Cargo.lock +++ b/core-rust/Cargo.lock @@ -1299,6 +1299,7 @@ version = "0.1.0" dependencies = [ "bech32", "blake2", + "itertools", "jni", "opentelemetry", "opentelemetry-jaeger", @@ -1312,6 +1313,7 @@ dependencies = [ "radix-substate-store-interface", "radix-substate-store-queries", "radix-transactions", + "rocksdb", "sbor", "tokio", "tokio-util", @@ -2395,7 +2397,6 @@ dependencies = [ "radix-transaction-scenarios", "radix-transactions", "rand 0.8.5", - "rocksdb", "sbor", "slotmap", "tempfile", diff --git a/core-rust/core-api-server/src/core_api/conversions/lts.rs b/core-rust/core-api-server/src/core_api/conversions/lts.rs index e6a10d47ca..91abe1271a 100644 --- a/core-rust/core-api-server/src/core_api/conversions/lts.rs +++ b/core-rust/core-api-server/src/core_api/conversions/lts.rs @@ -1,5 +1,5 @@ use crate::engine_prelude::*; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::store::consensus::traits::SubstateNodeAncestryStore; use state_manager::{ diff --git a/core-rust/core-api-server/src/core_api/conversions/receipt.rs b/core-rust/core-api-server/src/core_api/conversions/receipt.rs index df8c552064..1a2eb6433c 100644 --- a/core-rust/core-api-server/src/core_api/conversions/receipt.rs +++ b/core-rust/core-api-server/src/core_api/conversions/receipt.rs @@ -4,7 +4,7 @@ use super::addressing::*; use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::{ ApplicationEvent, BySubstate, DetailedTransactionOutcome, LedgerStateChanges, diff --git a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs index d1cea86e0b..890806c7d6 100644 --- a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs +++ b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs @@ -2,7 +2,7 @@ use crate::core_api::*; use crate::engine_prelude::*; use state_manager::protocol::ProtocolVersionName; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use std::ops::Deref; diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs index bab6f6d4da..c250830759 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs @@ -1,7 +1,7 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::store::consensus::traits::*; use state_manager::{LedgerProof, LedgerProofOrigin, StateVersion}; diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs index 08b84b8e2b..c966d8054b 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs @@ -5,7 +5,7 @@ use std::ops::Deref; use crate::core_api::*; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::store::consensus::traits::*; use state_manager::transaction::*; diff --git a/core-rust/core-api-server/src/core_api/helpers.rs b/core-rust/core-api-server/src/core_api/helpers.rs index c4dadc76f9..3a8139a739 100644 --- a/core-rust/core-api-server/src/core_api/helpers.rs +++ b/core-rust/core-api-server/src/core_api/helpers.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use serde::Serialize; -use state_manager::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use state_manager::consensus::rocks_db::StateManagerDatabase; use state_manager::store::consensus::traits::*; use state_manager::LedgerHeader; diff --git a/core-rust/node-common/Cargo.toml b/core-rust/node-common/Cargo.toml index 428b7c2087..d026182468 100644 --- a/core-rust/node-common/Cargo.toml +++ b/core-rust/node-common/Cargo.toml @@ -15,6 +15,7 @@ radix-substate-store-queries = { workspace = true } radix-rust = { workspace = true } # Non-Radix Engine Dependencies: +itertools = { workspace = true } jni = { workspace = true } tracing = { workspace = true } tokio = { workspace = true } @@ -28,3 +29,4 @@ tracing-opentelemetry = { version = "=0.18.0" } tracing-subscriber = { version = "=0.3.17" } opentelemetry = { version = "=0.18.0", default-features = false, features = ["rt-tokio", "trace"] } opentelemetry-jaeger = { version = "=0.17.0", features = ["rt-tokio"] } +rocksdb = { version = "=0.21.0" } diff --git a/core-rust/node-common/src/lib.rs b/core-rust/node-common/src/lib.rs index 1e782f4db0..3a408dcfae 100644 --- a/core-rust/node-common/src/lib.rs +++ b/core-rust/node-common/src/lib.rs @@ -71,6 +71,8 @@ pub mod locks; pub mod metrics; pub mod scheduler; pub mod utils; +pub mod store; +pub use rocksdb; pub(crate) mod engine_prelude { pub use radix_common::prelude::*; diff --git a/core-rust/state-manager/src/store/common/mod.rs b/core-rust/node-common/src/store/mod.rs similarity index 100% rename from core-rust/state-manager/src/store/common/mod.rs rename to core-rust/node-common/src/store/mod.rs diff --git a/core-rust/state-manager/src/store/common/rocks_db.rs b/core-rust/node-common/src/store/rocks_db.rs similarity index 100% rename from core-rust/state-manager/src/store/common/rocks_db.rs rename to core-rust/node-common/src/store/rocks_db.rs diff --git a/core-rust/state-manager/src/store/common/typed_cf_api.rs b/core-rust/node-common/src/store/typed_cf_api.rs similarity index 99% rename from core-rust/state-manager/src/store/common/typed_cf_api.rs rename to core-rust/node-common/src/store/typed_cf_api.rs index 4ffdd6d79c..7bf537418f 100644 --- a/core-rust/state-manager/src/store/common/typed_cf_api.rs +++ b/core-rust/node-common/src/store/typed_cf_api.rs @@ -63,7 +63,7 @@ */ use crate::engine_prelude::*; -use crate::store::common::rocks_db::{ReadableRocks, WriteableRocks}; +use crate::store::rocks_db::{ReadableRocks, WriteableRocks}; use itertools::Itertools; use rocksdb::{ColumnFamily, Direction, IteratorMode, WriteBatch}; use std::ops::Range; diff --git a/core-rust/state-manager/Cargo.toml b/core-rust/state-manager/Cargo.toml index e6964692cf..da15a8d934 100644 --- a/core-rust/state-manager/Cargo.toml +++ b/core-rust/state-manager/Cargo.toml @@ -27,7 +27,7 @@ blake2 = { workspace = true } hex = { workspace = true } rand = { workspace = true } -rocksdb = { version = "=0.21.0" } +#rocksdb = { version = "=0.21.0" } lru = { version = "=0.8.1", default-features = false } slotmap = { version = "=1.0.6" } im = { version = "=15.1.0" } diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index 10aa6e9745..ceadf978ea 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -69,7 +69,7 @@ use crate::engine_prelude::*; use crate::store::consensus::traits::indices::CreationId; use crate::store::consensus::traits::scenario::ScenarioSequenceNumber; -use crate::store::common::typed_cf_api::*; +use node_common::store::typed_cf_api::*; use crate::transaction::RawLedgerTransaction; use crate::StateVersion; diff --git a/core-rust/state-manager/src/store/consensus/column_families.rs b/core-rust/state-manager/src/store/consensus/column_families.rs index 72a9524eb8..07c78ed5ed 100644 --- a/core-rust/state-manager/src/store/consensus/column_families.rs +++ b/core-rust/state-manager/src/store/consensus/column_families.rs @@ -32,7 +32,7 @@ use crate::store::codecs::{ StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, TypeAndCreationIndexKeyDbCodec, }; -use crate::store::common::typed_cf_api::{ +use node_common::store::typed_cf_api::{ DefaultCf, DirectDbCodec, PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, }; diff --git a/core-rust/state-manager/src/store/consensus/rocks_db.rs b/core-rust/state-manager/src/store/consensus/rocks_db.rs index 347beb7555..2d2923ba73 100644 --- a/core-rust/state-manager/src/store/consensus/rocks_db.rs +++ b/core-rust/state-manager/src/store/consensus/rocks_db.rs @@ -65,7 +65,7 @@ use std::collections::HashSet; use std::path::PathBuf; -use rocksdb::{ +use node_common::rocksdb::{ ColumnFamilyDescriptor, DB, Direction, IteratorMode, Options, }; use tracing::{error, info, warn}; @@ -82,7 +82,7 @@ use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; use crate::consensus::column_families::*; use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; -use crate::store::common::rocks_db::*; +use node_common::store::rocks_db::*; use crate::store::consensus::traits::*; use crate::store::consensus::traits::extensions::*; use crate::store::consensus::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; @@ -94,7 +94,7 @@ use crate::store::consensus::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; use crate::store::historical_state::StateTreeBasedSubstateDatabase; -use crate::store::common::typed_cf_api::*; +use node_common::store::typed_cf_api::*; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; diff --git a/core-rust/state-manager/src/store/historical_state.rs b/core-rust/state-manager/src/store/historical_state.rs index bf42966790..31b12a4501 100644 --- a/core-rust/state-manager/src/store/historical_state.rs +++ b/core-rust/state-manager/src/store/historical_state.rs @@ -67,7 +67,7 @@ use std::ops::Deref; use crate::engine_prelude::entity_tier::EntityTier; use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; -use crate::store::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use crate::store::consensus::rocks_db::StateManagerDatabase; use crate::store::consensus::traits::*; use crate::consensus::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index c9d26dea1d..6d1cdd61d6 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -80,7 +80,6 @@ pub mod proofs_gc; mod codecs; pub mod p2p; pub mod consensus; -pub mod common; #[derive(Debug, Categorize, Encode, Decode, Clone)] pub struct DatabaseBackendConfig { diff --git a/core-rust/state-manager/src/store/p2p/column_families.rs b/core-rust/state-manager/src/store/p2p/column_families.rs index 5bc19a50a3..448bddacb7 100644 --- a/core-rust/state-manager/src/store/p2p/column_families.rs +++ b/core-rust/state-manager/src/store/p2p/column_families.rs @@ -1,5 +1,5 @@ -use crate::common::typed_cf_api::SborDbCodec; -use crate::store::common::typed_cf_api::{ +use node_common::store::typed_cf_api::SborDbCodec; +use node_common::store::typed_cf_api::{ DefaultCf, DirectDbCodec, UnitDbCodec, }; use crate::store::p2p::address_book_components::AddressBookNodeId; diff --git a/core-rust/state-manager/src/store/p2p/rocks_db.rs b/core-rust/state-manager/src/store/p2p/rocks_db.rs index 596a58f247..9686a3cc84 100644 --- a/core-rust/state-manager/src/store/p2p/rocks_db.rs +++ b/core-rust/state-manager/src/store/p2p/rocks_db.rs @@ -64,7 +64,7 @@ use std::path::PathBuf; -use rocksdb::{ +use node_common::rocksdb::{ ColumnFamilyDescriptor, DB, Options, }; @@ -72,9 +72,9 @@ use crate::engine_prelude::*; use crate::p2p::address_book_components::AddressBookNodeId; use crate::p2p::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; -use crate::store::common::rocks_db::*; +use node_common::store::rocks_db::*; use crate::store::p2p::migration::{MigrationId, MigrationStatus}; -use crate::store::common::typed_cf_api::*; +use node_common::store::typed_cf_api::*; /// A listing of all column family names used by the Node. /// diff --git a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs index 8710329403..709a139cd8 100644 --- a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs +++ b/core-rust/state-manager/src/store/p2p/typed_cf_api.rs @@ -64,7 +64,7 @@ use crate::p2p::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; -use crate::store::common::typed_cf_api::{ DbCodec, BoundedDbCodec}; +use node_common::store::typed_cf_api::{DbCodec, BoundedDbCodec}; #[derive(Clone, Default)] pub struct AddressBookNodeIdDbCodec {} diff --git a/core-rust/state-manager/src/store/proofs_gc.rs b/core-rust/state-manager/src/store/proofs_gc.rs index 21f5b71a4e..4942f13aaf 100644 --- a/core-rust/state-manager/src/store/proofs_gc.rs +++ b/core-rust/state-manager/src/store/proofs_gc.rs @@ -76,7 +76,7 @@ use crate::store::consensus::traits::proofs::QueryableProofStore; use crate::jni::LedgerSyncLimitsConfig; use crate::store::consensus::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; -use crate::store::common::rocks_db::ReadableRocks; +use node_common::store::rocks_db::ReadableRocks; use crate::consensus::traits::GetSyncableTxnsAndProofError::{ FailedToPrepareAResponseWithinLimits, NothingToServeAtTheGivenStateVersion, RefusedToServeGenesis, RefusedToServeProtocolUpdate, From 5fe0fcd31942f6e6ae71d592c322d2ba304beacc Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 14 Aug 2024 13:18:43 +0200 Subject: [PATCH 18/40] Move state-manager store files back to store module. --- .../src/core_api/conversions/lts.rs | 4 ++-- .../src/core_api/conversions/numerics.rs | 2 +- .../src/core_api/conversions/receipt.rs | 2 +- .../lts/stream_account_transaction_outcomes.rs | 2 +- .../handlers/lts/stream_transaction_outcomes.rs | 2 +- .../core_api/handlers/lts/transaction_status.rs | 2 +- .../core_api/handlers/state_consensus_manager.rs | 2 +- .../src/core_api/handlers/status_network_status.rs | 2 +- .../src/core_api/handlers/status_scenarios.rs | 2 +- .../src/core_api/handlers/stream_proofs.rs | 4 ++-- .../src/core_api/handlers/stream_transactions.rs | 4 ++-- .../src/core_api/handlers/transaction_preview.rs | 2 +- .../src/core_api/handlers/transaction_receipt.rs | 2 +- .../src/core_api/handlers/transaction_status.rs | 2 +- core-rust/core-api-server/src/core_api/helpers.rs | 4 ++-- .../src/engine_state_api/extras.rs | 2 +- .../src/engine_state_api/handlers/entity_info.rs | 2 +- .../handlers/extra_entity_search.rs | 4 ++-- core-rust/state-manager/src/commit_bundle.rs | 2 +- core-rust/state-manager/src/committer.rs | 4 ++-- .../state-manager/src/jni/node_rust_environment.rs | 2 +- .../state-manager/src/jni/test_state_reader.rs | 4 ++-- .../state-manager/src/jni/transaction_store.rs | 2 +- .../state-manager/src/jni/vertex_store_recovery.rs | 2 +- core-rust/state-manager/src/metrics.rs | 4 ++-- .../state-manager/src/protocol/protocol_state.rs | 4 ++-- .../definitions/anemone_definition.rs | 2 +- .../definitions/bottlenose_definition.rs | 2 +- .../definitions/custom_definition.rs | 2 +- .../definitions/default_definition.rs | 2 +- .../protocol/protocol_updates/definitions/mod.rs | 2 +- .../definitions/test_definition.rs | 2 +- .../protocol_updates/protocol_update_definition.rs | 2 +- .../protocol_updates/protocol_update_progress.rs | 2 +- core-rust/state-manager/src/protocol/test.rs | 2 +- core-rust/state-manager/src/staging/cache.rs | 6 +++--- core-rust/state-manager/src/staging/mod.rs | 4 ++-- .../src/staging/node_ancestry_resolver.rs | 2 +- core-rust/state-manager/src/staging/overlays.rs | 2 +- core-rust/state-manager/src/staging/result.rs | 4 ++-- core-rust/state-manager/src/state_manager.rs | 6 +++--- core-rust/state-manager/src/store/codecs.rs | 4 ++-- .../src/store/{consensus => }/column_families.rs | 8 ++++---- core-rust/state-manager/src/store/consensus/mod.rs | 3 --- .../state-manager/src/store/historical_state.rs | 6 +++--- core-rust/state-manager/src/store/jmt_gc.rs | 8 ++++---- core-rust/state-manager/src/store/mod.rs | 10 ++++++---- core-rust/state-manager/src/store/proofs_gc.rs | 12 ++++++------ .../src/store/{consensus => }/rocks_db.rs | 14 +++++++------- .../src/store/{consensus => }/traits.rs | 2 +- core-rust/state-manager/src/system_executor.rs | 8 ++++---- core-rust/state-manager/src/test/mod.rs | 2 +- .../state-manager/src/transaction/preparation.rs | 2 +- core-rust/state-manager/src/transaction/preview.rs | 2 +- .../src/transaction/series_execution.rs | 2 +- .../state-manager/src/transaction/validation.rs | 6 +++--- 56 files changed, 100 insertions(+), 101 deletions(-) rename core-rust/state-manager/src/store/{consensus => }/column_families.rs (98%) delete mode 100644 core-rust/state-manager/src/store/consensus/mod.rs rename core-rust/state-manager/src/store/{consensus => }/rocks_db.rs (99%) rename core-rust/state-manager/src/store/{consensus => }/traits.rs (99%) diff --git a/core-rust/core-api-server/src/core_api/conversions/lts.rs b/core-rust/core-api-server/src/core_api/conversions/lts.rs index 91abe1271a..8d3bae4244 100644 --- a/core-rust/core-api-server/src/core_api/conversions/lts.rs +++ b/core-rust/core-api-server/src/core_api/conversions/lts.rs @@ -1,7 +1,7 @@ use crate::engine_prelude::*; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; -use state_manager::store::consensus::traits::SubstateNodeAncestryStore; +use state_manager::store::rocks_db::StateManagerDatabase; +use state_manager::store::traits::SubstateNodeAncestryStore; use state_manager::{ CommittedTransactionIdentifiers, LedgerTransactionOutcome, LocalTransactionReceipt, StateVersion, TransactionTreeHash, diff --git a/core-rust/core-api-server/src/core_api/conversions/numerics.rs b/core-rust/core-api-server/src/core_api/conversions/numerics.rs index 45d55c4f19..575da7754c 100644 --- a/core-rust/core-api-server/src/core_api/conversions/numerics.rs +++ b/core-rust/core-api-server/src/core_api/conversions/numerics.rs @@ -3,7 +3,7 @@ use std::any::type_name; use std::ops::RangeInclusive; use crate::engine_prelude::*; -use state_manager::store::consensus::traits::scenario::ScenarioSequenceNumber; +use state_manager::store::traits::scenario::ScenarioSequenceNumber; use state_manager::StateVersion; use crate::core_api::models; diff --git a/core-rust/core-api-server/src/core_api/conversions/receipt.rs b/core-rust/core-api-server/src/core_api/conversions/receipt.rs index 1a2eb6433c..9703b26bee 100644 --- a/core-rust/core-api-server/src/core_api/conversions/receipt.rs +++ b/core-rust/core-api-server/src/core_api/conversions/receipt.rs @@ -5,7 +5,7 @@ use crate::core_api::*; use crate::engine_prelude::*; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::rocks_db::StateManagerDatabase; use state_manager::{ ApplicationEvent, BySubstate, DetailedTransactionOutcome, LedgerStateChanges, LocalTransactionReceipt, PartitionChangeAction, PartitionReference, SubstateChangeAction, diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs b/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs index aff88c7eb2..6dbd3eefe8 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/stream_account_transaction_outcomes.rs @@ -1,5 +1,5 @@ use crate::core_api::*; -use state_manager::store::consensus::traits::{ +use state_manager::store::traits::{ extensions::IterableAccountChangeIndex, ConfigurableDatabase, QueryableProofStore, QueryableTransactionStore, }; diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs b/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs index c6f8d7f713..8a8f717d73 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/stream_transaction_outcomes.rs @@ -1,5 +1,5 @@ use crate::core_api::*; -use state_manager::store::consensus::traits::{ +use state_manager::store::traits::{ CommittedTransactionBundle, ConfigurableDatabase, IterableTransactionStore, QueryableProofStore, }; use std::ops::Deref; diff --git a/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs b/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs index 41e618944a..b24e70e848 100644 --- a/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/lts/transaction_status.rs @@ -9,7 +9,7 @@ use state_manager::{ use state_manager::mempool::pending_transaction_result_cache::PendingTransactionRecord; use state_manager::query::StateManagerSubstateQueries; -use state_manager::store::consensus::traits::*; +use state_manager::store::traits::*; #[tracing::instrument(skip(state))] pub(crate) async fn handle_lts_transaction_status( diff --git a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs index 890806c7d6..28b6806ac4 100644 --- a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs +++ b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs @@ -3,7 +3,7 @@ use crate::engine_prelude::*; use state_manager::protocol::ProtocolVersionName; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; +use state_manager::store::rocks_db::StateManagerDatabase; use std::ops::Deref; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs b/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs index 8f18e9a760..418f118713 100644 --- a/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/status_network_status.rs @@ -3,7 +3,7 @@ use crate::core_api::*; use crate::engine_prelude::*; use state_manager::query::TransactionIdentifierLoader; -use state_manager::store::consensus::traits::*; +use state_manager::store::traits::*; use state_manager::{LedgerHashes, LedgerProof, LedgerStateSummary, StateVersion}; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs b/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs index bcf7403ada..7227fabcd9 100644 --- a/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs +++ b/core-rust/core-api-server/src/core_api/handlers/status_scenarios.rs @@ -2,7 +2,7 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::store::consensus::traits::scenario::{ +use state_manager::store::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ExecutedScenarioTransaction, ScenarioSequenceNumber, }; diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs index c250830759..a3565061b3 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_proofs.rs @@ -2,8 +2,8 @@ use crate::core_api::*; use crate::engine_prelude::*; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; -use state_manager::store::consensus::traits::*; +use state_manager::store::rocks_db::StateManagerDatabase; +use state_manager::store::traits::*; use state_manager::{LedgerProof, LedgerProofOrigin, StateVersion}; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs index c966d8054b..8a5caabefa 100644 --- a/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs +++ b/core-rust/core-api-server/src/core_api/handlers/stream_transactions.rs @@ -6,8 +6,8 @@ use std::ops::Deref; use crate::core_api::*; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; -use state_manager::store::consensus::traits::*; +use state_manager::store::rocks_db::StateManagerDatabase; +use state_manager::store::traits::*; use state_manager::transaction::*; use state_manager::{ CommittedTransactionIdentifiers, LedgerHeader, LedgerProof, LedgerProofOrigin, diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs index bbde1907ad..0d8608ad7c 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_preview.rs @@ -1,7 +1,7 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::store::consensus::rocks_db::ActualStateManagerDatabase; +use state_manager::store::rocks_db::ActualStateManagerDatabase; use state_manager::transaction::ProcessedPreviewResult; use state_manager::{ExecutionFeeData, LocalTransactionReceipt, PreviewRequest}; diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs index 8c9c8a313d..8ded896fb1 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_receipt.rs @@ -1,7 +1,7 @@ use crate::core_api::handlers::to_api_committed_transaction; use crate::core_api::*; -use state_manager::store::consensus::traits::*; +use state_manager::store::traits::*; use state_manager::transaction::*; #[tracing::instrument(skip(state))] diff --git a/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs b/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs index df2abed1e1..75e1276df7 100644 --- a/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs +++ b/core-rust/core-api-server/src/core_api/handlers/transaction_status.rs @@ -9,7 +9,7 @@ use state_manager::{ use state_manager::mempool::pending_transaction_result_cache::PendingTransactionRecord; use state_manager::query::StateManagerSubstateQueries; -use state_manager::store::consensus::traits::*; +use state_manager::store::traits::*; #[tracing::instrument(skip(state))] pub(crate) async fn handle_transaction_status( diff --git a/core-rust/core-api-server/src/core_api/helpers.rs b/core-rust/core-api-server/src/core_api/helpers.rs index 3a8139a739..aba7c23884 100644 --- a/core-rust/core-api-server/src/core_api/helpers.rs +++ b/core-rust/core-api-server/src/core_api/helpers.rs @@ -1,8 +1,8 @@ use crate::engine_prelude::*; use serde::Serialize; use node_common::store::rocks_db::ReadableRocks; -use state_manager::consensus::rocks_db::StateManagerDatabase; -use state_manager::store::consensus::traits::*; +use state_manager::store::rocks_db::StateManagerDatabase; +use state_manager::store::traits::*; use state_manager::LedgerHeader; use std::io::Write; diff --git a/core-rust/engine-state-api-server/src/engine_state_api/extras.rs b/core-rust/engine-state-api-server/src/engine_state_api/extras.rs index a6bdfdacc5..142932b351 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/extras.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/extras.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use itertools::Itertools; -use state_manager::store::consensus::traits::indices::{ +use state_manager::store::traits::indices::{ CreationId, EntityBlueprintId, EntityBlueprintIdV1, EntityListingIndex, }; diff --git a/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs b/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs index 73fd9c2937..6d403ae84f 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/handlers/entity_info.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::engine_state_api::factories::EngineStateLoaderFactory; use crate::engine_state_api::*; use state_manager::historical_state::VersionScopingSupport; -use state_manager::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use state_manager::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; pub(crate) async fn handle_entity_info( state: State, diff --git a/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs b/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs index e031c4d43c..7b1208f4d6 100644 --- a/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs +++ b/core-rust/engine-state-api-server/src/engine_state_api/handlers/extra_entity_search.rs @@ -3,8 +3,8 @@ use crate::engine_state_api::*; use crate::engine_prelude::*; use state_manager::historical_state::VersionScopingSupport; -use state_manager::store::consensus::traits::indices::CreationId; -use state_manager::store::consensus::traits::ConfigurableDatabase; +use state_manager::store::traits::indices::CreationId; +use state_manager::store::traits::ConfigurableDatabase; use crate::engine_state_api::extras::{EngineEntityLister, EntitySummary}; use state_manager::StateVersion; diff --git a/core-rust/state-manager/src/commit_bundle.rs b/core-rust/state-manager/src/commit_bundle.rs index 9664773e30..9a873a6272 100644 --- a/core-rust/state-manager/src/commit_bundle.rs +++ b/core-rust/state-manager/src/commit_bundle.rs @@ -64,7 +64,7 @@ use crate::staging::epoch_handling::EpochAwareAccuTreeFactory; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::transaction::*; use crate::*; diff --git a/core-rust/state-manager/src/committer.rs b/core-rust/state-manager/src/committer.rs index eefb55e6bc..3b5890533f 100644 --- a/core-rust/state-manager/src/committer.rs +++ b/core-rust/state-manager/src/committer.rs @@ -64,7 +64,7 @@ use crate::mempool_manager::MempoolManager; use crate::staging::epoch_handling::EpochAwareAccuTreeFactory; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::transaction::*; use crate::types::CommitRequest; use crate::*; @@ -80,7 +80,7 @@ use crate::system_commits::*; use crate::accumulator_tree::storage::ReadableAccuTreeStore; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use std::ops::Deref; use std::sync::Arc; use std::time::SystemTime; diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index bb9960d0dd..52e622826d 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -87,7 +87,7 @@ use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; use crate::store::p2p::rocks_db::ActualAddressBookDatabase; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::transaction::Preparator; use crate::{Committer, DatabaseBackendConfig, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; diff --git a/core-rust/state-manager/src/jni/test_state_reader.rs b/core-rust/state-manager/src/jni/test_state_reader.rs index b0f987212a..08f73a3ab3 100644 --- a/core-rust/state-manager/src/jni/test_state_reader.rs +++ b/core-rust/state-manager/src/jni/test_state_reader.rs @@ -73,11 +73,11 @@ use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::query::StateManagerSubstateQueries; use node_common::java::*; -use crate::store::consensus::traits::{ +use crate::store::traits::{ gc::StateTreeGcStore, IterableProofStore, QueryableProofStore, QueryableTransactionStore, SubstateNodeAncestryStore, }; -use crate::consensus::traits::measurement::MeasurableDatabase; +use crate::store::traits::measurement::MeasurableDatabase; use crate::transaction::LedgerTransactionHash; // diff --git a/core-rust/state-manager/src/jni/transaction_store.rs b/core-rust/state-manager/src/jni/transaction_store.rs index 219b2eb9d3..1561380fbc 100644 --- a/core-rust/state-manager/src/jni/transaction_store.rs +++ b/core-rust/state-manager/src/jni/transaction_store.rs @@ -66,7 +66,7 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::jni::LedgerSyncLimitsConfig; use crate::protocol::epoch_change_iter; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::{LedgerProof, StateVersion}; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; diff --git a/core-rust/state-manager/src/jni/vertex_store_recovery.rs b/core-rust/state-manager/src/jni/vertex_store_recovery.rs index b0f411519b..a5b8ca0a47 100644 --- a/core-rust/state-manager/src/jni/vertex_store_recovery.rs +++ b/core-rust/state-manager/src/jni/vertex_store_recovery.rs @@ -63,7 +63,7 @@ */ use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::store::consensus::traits::{RecoverableVertexStore, VertexStoreBlobV1, WriteableVertexStore}; +use crate::store::traits::{RecoverableVertexStore, VertexStoreBlobV1, WriteableVertexStore}; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; diff --git a/core-rust/state-manager/src/metrics.rs b/core-rust/state-manager/src/metrics.rs index 445170bc7b..f1cdbc5e35 100644 --- a/core-rust/state-manager/src/metrics.rs +++ b/core-rust/state-manager/src/metrics.rs @@ -80,8 +80,8 @@ use crate::engine_prelude::*; use crate::protocol::{ PendingProtocolUpdateState, ProtocolState, ProtocolUpdateEnactmentCondition, }; -use crate::store::consensus::traits::measurement::CategoryDbVolumeStatistic; -use crate::consensus::traits::QueryableProofStore; +use crate::store::traits::measurement::CategoryDbVolumeStatistic; +use crate::store::traits::QueryableProofStore; pub struct LedgerMetrics { address_encoder: AddressBech32Encoder, // for label rendering only diff --git a/core-rust/state-manager/src/protocol/protocol_state.rs b/core-rust/state-manager/src/protocol/protocol_state.rs index 2326abc08f..978e44728e 100644 --- a/core-rust/state-manager/src/protocol/protocol_state.rs +++ b/core-rust/state-manager/src/protocol/protocol_state.rs @@ -8,9 +8,9 @@ use std::sync::Arc; use tracing::info; use crate::protocol::*; -use crate::consensus::traits::{IterableProofStore, QueryableProofStore, QueryableTransactionStore}; +use crate::store::traits::{IterableProofStore, QueryableProofStore, QueryableTransactionStore}; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::{ LocalTransactionReceipt, ProtocolMetrics, ScenariosExecutionConfig, StateVersion, SystemExecutor, diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs index de4a89d3ca..9b36eb988b 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/anemone_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs index 301c966332..48530042a7 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/bottlenose_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs index 4aa56f0341..8a7fbff9d9 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/custom_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs index c9a05600a3..8491f2fe7e 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/default_definition.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs index b8d5d0f159..8913985b9f 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/mod.rs @@ -12,7 +12,7 @@ pub use test_definition::*; use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::transaction::*; use node_common::locks::DbLock; use std::ops::Deref; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs index ac0d12e45d..c7e9c2432b 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/definitions/test_definition.rs @@ -1,5 +1,5 @@ use crate::engine_prelude::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::{protocol::*, transaction::FlashTransactionV1}; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs index 8b2d8a1d6a..5e14d7bbb5 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_definition.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use node_common::locks::DbLock; use std::sync::Arc; diff --git a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs index 18b84dd3e4..a9c9f1f8c4 100644 --- a/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs +++ b/core-rust/state-manager/src/protocol/protocol_updates/protocol_update_progress.rs @@ -2,7 +2,7 @@ use crate::engine_prelude::*; use crate::protocol::*; -use crate::consensus::traits::*; +use crate::store::traits::*; use crate::LedgerProofOrigin; diff --git a/core-rust/state-manager/src/protocol/test.rs b/core-rust/state-manager/src/protocol/test.rs index ddfec4df5f..a7a9f8561d 100644 --- a/core-rust/state-manager/src/protocol/test.rs +++ b/core-rust/state-manager/src/protocol/test.rs @@ -63,7 +63,7 @@ */ use crate::engine_prelude::*; -use crate::store::consensus::traits::QueryableProofStore; +use crate::store::traits::QueryableProofStore; use crate::protocol::*; use crate::{LedgerProof, LedgerProofOrigin, StateManagerConfig}; diff --git a/core-rust/state-manager/src/staging/cache.rs b/core-rust/state-manager/src/staging/cache.rs index 294db4541d..8ee25820f9 100644 --- a/core-rust/state-manager/src/staging/cache.rs +++ b/core-rust/state-manager/src/staging/cache.rs @@ -86,9 +86,9 @@ use crate::{ use im::hashmap::HashMap as ImmutableHashMap; use itertools::Itertools; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; -use crate::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; -use crate::store::consensus::traits::{ConfigurableDatabase, QueryableProofStore}; +use crate::store::rocks_db::ActualStateManagerDatabase; +use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::store::traits::{ConfigurableDatabase, QueryableProofStore}; use node_common::locks::{DbLock, LockFactory, Mutex}; use slotmap::SecondaryMap; diff --git a/core-rust/state-manager/src/staging/mod.rs b/core-rust/state-manager/src/staging/mod.rs index 487c5c049f..eec4594d32 100644 --- a/core-rust/state-manager/src/staging/mod.rs +++ b/core-rust/state-manager/src/staging/mod.rs @@ -73,8 +73,8 @@ use crate::accumulator_tree::storage::ReadableAccuTreeStore; use crate::engine_prelude::*; use crate::{ReceiptTreeHash, StateVersion, TransactionTreeHash}; -use crate::store::consensus::traits::SubstateNodeAncestryStore; -use crate::consensus::traits::ConfigurableDatabase; +use crate::store::traits::SubstateNodeAncestryStore; +use crate::store::traits::ConfigurableDatabase; pub use cache::*; pub use result::*; diff --git a/core-rust/state-manager/src/staging/node_ancestry_resolver.rs b/core-rust/state-manager/src/staging/node_ancestry_resolver.rs index 00c8ec3baa..763fd8d115 100644 --- a/core-rust/state-manager/src/staging/node_ancestry_resolver.rs +++ b/core-rust/state-manager/src/staging/node_ancestry_resolver.rs @@ -63,7 +63,7 @@ */ use crate::engine_prelude::*; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::{SubstateChangeAction, SubstateReference}; use std::borrow::Borrow; use std::collections::hash_map::Entry; diff --git a/core-rust/state-manager/src/staging/overlays.rs b/core-rust/state-manager/src/staging/overlays.rs index c1fc1f2d93..ac54cbdc0e 100644 --- a/core-rust/state-manager/src/staging/overlays.rs +++ b/core-rust/state-manager/src/staging/overlays.rs @@ -3,7 +3,7 @@ use std::cmp::Ordering; use std::hash::Hash; use std::iter::Peekable; -use crate::store::consensus::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; pub struct SubstateOverlayIterator<'a> { root_db: Peekable + 'a>>, diff --git a/core-rust/state-manager/src/staging/result.rs b/core-rust/state-manager/src/staging/result.rs index db15c58703..9ced5cf51c 100644 --- a/core-rust/state-manager/src/staging/result.rs +++ b/core-rust/state-manager/src/staging/result.rs @@ -79,8 +79,8 @@ use crate::staging::ReadableStore; use crate::staging::node_ancestry_resolver::NodeAncestryResolver; use crate::staging::overlays::{MapSubstateNodeAncestryStore, StagedSubstateNodeAncestryStore}; -use crate::store::consensus::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; -use crate::consensus::traits::{ConfigurableDatabase, LeafSubstateKeyAssociation}; +use crate::store::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; +use crate::store::traits::{ConfigurableDatabase, LeafSubstateKeyAssociation}; use node_common::utils::IsAccountExt; pub enum ProcessedTransactionReceipt { diff --git a/core-rust/state-manager/src/state_manager.rs b/core-rust/state-manager/src/state_manager.rs index 0a400034dc..6dbbc8bac6 100644 --- a/core-rust/state-manager/src/state_manager.rs +++ b/core-rust/state-manager/src/state_manager.rs @@ -72,11 +72,11 @@ use crate::jni::LedgerSyncLimitsConfig; use crate::protocol::{ ProtocolConfig, ProtocolManager, ProtocolUpdateExecutor, ProtocolVersionName, }; -use crate::store::consensus::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; +use crate::store::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; use crate::store::jmt_gc::StateTreeGcConfig; use crate::store::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; -use crate::store::consensus::traits::proofs::QueryableProofStore; -use crate::store::consensus::traits::DatabaseConfigValidationError; +use crate::store::traits::proofs::QueryableProofStore; +use crate::store::traits::DatabaseConfigValidationError; use crate::transaction::{ ExecutionConfigurator, LedgerTransactionValidator, Preparator, TransactionExecutorFactory, }; diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index ceadf978ea..121348d73b 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -67,8 +67,8 @@ use std::ops::Range; use crate::engine_prelude::*; -use crate::store::consensus::traits::indices::CreationId; -use crate::store::consensus::traits::scenario::ScenarioSequenceNumber; +use crate::store::traits::indices::CreationId; +use crate::store::traits::scenario::ScenarioSequenceNumber; use node_common::store::typed_cf_api::*; use crate::transaction::RawLedgerTransaction; use crate::StateVersion; diff --git a/core-rust/state-manager/src/store/consensus/column_families.rs b/core-rust/state-manager/src/store/column_families.rs similarity index 98% rename from core-rust/state-manager/src/store/consensus/column_families.rs rename to core-rust/state-manager/src/store/column_families.rs index 07c78ed5ed..ba14a26556 100644 --- a/core-rust/state-manager/src/store/consensus/column_families.rs +++ b/core-rust/state-manager/src/store/column_families.rs @@ -13,17 +13,17 @@ use crate::{ LocalTransactionExecution, StateVersion, VersionedCommittedTransactionIdentifiers, VersionedLedgerProof, VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, }; -use crate::consensus::traits::{ +use crate::store::traits::{ ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, }; -use crate::consensus::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; -use crate::consensus::traits::indices::{ +use crate::store::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; +use crate::store::traits::indices::{ CreationId, EntityBlueprintId, ObjectBlueprintName, VersionedEntityBlueprintId, VersionedObjectBlueprintName, }; -use crate::consensus::traits::scenario::{ +use crate::store::traits::scenario::{ ExecutedScenario, ScenarioSequenceNumber, VersionedExecutedScenario, }; use crate::store::codecs::{ diff --git a/core-rust/state-manager/src/store/consensus/mod.rs b/core-rust/state-manager/src/store/consensus/mod.rs deleted file mode 100644 index 0c060420bd..0000000000 --- a/core-rust/state-manager/src/store/consensus/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod traits; -pub mod rocks_db; -pub mod column_families; diff --git a/core-rust/state-manager/src/store/historical_state.rs b/core-rust/state-manager/src/store/historical_state.rs index 31b12a4501..dcc4ef791d 100644 --- a/core-rust/state-manager/src/store/historical_state.rs +++ b/core-rust/state-manager/src/store/historical_state.rs @@ -68,9 +68,9 @@ use crate::engine_prelude::entity_tier::EntityTier; use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; use node_common::store::rocks_db::ReadableRocks; -use crate::store::consensus::rocks_db::StateManagerDatabase; -use crate::store::consensus::traits::*; -use crate::consensus::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; +use crate::store::rocks_db::StateManagerDatabase; +use crate::store::traits::*; +use crate::store::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; use crate::{CommittedTransactionIdentifiers, LedgerStateSummary, StateVersion}; /// An implementation of a [`SubstateDatabase`] viewed at a specific [`StateVersion`]. diff --git a/core-rust/state-manager/src/store/jmt_gc.rs b/core-rust/state-manager/src/store/jmt_gc.rs index d93bcb2c5d..a2ba3822d6 100644 --- a/core-rust/state-manager/src/store/jmt_gc.rs +++ b/core-rust/state-manager/src/store/jmt_gc.rs @@ -70,10 +70,10 @@ use std::sync::Arc; use std::time::Duration; use tracing::info; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; -use crate::store::consensus::traits::gc::StateTreeGcStore; -use crate::store::consensus::traits::proofs::QueryableProofStore; -use crate::store::consensus::traits::StaleTreePartsV1; +use crate::store::rocks_db::ActualStateManagerDatabase; +use crate::store::traits::gc::StateTreeGcStore; +use crate::store::traits::proofs::QueryableProofStore; +use crate::store::traits::StaleTreePartsV1; use crate::{StateVersion, StateVersionDelta}; /// A maximum number of JMT nodes collected into "batch delete" buffer. diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index 6d1cdd61d6..00eb741444 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -68,18 +68,20 @@ use prometheus::Registry; use sbor::{Categorize, Decode, Encode}; use node_common::locks::DbLock; -use consensus::rocks_db::ActualStateManagerDatabase; +use rocks_db::ActualStateManagerDatabase; use crate::RawDbMetrics; -pub use crate::store::consensus::traits::DatabaseConfig; -use crate::store::consensus::traits::measurement::MeasurableDatabase; +pub use traits::DatabaseConfig; +use traits::measurement::MeasurableDatabase; pub mod historical_state; pub mod jmt_gc; pub mod proofs_gc; mod codecs; pub mod p2p; -pub mod consensus; +pub mod column_families; +pub mod rocks_db; +pub mod traits; #[derive(Debug, Categorize, Encode, Decode, Clone)] pub struct DatabaseBackendConfig { diff --git a/core-rust/state-manager/src/store/proofs_gc.rs b/core-rust/state-manager/src/store/proofs_gc.rs index 4942f13aaf..0194971d40 100644 --- a/core-rust/state-manager/src/store/proofs_gc.rs +++ b/core-rust/state-manager/src/store/proofs_gc.rs @@ -69,15 +69,15 @@ use std::sync::Arc; use std::time::Duration; use tracing::{error, info}; -use crate::store::consensus::traits::gc::{ +use crate::store::traits::gc::{ LedgerProofsGcProgress, LedgerProofsGcProgressV1, LedgerProofsGcStore, }; -use crate::store::consensus::traits::proofs::QueryableProofStore; +use crate::store::traits::proofs::QueryableProofStore; use crate::jni::LedgerSyncLimitsConfig; -use crate::store::consensus::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; +use crate::store::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; use node_common::store::rocks_db::ReadableRocks; -use crate::consensus::traits::GetSyncableTxnsAndProofError::{ +use crate::store::traits::GetSyncableTxnsAndProofError::{ FailedToPrepareAResponseWithinLimits, NothingToServeAtTheGivenStateVersion, RefusedToServeGenesis, RefusedToServeProtocolUpdate, }; @@ -312,9 +312,9 @@ mod tests { use crate::jni::LedgerSyncLimitsConfig; use crate::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; use crate::protocol::*; - use crate::store::consensus::traits::proofs::QueryableProofStore; + use crate::store::traits::proofs::QueryableProofStore; use crate::test::{commit_round_updates_until_epoch, create_state_manager}; - use crate::consensus::traits::GetSyncableTxnsAndProofError; + use crate::store::traits::GetSyncableTxnsAndProofError; use crate::{StateManagerConfig, StateVersion}; use std::time::Duration; diff --git a/core-rust/state-manager/src/store/consensus/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs similarity index 99% rename from core-rust/state-manager/src/store/consensus/rocks_db.rs rename to core-rust/state-manager/src/store/rocks_db.rs index 2d2923ba73..ad1ebfee2e 100644 --- a/core-rust/state-manager/src/store/consensus/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -79,18 +79,18 @@ use crate::{ StateVersion, SubstateChangeAction, TransactionTreeHash, }; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; -use crate::consensus::column_families::*; +use crate::store::column_families::*; use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; use node_common::store::rocks_db::*; -use crate::store::consensus::traits::*; -use crate::store::consensus::traits::extensions::*; -use crate::store::consensus::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; -use crate::store::consensus::traits::indices::{ +use crate::store::traits::*; +use crate::store::traits::extensions::*; +use crate::store::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; +use crate::store::traits::indices::{ CreationId, EntityBlueprintId, EntityListingIndex, ObjectBlueprintNameV1, }; -use crate::store::consensus::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDatabase}; -use crate::store::consensus::traits::scenario::{ +use crate::store::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDatabase}; +use crate::store::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; use crate::store::historical_state::StateTreeBasedSubstateDatabase; diff --git a/core-rust/state-manager/src/store/consensus/traits.rs b/core-rust/state-manager/src/store/traits.rs similarity index 99% rename from core-rust/state-manager/src/store/consensus/traits.rs rename to core-rust/state-manager/src/store/traits.rs index 44ceb64c29..c041434502 100644 --- a/core-rust/state-manager/src/store/consensus/traits.rs +++ b/core-rust/state-manager/src/store/traits.rs @@ -264,7 +264,7 @@ pub mod substate { pub mod transactions { use super::*; - use crate::store::consensus::traits::CommittedTransactionBundle; + use crate::store::traits::CommittedTransactionBundle; use crate::{ CommittedTransactionIdentifiers, LedgerHashes, LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, diff --git a/core-rust/state-manager/src/system_executor.rs b/core-rust/state-manager/src/system_executor.rs index da26a7307e..0b02dde307 100644 --- a/core-rust/state-manager/src/system_executor.rs +++ b/core-rust/state-manager/src/system_executor.rs @@ -64,7 +64,7 @@ use crate::query::*; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::transaction::*; use crate::*; @@ -75,14 +75,14 @@ use node_common::locks::DbLock; use tracing::info; -use crate::store::consensus::traits::scenario::{ +use crate::store::traits::scenario::{ DescribedAddressRendering, ExecutedScenario, ExecutedScenarioStore, ExecutedScenarioTransaction, }; use crate::system_commits::*; use crate::protocol::{ProtocolUpdateNodeBatch, ProtocolVersionName}; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; -use crate::consensus::traits::scenario::ExecutedScenarioV1; +use crate::store::rocks_db::ActualStateManagerDatabase; +use crate::store::traits::scenario::ExecutedScenarioV1; use radix_transaction_scenarios::scenarios::ALL_SCENARIOS; use std::sync::Arc; use std::time::Instant; diff --git a/core-rust/state-manager/src/test/mod.rs b/core-rust/state-manager/src/test/mod.rs index f8b27cbcb3..1fa48508f0 100644 --- a/core-rust/state-manager/src/test/mod.rs +++ b/core-rust/state-manager/src/test/mod.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; -use crate::store::consensus::traits::QueryableProofStore; +use crate::store::traits::QueryableProofStore; use crate::{ CommitRequest, CommitSummary, LedgerHeader, LedgerProof, LedgerProofOrigin, PrepareRequest, PrepareResult, RoundHistory, StateManager, StateManagerConfig, diff --git a/core-rust/state-manager/src/transaction/preparation.rs b/core-rust/state-manager/src/transaction/preparation.rs index f825d4420e..f6facf6d00 100644 --- a/core-rust/state-manager/src/transaction/preparation.rs +++ b/core-rust/state-manager/src/transaction/preparation.rs @@ -72,7 +72,7 @@ use tracing::{debug, info}; use crate::engine_prelude::*; use crate::limits::*; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::*; use crate::system_commits::*; diff --git a/core-rust/state-manager/src/transaction/preview.rs b/core-rust/state-manager/src/transaction/preview.rs index c9a9bbc7b1..eeb0e5264b 100644 --- a/core-rust/state-manager/src/transaction/preview.rs +++ b/core-rust/state-manager/src/transaction/preview.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use crate::historical_state::{StateHistoryError, VersionScopingSupport}; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; +use crate::store::rocks_db::ActualStateManagerDatabase; use crate::transaction::*; use crate::{ GlobalBalanceSummary, LedgerStateChanges, LedgerStateSummary, PreviewRequest, diff --git a/core-rust/state-manager/src/transaction/series_execution.rs b/core-rust/state-manager/src/transaction/series_execution.rs index 2b38ee0d35..6e1dc88505 100644 --- a/core-rust/state-manager/src/transaction/series_execution.rs +++ b/core-rust/state-manager/src/transaction/series_execution.rs @@ -71,7 +71,7 @@ use crate::commit_bundle::CommitBundleBuilder; use crate::protocol::*; use crate::query::*; use crate::staging::ReadableStore; -use crate::store::consensus::traits::*; +use crate::store::traits::*; use crate::transaction::*; use crate::*; diff --git a/core-rust/state-manager/src/transaction/validation.rs b/core-rust/state-manager/src/transaction/validation.rs index 0ee5729eab..21b8b7e6d1 100644 --- a/core-rust/state-manager/src/transaction/validation.rs +++ b/core-rust/state-manager/src/transaction/validation.rs @@ -7,9 +7,9 @@ use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; -use crate::store::consensus::rocks_db::ActualStateManagerDatabase; -use crate::store::consensus::traits::transactions::QueryableTransactionStore; -use crate::store::consensus::traits::{QueryableProofStore, TransactionIndex}; +use crate::store::rocks_db::ActualStateManagerDatabase; +use crate::store::traits::transactions::QueryableTransactionStore; +use crate::store::traits::{QueryableProofStore, TransactionIndex}; use crate::transaction::{ExecutionConfigurator, TransactionLogic}; use crate::{ AlreadyCommittedError, AtSpecificState, AtState, ExecutionRejectionReason, From 8679a9361f4aeb936b70794831a1773b8b2dd7c5 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 14 Aug 2024 15:00:25 +0200 Subject: [PATCH 19/40] Move p2p files to dedicated crate. --- core-rust/Cargo.lock | 11 +++++++++++ core-rust/p2p/Cargo.toml | 13 +++++++++++++ .../p2p => p2p/src}/address_book_components.rs | 0 .../src/store/p2p => p2p/src}/column_families.rs | 6 +++--- .../src/store/p2p/mod.rs => p2p/src/lib.rs} | 8 ++++++-- .../src/store/p2p => p2p/src}/migration.rs | 0 .../src/store/p2p => p2p/src}/rocks_db.rs | 8 ++++---- .../src/store/p2p => p2p/src}/traits.rs | 4 ++-- .../src/store/p2p => p2p/src}/typed_cf_api.rs | 2 +- core-rust/state-manager/Cargo.toml | 2 ++ .../state-manager/src/jni/node_rust_environment.rs | 4 ++-- core-rust/state-manager/src/jni/p2p/address_book.rs | 4 ++-- .../src/jni/p2p/high_priority_peers.rs | 2 +- .../state-manager/src/jni/p2p/migration_store.rs | 4 ++-- core-rust/state-manager/src/jni/p2p/mod.rs | 2 ++ core-rust/state-manager/src/jni/p2p/safety_store.rs | 2 +- core-rust/state-manager/src/store/mod.rs | 1 - 17 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 core-rust/p2p/Cargo.toml rename core-rust/{state-manager/src/store/p2p => p2p/src}/address_book_components.rs (100%) rename core-rust/{state-manager/src/store/p2p => p2p/src}/column_families.rs (86%) rename core-rust/{state-manager/src/store/p2p/mod.rs => p2p/src/lib.rs} (66%) rename core-rust/{state-manager/src/store/p2p => p2p/src}/migration.rs (100%) rename core-rust/{state-manager/src/store/p2p => p2p/src}/rocks_db.rs (96%) rename core-rust/{state-manager/src/store/p2p => p2p/src}/traits.rs (97%) rename core-rust/{state-manager/src/store/p2p => p2p/src}/typed_cf_api.rs (98%) diff --git a/core-rust/Cargo.lock b/core-rust/Cargo.lock index 31dba83529..9c1b774564 100644 --- a/core-rust/Cargo.lock +++ b/core-rust/Cargo.lock @@ -1498,6 +1498,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p2p" +version = "0.1.0" +dependencies = [ + "jni", + "node-common", + "radix-common", + "sbor", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2386,6 +2396,7 @@ dependencies = [ "jni", "lru", "node-common", + "p2p", "prometheus", "radix-common", "radix-engine", diff --git a/core-rust/p2p/Cargo.toml b/core-rust/p2p/Cargo.toml new file mode 100644 index 0000000000..7045c14441 --- /dev/null +++ b/core-rust/p2p/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "p2p" +version = "0.1.0" +edition = "2021" + +[dependencies] +node-common = { path = "../node-common" } + +sbor = { workspace = true } +radix-common = { workspace = true } + +# Non-Radix Engine Dependencies: +jni = { workspace = true } diff --git a/core-rust/state-manager/src/store/p2p/address_book_components.rs b/core-rust/p2p/src/address_book_components.rs similarity index 100% rename from core-rust/state-manager/src/store/p2p/address_book_components.rs rename to core-rust/p2p/src/address_book_components.rs diff --git a/core-rust/state-manager/src/store/p2p/column_families.rs b/core-rust/p2p/src/column_families.rs similarity index 86% rename from core-rust/state-manager/src/store/p2p/column_families.rs rename to core-rust/p2p/src/column_families.rs index 448bddacb7..9b0793f04c 100644 --- a/core-rust/state-manager/src/store/p2p/column_families.rs +++ b/core-rust/p2p/src/column_families.rs @@ -2,9 +2,9 @@ use node_common::store::typed_cf_api::SborDbCodec; use node_common::store::typed_cf_api::{ DefaultCf, DirectDbCodec, UnitDbCodec, }; -use crate::store::p2p::address_book_components::AddressBookNodeId; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; -use crate::store::p2p::typed_cf_api::AddressBookNodeIdDbCodec; +use crate::address_book_components::AddressBookNodeId; +use crate::migration::{MigrationId, MigrationStatus}; +use crate::typed_cf_api::AddressBookNodeIdDbCodec; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; diff --git a/core-rust/state-manager/src/store/p2p/mod.rs b/core-rust/p2p/src/lib.rs similarity index 66% rename from core-rust/state-manager/src/store/p2p/mod.rs rename to core-rust/p2p/src/lib.rs index 21d63c8211..5e67756683 100644 --- a/core-rust/state-manager/src/store/p2p/mod.rs +++ b/core-rust/p2p/src/lib.rs @@ -1,6 +1,10 @@ pub mod address_book_components; -pub mod traits; -pub mod migration; pub mod column_families; +pub mod migration; pub mod rocks_db; +pub mod traits; pub mod typed_cf_api; + +pub mod engine_prelude { + pub use radix_common::prelude::*; +} diff --git a/core-rust/state-manager/src/store/p2p/migration.rs b/core-rust/p2p/src/migration.rs similarity index 100% rename from core-rust/state-manager/src/store/p2p/migration.rs rename to core-rust/p2p/src/migration.rs diff --git a/core-rust/state-manager/src/store/p2p/rocks_db.rs b/core-rust/p2p/src/rocks_db.rs similarity index 96% rename from core-rust/state-manager/src/store/p2p/rocks_db.rs rename to core-rust/p2p/src/rocks_db.rs index 9686a3cc84..3bc4d1be71 100644 --- a/core-rust/state-manager/src/store/p2p/rocks_db.rs +++ b/core-rust/p2p/src/rocks_db.rs @@ -69,11 +69,11 @@ use node_common::rocksdb::{ }; use crate::engine_prelude::*; -use crate::p2p::address_book_components::AddressBookNodeId; -use crate::p2p::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; -use crate::p2p::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; +use crate::address_book_components::AddressBookNodeId; +use crate::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; +use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; +use crate::migration::{MigrationId, MigrationStatus}; use node_common::store::rocks_db::*; -use crate::store::p2p::migration::{MigrationId, MigrationStatus}; use node_common::store::typed_cf_api::*; /// A listing of all column family names used by the Node. diff --git a/core-rust/state-manager/src/store/p2p/traits.rs b/core-rust/p2p/src/traits.rs similarity index 97% rename from core-rust/state-manager/src/store/p2p/traits.rs rename to core-rust/p2p/src/traits.rs index aba42cc1af..e5c875198f 100644 --- a/core-rust/state-manager/src/store/p2p/traits.rs +++ b/core-rust/p2p/src/traits.rs @@ -65,8 +65,8 @@ use crate::engine_prelude::*; pub mod node { - use crate::p2p::address_book_components::AddressBookNodeId; - use crate::p2p::migration::MigrationId; + use crate::address_book_components::AddressBookNodeId; + use crate::migration::MigrationId; use super::*; diff --git a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs b/core-rust/p2p/src/typed_cf_api.rs similarity index 98% rename from core-rust/state-manager/src/store/p2p/typed_cf_api.rs rename to core-rust/p2p/src/typed_cf_api.rs index 709a139cd8..0325f5bc30 100644 --- a/core-rust/state-manager/src/store/p2p/typed_cf_api.rs +++ b/core-rust/p2p/src/typed_cf_api.rs @@ -62,7 +62,7 @@ * permissions under this License. */ -use crate::p2p::address_book_components::AddressBookNodeId; +use crate::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; use node_common::store::typed_cf_api::{DbCodec, BoundedDbCodec}; diff --git a/core-rust/state-manager/Cargo.toml b/core-rust/state-manager/Cargo.toml index da15a8d934..44959da920 100644 --- a/core-rust/state-manager/Cargo.toml +++ b/core-rust/state-manager/Cargo.toml @@ -5,6 +5,8 @@ edition = "2021" [dependencies] node-common = { path = "../node-common" } +# Eventually this should be removed, but we need to move JNINodeRustEnvironment to other place firts. +p2p = { path = "../p2p" } sbor = { workspace = true } radix-transactions = { workspace = true } diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index 52e622826d..d603b21546 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -86,12 +86,12 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use crate::store::p2p::rocks_db::ActualAddressBookDatabase; +use p2p::rocks_db::ActualAddressBookDatabase; use crate::store::rocks_db::ActualStateManagerDatabase; use crate::transaction::Preparator; use crate::{Committer, DatabaseBackendConfig, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; -use crate::p2p::rocks_db::{ActualMigrationDatabase, ActualSafetyStoreDatabase}; +use p2p::rocks_db::{ActualMigrationDatabase, ActualSafetyStoreDatabase}; const POINTER_JNI_FIELD_NAME: &str = "rustNodeRustEnvironmentPointer"; diff --git a/core-rust/state-manager/src/jni/p2p/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs index c71374f7c0..4ae4822e13 100644 --- a/core-rust/state-manager/src/jni/p2p/address_book.rs +++ b/core-rust/state-manager/src/jni/p2p/address_book.rs @@ -64,12 +64,12 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::p2p::traits::node::AddressBookStore; +use p2p::traits::node::AddressBookStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; -use crate::p2p::address_book_components::{AddressBookEntry, AddressBookNodeId}; +use p2p::address_book_components::{AddressBookEntry, AddressBookNodeId}; #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( diff --git a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs index 942c9ea016..8e408fe497 100644 --- a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs @@ -70,7 +70,7 @@ use node_common::java::*; use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::p2p::traits::node::HighPriorityPeersStore; +use p2p::traits::node::HighPriorityPeersStore; #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAllHighPriorityPeers( diff --git a/core-rust/state-manager/src/jni/p2p/migration_store.rs b/core-rust/state-manager/src/jni/p2p/migration_store.rs index 9f1e23a4f1..d14d6d5241 100644 --- a/core-rust/state-manager/src/jni/p2p/migration_store.rs +++ b/core-rust/state-manager/src/jni/p2p/migration_store.rs @@ -69,8 +69,8 @@ use jni::sys::jbyteArray; use node_common::java::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::store::p2p::migration::MigrationId; -use crate::p2p::traits::node::MigrationStore; +use p2p::migration::MigrationId; +use p2p::traits::node::MigrationStore; #[no_mangle] extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_migrationDone( diff --git a/core-rust/state-manager/src/jni/p2p/mod.rs b/core-rust/state-manager/src/jni/p2p/mod.rs index 97b8a8e6b4..45c5fbaa61 100644 --- a/core-rust/state-manager/src/jni/p2p/mod.rs +++ b/core-rust/state-manager/src/jni/p2p/mod.rs @@ -1,3 +1,5 @@ +// This module should be eventually moved to p2p crate. + pub mod address_book; pub mod high_priority_peers; pub mod migration_store; diff --git a/core-rust/state-manager/src/jni/p2p/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs index 1bb142bad2..616250fe43 100644 --- a/core-rust/state-manager/src/jni/p2p/safety_store.rs +++ b/core-rust/state-manager/src/jni/p2p/safety_store.rs @@ -64,7 +64,7 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use crate::store::p2p::traits::node::SafetyStateStore; +use p2p::traits::node::SafetyStateStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index 00eb741444..495fa7c86d 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -78,7 +78,6 @@ pub mod historical_state; pub mod jmt_gc; pub mod proofs_gc; mod codecs; -pub mod p2p; pub mod column_families; pub mod rocks_db; pub mod traits; From a428d18ea49b1d4fe2400dcc4ff852d0fb2d43f8 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Thu, 15 Aug 2024 13:57:13 +0200 Subject: [PATCH 20/40] Address review comments. --- .github/workflows/ci.yml | 8 +- .../db/RocksDbMigrationStoreTest.java | 46 +------ .../helper/NodeRustEnvironmentTestBase.java | 116 ++++++++++++++++++ .../p2p/RocksDbAddressBookStoreTest.java | 44 +------ .../RocksDbHighPriorityPeersStoreTest.java | 51 +------- .../safety/RocksDbSafetyStoreTest.java | 44 +------ .../src/jni/node_rust_environment.rs | 2 +- .../PrefixedNodeStorageLocationModule.java | 2 +- .../safety/RocksSafetyStateStoreTest.java | 45 +------ .../helper/NodeRustEnvironmentTestBase.java} | 49 ++++++-- .../RocksAddressBookStoreTest.java | 45 +------ 11 files changed, 174 insertions(+), 278 deletions(-) create mode 100644 core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java rename core/src/{main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java => test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java} (68%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16591b658..39293fff88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ concurrency: on: pull_request: - # Runs on all PRs +# Runs on all PRs push: branches: - develop @@ -233,7 +233,8 @@ jobs: restore-keys: ${{ runner.os }}-gradle - name: Run steady-state integration tests env: - RADIXDLT_LOG_LEVEL: warn + # Might be set to warn for debugging purposes. Warning, log file will be huge. + RADIXDLT_LOG_LEVEL: error run: ./gradlew clean runSteadyStateIntegrationTests --info --refresh-dependencies targeted-integration: name: Targeted integration tests @@ -262,7 +263,8 @@ jobs: restore-keys: ${{ runner.os }}-gradle - name: Run targeted integration tests env: - RADIXDLT_LOG_LEVEL: warn + # Might be set to warn for debugging purposes. Warning, log file will be huge. + RADIXDLT_LOG_LEVEL: error run: ./gradlew clean runTargetedIntegrationTests --info --refresh-dependencies --parallel cross-xwin: name: Cross compile to Windows diff --git a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java index 2a6444b483..90336464d3 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java @@ -66,22 +66,11 @@ import static org.junit.Assert.*; -import com.radixdlt.environment.*; -import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class RocksDbMigrationStoreTest { - - @Rule public TemporaryFolder folder = new TemporaryFolder(); +public class RocksDbMigrationStoreTest extends NodeRustEnvironmentTestBase { @Test public void migration_status_is_properly_reported() throws Exception { var migrationStore = @@ -97,35 +86,4 @@ public void migration_status_is_properly_reported() throws Exception { migrationStore.migrationDone(StoreId.ADDRESS_BOOK); assertTrue(migrationStore.isMigrated(StoreId.ADDRESS_BOOK)); } - - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java b/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java new file mode 100644 index 0000000000..3768ab9e08 --- /dev/null +++ b/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java @@ -0,0 +1,116 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.helper; + +import com.radixdlt.crypto.ECKeyPair; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.p2p.NodeIdDTO; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public abstract class NodeRustEnvironmentTestBase { + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + protected NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); + + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config); + } + + protected static NodeIdDTO newNodeId() { + return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); + } +} diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index b563566a64..3d99a6d0dd 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -69,28 +69,19 @@ import static org.junit.Assert.*; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.environment.*; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.p2p.PeerAddressEntryDTO.ConnectionStatus; -import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; import java.util.Random; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class RocksDbAddressBookStoreTest { +public class RocksDbAddressBookStoreTest extends NodeRustEnvironmentTestBase { private static final Random RANDOM = new Random(); - @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Test public void test_address_book_entries_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { @@ -193,37 +184,6 @@ public void test_address_book_can_be_reset() throws Exception { } } - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } - private static AddressBookEntryDTO newAddressBookEntry(int id) { return new AddressBookEntryDTO( new NodeIdDTO(ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey()), diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java index 74d743ae00..dd32f410fd 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java @@ -64,26 +64,14 @@ package com.radixdlt.p2p; -import static com.radixdlt.lang.Option.some; import static org.junit.Assert.*; -import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.environment.*; -import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; import java.util.List; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class RocksDbHighPriorityPeersStoreTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +public class RocksDbHighPriorityPeersStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_high_priority_peers_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { @@ -124,39 +112,4 @@ public void test_high_priority_peers_can_be_saved_and_restored() throws Exceptio assertTrue(empty.isEmpty()); } } - - private static NodeIdDTO newNodeId() { - return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); - } - - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 19936a4c87..6fd42459b2 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -68,23 +68,14 @@ import com.google.common.primitives.Bytes; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.environment.*; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.protocol.ProtocolConfig; import com.radixdlt.rev2.ComponentAddress; -import com.radixdlt.rev2.NetworkDefinition; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; import java.util.Collections; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class RocksDbSafetyStoreTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +public class RocksDbSafetyStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_safety_state_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { @@ -113,35 +104,4 @@ public void test_safety_state_can_be_saved_and_restored() throws Exception { assertEquals(originalSafetyState, present.orElseThrow()); } } - - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - Option.some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } } diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index d603b21546..351cf128e4 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -166,7 +166,7 @@ impl JNINodeRustEnvironment { let running_task_tracker = scheduler.into_task_tracker(); let address_book_db_path = Self::combine(&base_path, "address_book"); - let safety_store_db_path = Self::combine(&base_path, "safety_store"); + let safety_store_db_path = Self::combine(&base_path, "consensus_safety_store"); let migration_store_db_path = Self::combine(&base_path, "migration_store"); let jni_node_rust_env = JNINodeRustEnvironment { diff --git a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java index f089841f24..8535976384 100644 --- a/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/PrefixedNodeStorageLocationModule.java @@ -86,7 +86,7 @@ public PrefixedNodeStorageLocationModule(String baseLocation) { @Provides @Singleton @NodeStorageLocation - private String stateManagerStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { + private String nodeStorageLocation(@Self ECDSASecp256k1PublicKey publicKey) { return new File(baseLocation, publicKey.toHex()).getPath(); } } diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index 11de83c2e0..3a956c1805 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -64,28 +64,18 @@ package com.radixdlt.consensus.safety; -import static com.radixdlt.lang.Option.some; import static org.junit.Assert.*; import com.radixdlt.consensus.bft.BFTValidatorId; -import com.radixdlt.environment.*; -import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.safety.RocksDbSafetyStore; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; import java.io.IOException; import java.util.function.Consumer; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class RocksSafetyStateStoreTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +public class RocksSafetyStateStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_address_book_entries_can_be_saved_and_restored() { runTest( @@ -118,35 +108,4 @@ private void runTest(Consumer test) { private static Metrics newMetrics() { return new MetricsInitializer().initialize(); } - - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } } diff --git a/core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java b/core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java similarity index 68% rename from core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java rename to core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java index 248221a968..ff5365dddb 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/MigratingSafetyStore.java +++ b/core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java @@ -62,20 +62,49 @@ * permissions under this License. */ -package com.radixdlt.consensus.safety; +package com.radixdlt.helper; -import java.util.Optional; +import com.radixdlt.environment.*; +import com.radixdlt.lang.Option; +import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.protocol.ProtocolConfig; +import com.radixdlt.rev2.NetworkDefinition; +import com.radixdlt.transaction.LedgerSyncLimitsConfig; +import java.io.IOException; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; -public class MigratingSafetyStore implements PersistentSafetyStateStore { +public abstract class NodeRustEnvironmentTestBase { + @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Override - public void commitState(SafetyState safetyState) {} + protected NodeRustEnvironment createNodeRustEnvironment() throws IOException { + final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; + final var mempoolMaxTransactionCount = 20; + final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - @Override - public void close() {} + final var config = + new StateManagerConfig( + NetworkDefinition.INT_TEST_NET, + Option.some( + new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), + Option.none(), + stateManagerDbConfig, + new DatabaseConfig(false, false, false, false), + LoggingConfig.getDefault(), + StateTreeGcConfig.forTesting(), + LedgerProofsGcConfig.forTesting(), + LedgerSyncLimitsConfig.defaults(), + ProtocolConfig.testingDefault(), + false, + ScenariosExecutionConfig.NONE); - @Override - public Optional get() { - return Optional.empty(); + return new NodeRustEnvironment( + tx -> {}, // A no-op dispatcher of transactions to be relayed. + () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this + // one) will observe + // panics as runtime exceptions propagated up the stack (through JNI), which will fail the + // test + // gracefully anyway. + config); } } diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index 95b1f41a38..03fd4a7e4d 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -64,39 +64,29 @@ package com.radixdlt.p2p.addressbook; -import static com.radixdlt.lang.Option.some; import static org.junit.Assert.*; import com.google.common.collect.ImmutableSet; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.environment.*; -import com.radixdlt.lang.Option; -import com.radixdlt.mempool.RustMempoolConfig; +import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.p2p.*; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.FailedHandshake; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.LatestConnectionStatus; -import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; -import com.radixdlt.transaction.LedgerSyncLimitsConfig; import java.io.IOException; import java.time.Instant; import java.util.Optional; import java.util.Random; import java.util.function.Consumer; import java.util.stream.IntStream; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -public class RocksAddressBookStoreTest { +public class RocksAddressBookStoreTest extends NodeRustEnvironmentTestBase { private static final Random RANDOM = new Random(); - @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Test public void test_address_book_entries_can_be_saved_and_restored() { runTest( @@ -153,37 +143,6 @@ private static Metrics newMetrics() { return new MetricsInitializer().initialize(); } - private NodeRustEnvironment createNodeRustEnvironment() throws IOException { - final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; - final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); - - final var config = - new StateManagerConfig( - NetworkDefinition.INT_TEST_NET, - some( - new RustMempoolConfig(mempoolMaxTotalTransactionsSize, mempoolMaxTransactionCount)), - Option.none(), - stateManagerDbConfig, - new DatabaseConfig(false, false, false, false), - LoggingConfig.getDefault(), - StateTreeGcConfig.forTesting(), - LedgerProofsGcConfig.forTesting(), - LedgerSyncLimitsConfig.defaults(), - ProtocolConfig.testingDefault(), - false, - ScenariosExecutionConfig.NONE); - - return new NodeRustEnvironment( - tx -> {}, // A no-op dispatcher of transactions to be relayed. - () -> {}, // A no-op fatal panic handler. Please note that a JNI-invoking test (like this - // one) will observe - // panics as runtime exceptions propagated up the stack (through JNI), which will fail the - // test - // gracefully anyway. - config); - } - private static AddressBookEntry newAddressBookEntry(int id) { var pubKey = ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey(); var bannedUntil = newBannedUntil(); From 66c44a640be3d0e48cd6c395d2a59bd471fb57d4 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Thu, 15 Aug 2024 17:06:52 +0200 Subject: [PATCH 21/40] Let address book and safety store hold their migration status. --- .../radixdlt/db/RocksDbMigrationStore.java | 120 ------------------ .../main/java/com/radixdlt/db/StoreId.java | 83 ------------ .../radixdlt/p2p/RocksDbAddressBookStore.java | 24 ++++ .../radixdlt/safety/RocksDbSafetyStore.java | 23 ++++ .../com/radixdlt/sbor/NodeSborCodecs.java | 2 - .../db/RocksDbMigrationStoreTest.java | 89 ------------- .../p2p/RocksDbAddressBookStoreTest.java | 15 +++ .../safety/RocksDbSafetyStoreTest.java | 14 ++ core-rust/p2p/src/column_families.rs | 6 +- core-rust/p2p/src/migration.rs | 8 -- core-rust/p2p/src/rocks_db.rs | 51 +++----- core-rust/p2p/src/traits.rs | 11 +- .../src/jni/node_rust_environment.rs | 22 +--- .../state-manager/src/jni/p2p/address_book.rs | 26 ++++ .../src/jni/p2p/migration_store.rs | 101 --------------- core-rust/state-manager/src/jni/p2p/mod.rs | 1 - .../state-manager/src/jni/p2p/safety_store.rs | 26 ++++ .../safety/RocksSafetyStateStore.java | 8 ++ .../addressbook/RocksAddressBookStore.java | 8 ++ .../rev2/modules/NodePersistenceModule.java | 95 +++++--------- 20 files changed, 210 insertions(+), 523 deletions(-) delete mode 100644 core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java delete mode 100644 core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java delete mode 100644 core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java delete mode 100644 core-rust/state-manager/src/jni/p2p/migration_store.rs diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java b/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java deleted file mode 100644 index f2be16a5ee..0000000000 --- a/core-rust-bridge/src/main/java/com/radixdlt/db/RocksDbMigrationStore.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.db; - -import com.google.common.reflect.TypeToken; -import com.radixdlt.environment.NodeRustEnvironment; -import com.radixdlt.lang.Tuple.Tuple0; -import com.radixdlt.monitoring.Metrics; -import com.radixdlt.sbor.Natives; - -public class RocksDbMigrationStore { - static { - System.loadLibrary("corerust"); - } - - private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - - private static native byte[] migrationDone( - NodeRustEnvironment nodeRustEnvironment, byte[] payload); - - /** - * Stores a pointer to the rust core api server across JNI calls. In the JNI model, this is - * equivalent to the CoreApiServer "owning" the rust core api server memory. On each call into - * Rust, we map the rustCoreApiServerPointer onto a concrete implementation in Rust land, and it - * uses that to access all state and make calls. - */ - @SuppressWarnings("unused") - private final long rustCoreApiServerPointer = 0; - - public static RocksDbMigrationStore create( - Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - return new RocksDbMigrationStore(metrics, nodeRustEnvironment); - } - - private RocksDbMigrationStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { - final var timer = metrics.stateManager().nativeCall(); - isMigratedFunc = - Natives.builder(nodeRustEnvironment, RocksDbMigrationStore::isMigrated) - .measure(timer.label(new Metrics.MethodId(RocksDbMigrationStore.class, "isMigrated"))) - .build(new TypeToken<>() {}); - migrationDoneFunc = - Natives.builder(nodeRustEnvironment, RocksDbMigrationStore::migrationDone) - .measure( - timer.label(new Metrics.MethodId(RocksDbMigrationStore.class, "migrationDone"))) - .build(new TypeToken<>() {}); - } - - public boolean isMigrated(StoreId storeId) { - return this.isMigratedFunc.call(storeId); - } - - public void migrationDone(StoreId storeId) { - this.migrationDoneFunc.call(storeId); - } - - private final Natives.Call1 isMigratedFunc; - private final Natives.Call1 migrationDoneFunc; -} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java b/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java deleted file mode 100644 index 48a6506361..0000000000 --- a/core-rust-bridge/src/main/java/com/radixdlt/db/StoreId.java +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.db; - -import com.radixdlt.sbor.codec.CodecMap; -import com.radixdlt.sbor.codec.EnumCodec; - -/** Java side representation of the StoreId enum in Rust */ -public sealed interface StoreId { - record AddressBookStoreId() implements StoreId {} - - record SafetyStoreId() implements StoreId {} - - AddressBookStoreId ADDRESS_BOOK = new AddressBookStoreId(); - SafetyStoreId SAFETY_STORE = new SafetyStoreId(); - - static void registerCodec(CodecMap codecMap) { - codecMap.register( - StoreId.class, codecs -> EnumCodec.fromPermittedRecordSubclasses(StoreId.class, codecs)); - } -} diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java index aaafe4eb20..9daee86d94 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java @@ -88,6 +88,11 @@ public class RocksDbAddressBookStore { private static native byte[] getAll(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] markAsMigrated( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private final Codec dtoCodec; public static RocksDbAddressBookStore create( @@ -113,6 +118,15 @@ private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnv Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::getAll) .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "getAll"))) .build(new TypeToken<>() {}); + markAsMigratedFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::markAsMigrated) + .measure( + timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "markAsMigrated"))) + .build(new TypeToken<>() {}); + isMigratedFunc = + Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::isMigrated) + .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "isMigrated"))) + .build(new TypeToken<>() {}); dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } @@ -135,8 +149,18 @@ public ImmutableList getAllEntries() { .collect(ImmutableList.toImmutableList()); } + public boolean isMigrated() { + return this.isMigratedFunc.call(tuple()); + } + + public void markAsMigrated() { + this.markAsMigratedFunc.call(tuple()); + } + private final Natives.Call1 removeOneFunc; private final Natives.Call1 upsertOneFunc; private final Natives.Call1 resetFunc; private final Natives.Call1> getAllFunc; + private final Natives.Call1 isMigratedFunc; + private final Natives.Call1 markAsMigratedFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java index 7338b9de51..b0bbdbd6f6 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java @@ -83,6 +83,11 @@ public class RocksDbSafetyStore { private static native byte[] get(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private static native byte[] markAsMigrated( + NodeRustEnvironment nodeRustEnvironment, byte[] payload); + + private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); + private final Codec dtoCodec; public static RocksDbSafetyStore create( @@ -100,6 +105,14 @@ private RocksDbSafetyStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironm Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::get) .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "get"))) .build(new TypeToken<>() {}); + markAsMigratedFunc = + Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::markAsMigrated) + .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "markAsMigrated"))) + .build(new TypeToken<>() {}); + isMigratedFunc = + Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::isMigrated) + .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "isMigrated"))) + .build(new TypeToken<>() {}); dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } @@ -112,6 +125,16 @@ public Option get() { return this.getFunc.call(tuple()).map(value -> NodeSborCodecs.decode(value, dtoCodec)); } + public boolean isMigrated() { + return this.isMigratedFunc.call(tuple()); + } + + public void markAsMigrated() { + this.markAsMigratedFunc.call(tuple()); + } + private final Natives.Call1 upsertFunc; private final Natives.Call1> getFunc; + private final Natives.Call1 isMigratedFunc; + private final Natives.Call1 markAsMigratedFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index ff5534fcd2..002bd87484 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -67,7 +67,6 @@ import com.google.common.hash.HashCode; import com.google.common.reflect.TypeToken; import com.radixdlt.crypto.*; -import com.radixdlt.db.StoreId; import com.radixdlt.environment.*; import com.radixdlt.exceptions.StateManagerRuntimeError; import com.radixdlt.genesis.*; @@ -238,7 +237,6 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { AddressBookEntryDTO.registerCodec(codecMap); NodeIdDTO.registerCodec(codecMap); - StoreId.registerCodec(codecMap); } public static void registerCodecsForExistingTypes(CodecMap codecMap) { diff --git a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java deleted file mode 100644 index 90336464d3..0000000000 --- a/core-rust-bridge/src/test/java/com/radixdlt/db/RocksDbMigrationStoreTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.db; - -import static org.junit.Assert.*; - -import com.radixdlt.helper.NodeRustEnvironmentTestBase; -import com.radixdlt.monitoring.MetricsInitializer; -import org.junit.Test; - -public class RocksDbMigrationStoreTest extends NodeRustEnvironmentTestBase { - @Test - public void migration_status_is_properly_reported() throws Exception { - var migrationStore = - RocksDbMigrationStore.create( - new MetricsInitializer().initialize(), createNodeRustEnvironment()); - - assertFalse(migrationStore.isMigrated(StoreId.SAFETY_STORE)); - assertFalse(migrationStore.isMigrated(StoreId.ADDRESS_BOOK)); - - migrationStore.migrationDone(StoreId.SAFETY_STORE); - assertTrue(migrationStore.isMigrated(StoreId.SAFETY_STORE)); - - migrationStore.migrationDone(StoreId.ADDRESS_BOOK); - assertTrue(migrationStore.isMigrated(StoreId.ADDRESS_BOOK)); - } -} diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index 3d99a6d0dd..64e8056fdd 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -82,6 +82,21 @@ public class RocksDbAddressBookStoreTest extends NodeRustEnvironmentTestBase { private static final Random RANDOM = new Random(); + @Test + public void test_address_book_can_be_marked_as_migrated() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var addressBookStore = + RocksDbAddressBookStore.create( + new MetricsInitializer().initialize(), nodeRustEnvironment); + + assertFalse(addressBookStore.isMigrated()); + + addressBookStore.markAsMigrated(); + + assertTrue(addressBookStore.isMigrated()); + } + } + @Test public void test_address_book_entries_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 6fd42459b2..84dfdf2b5f 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -76,6 +76,20 @@ import org.junit.Test; public class RocksDbSafetyStoreTest extends NodeRustEnvironmentTestBase { + @Test + public void test_safety_store_can_be_marked_as_migrated() throws Exception { + try (var nodeRustEnvironment = createNodeRustEnvironment()) { + var safetyStore = + RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); + + assertFalse(safetyStore.isMigrated()); + + safetyStore.markAsMigrated(); + + assertTrue(safetyStore.isMigrated()); + } + } + @Test public void test_safety_state_can_be_saved_and_restored() throws Exception { try (var nodeRustEnvironment = createNodeRustEnvironment()) { diff --git a/core-rust/p2p/src/column_families.rs b/core-rust/p2p/src/column_families.rs index 9b0793f04c..b8f92016d8 100644 --- a/core-rust/p2p/src/column_families.rs +++ b/core-rust/p2p/src/column_families.rs @@ -3,17 +3,17 @@ use node_common::store::typed_cf_api::{ DefaultCf, DirectDbCodec, UnitDbCodec, }; use crate::address_book_components::AddressBookNodeId; -use crate::migration::{MigrationId, MigrationStatus}; +use crate::migration::MigrationStatus; use crate::typed_cf_api::AddressBookNodeIdDbCodec; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; impl DefaultCf for MigrationStatusCf { - type Key = MigrationId; + type Key = (); type Value = MigrationStatus; const DEFAULT_NAME: &'static str = "migration_status"; - type KeyCodec = SborDbCodec; + type KeyCodec = UnitDbCodec; type ValueCodec = SborDbCodec; } diff --git a/core-rust/p2p/src/migration.rs b/core-rust/p2p/src/migration.rs index 88b2c40f2d..d2894e1811 100644 --- a/core-rust/p2p/src/migration.rs +++ b/core-rust/p2p/src/migration.rs @@ -1,13 +1,5 @@ use sbor::Sbor; -/// Identifiers for the migrated stores -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Sbor)] -pub enum MigrationId { - AddressBook, - SafetyState, -} - /// Status of the migration #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Sbor)] diff --git a/core-rust/p2p/src/rocks_db.rs b/core-rust/p2p/src/rocks_db.rs index 3bc4d1be71..de70abcefc 100644 --- a/core-rust/p2p/src/rocks_db.rs +++ b/core-rust/p2p/src/rocks_db.rs @@ -71,8 +71,8 @@ use node_common::rocksdb::{ use crate::engine_prelude::*; use crate::address_book_components::AddressBookNodeId; use crate::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; -use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, MigrationStore, SafetyStateStore}; -use crate::migration::{MigrationId, MigrationStatus}; +use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, SafetyStateStore}; +use crate::migration::MigrationStatus; use node_common::store::rocks_db::*; use node_common::store::typed_cf_api::*; @@ -95,20 +95,19 @@ use node_common::store::typed_cf_api::*; /// family names. Any change would effectively mean a ledger wipe. For this reason, we choose to /// define them manually (rather than using the `Into`, which is refactor-sensitive). -const ALL_ADDRESS_BOOK_COLUMN_FAMILIES: [&str; 2] = [ +const ALL_ADDRESS_BOOK_COLUMN_FAMILIES: [&str; 3] = [ AddressBookCf::DEFAULT_NAME, HighPriorityPeersCf::DEFAULT_NAME, + MigrationStatusCf::DEFAULT_NAME, ]; -const ALL_SAFETY_STORE_COLUMN_FAMILIES: [&str; 1] = [ + +const ALL_SAFETY_STORE_COLUMN_FAMILIES: [&str; 2] = [ SafetyStoreCf::DEFAULT_NAME, -]; -const ALL_MIGRATION_STORE_COLUMN_FAMILIES: [&str; 1] = [ MigrationStatusCf::DEFAULT_NAME, ]; pub type ActualAddressBookDatabase = AddressBookDatabase; pub type ActualSafetyStoreDatabase = SafetyStoreDatabase; -pub type ActualMigrationDatabase = MigrationDatabase; /// A RocksDB-backed persistence layer for address book. pub struct AddressBookDatabase { @@ -122,12 +121,6 @@ pub struct SafetyStoreDatabase { rocks: R, } -/// A RocksDB-backed persistence layer for migration database. -pub struct MigrationDatabase { - /// Underlying RocksDB instance. - rocks: R, -} - fn new_rocks_db(root_path: PathBuf, column_families: &[&str]) -> DB { let mut db_opts = Options::default(); db_opts.create_if_missing(true); @@ -165,16 +158,6 @@ impl ActualSafetyStoreDatabase { } } -impl ActualMigrationDatabase { - pub fn new( - root_path: PathBuf, - ) -> ActualMigrationDatabase { - ActualMigrationDatabase { - rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_MIGRATION_STORE_COLUMN_FAMILIES) }, - } - } -} - impl AddressBookStore for AddressBookDatabase { fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { let binding = open_rw_context(&self.rocks); @@ -204,6 +187,16 @@ impl AddressBookStore for AddressBookDatabase { open_rw_context(&self.rocks).cf(AddressBookCf) .get_all() } + + fn is_migrated(&self) -> bool { + open_rw_context(&self.rocks).cf(MigrationStatusCf) + .get(&()).is_some() + } + + fn mark_as_migrated(&self) { + open_rw_context(&self.rocks).cf(MigrationStatusCf) + .put(&(), &MigrationStatus::Completed) + } } impl HighPriorityPeersStore for AddressBookDatabase { @@ -230,16 +223,14 @@ impl SafetyStateStore for SafetyStoreDatabase { open_rw_context(&self.rocks).cf(SafetyStoreCf) .get(&()) } -} -impl MigrationStore for MigrationDatabase { - fn is_migration_done(&self, store_id: MigrationId) -> bool { + fn is_migrated(&self) -> bool { open_rw_context(&self.rocks).cf(MigrationStatusCf) - .get(&store_id).is_some() + .get(&()).is_some() } - - fn migration_done(&self, store_id: MigrationId) { + + fn mark_as_migrated(&self) { open_rw_context(&self.rocks).cf(MigrationStatusCf) - .put(&store_id, &MigrationStatus::Completed) + .put(&(), &MigrationStatus::Completed) } } diff --git a/core-rust/p2p/src/traits.rs b/core-rust/p2p/src/traits.rs index e5c875198f..dd5218b9cd 100644 --- a/core-rust/p2p/src/traits.rs +++ b/core-rust/p2p/src/traits.rs @@ -66,7 +66,6 @@ use crate::engine_prelude::*; pub mod node { use crate::address_book_components::AddressBookNodeId; - use crate::migration::MigrationId; use super::*; @@ -75,8 +74,11 @@ pub mod node { fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; fn reset(&self); fn get_all(&self, ) -> Vec>; + fn is_migrated(&self) -> bool; + fn mark_as_migrated(&self); } + // At Java side it is represented as part of AddressBookStore pub trait HighPriorityPeersStore { fn upsert_all_peers(&self, peers: &[u8]); fn get_all_peers(&self) -> Option>; @@ -86,10 +88,7 @@ pub mod node { pub trait SafetyStateStore { fn upsert_safety_state(&self, safety_state: &[u8]); fn get_safety_state(&self) -> Option>; - } - - pub trait MigrationStore { - fn is_migration_done(&self, store_id: MigrationId) -> bool; - fn migration_done(&self, store_id: MigrationId); + fn is_migrated(&self) -> bool; + fn mark_as_migrated(&self); } } diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index 351cf128e4..201ab331e9 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -62,7 +62,7 @@ * permissions under this License. */ use std::ops::Deref; -use std::path::{MAIN_SEPARATOR, PathBuf}; +use std::path::{PathBuf, MAIN_SEPARATOR}; use std::str::FromStr; use std::sync::Arc; @@ -86,12 +86,11 @@ use crate::priority_mempool::PriorityMempool; use super::fatal_panic_handler::FatalPanicHandler; use crate::protocol::ProtocolManager; -use p2p::rocks_db::ActualAddressBookDatabase; use crate::store::rocks_db::ActualStateManagerDatabase; use crate::transaction::Preparator; use crate::{Committer, DatabaseBackendConfig, LedgerMetrics, SystemExecutor}; use crate::{StateManager, StateManagerConfig}; -use p2p::rocks_db::{ActualMigrationDatabase, ActualSafetyStoreDatabase}; +use p2p::rocks_db::{ActualAddressBookDatabase, ActualSafetyStoreDatabase}; const POINTER_JNI_FIELD_NAME: &str = "rustNodeRustEnvironmentPointer"; @@ -126,7 +125,6 @@ pub struct JNINodeRustEnvironment { pub running_task_tracker: UntilDropTracker, pub address_book_store: Arc, pub safety_store_store: Arc, - pub migration_store: Arc, } impl JNINodeRustEnvironment { @@ -154,7 +152,7 @@ impl JNINodeRustEnvironment { .use_tokio(runtime.deref()) .track_running_tasks() .measured(metric_registry.deref()); - + let state_manager = StateManager::new( config, Some(MempoolRelayDispatcher::new(env, j_node_rust_env).unwrap()), @@ -167,8 +165,6 @@ impl JNINodeRustEnvironment { let address_book_db_path = Self::combine(&base_path, "address_book"); let safety_store_db_path = Self::combine(&base_path, "consensus_safety_store"); - let migration_store_db_path = Self::combine(&base_path, "migration_store"); - let jni_node_rust_env = JNINodeRustEnvironment { runtime, network, @@ -177,7 +173,6 @@ impl JNINodeRustEnvironment { running_task_tracker, address_book_store: Arc::new(ActualAddressBookDatabase::new(address_book_db_path)), safety_store_store: Arc::new(ActualSafetyStoreDatabase::new(safety_store_db_path)), - migration_store: Arc::new(ActualMigrationDatabase::new(migration_store_db_path)), }; env.set_rust_field(j_node_rust_env, POINTER_JNI_FIELD_NAME, jni_node_rust_env) @@ -198,7 +193,7 @@ impl JNINodeRustEnvironment { }, ..config }; - + (base_path, config) } @@ -206,7 +201,7 @@ impl JNINodeRustEnvironment { let mut path = base.clone(); path.push(MAIN_SEPARATOR); path.push_str(ext); - + PathBuf::from(path) } @@ -257,13 +252,6 @@ impl JNINodeRustEnvironment { Self::get(env, j_node_rust_env).safety_store_store.clone() } - pub fn get_migration_database( - env: &JNIEnv, - j_node_rust_env: JObject, - ) -> Arc { - Self::get(env, j_node_rust_env).migration_store.clone() - } - pub fn get_mempool(env: &JNIEnv, j_node_rust_env: JObject) -> Arc> { Self::get(env, j_node_rust_env) .state_manager diff --git a/core-rust/state-manager/src/jni/p2p/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs index 4ae4822e13..5b8f36eb74 100644 --- a/core-rust/state-manager/src/jni/p2p/address_book.rs +++ b/core-rust/state-manager/src/jni/p2p/address_book.rs @@ -127,4 +127,30 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( }) } +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_markAsMigrated( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_: ()| { + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) + .mark_as_migrated(); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_isMigrated( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_: ()| -> bool { + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) + .is_migrated() + }) +} + pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/p2p/migration_store.rs b/core-rust/state-manager/src/jni/p2p/migration_store.rs deleted file mode 100644 index d14d6d5241..0000000000 --- a/core-rust/state-manager/src/jni/p2p/migration_store.rs +++ /dev/null @@ -1,101 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -use jni::JNIEnv; -use jni::objects::{JClass, JObject}; -use jni::sys::jbyteArray; - -use node_common::java::*; - -use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use p2p::migration::MigrationId; -use p2p::traits::node::MigrationStore; - -#[no_mangle] -extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_migrationDone( - env: JNIEnv, - _class: JClass, - j_rust_global_context: JObject, - migration_id: jbyteArray, -) -> jbyteArray { - jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| { - JNINodeRustEnvironment::get_migration_database(&env, j_rust_global_context) - .migration_done(migration_store_id); - }) -} - -#[no_mangle] -extern "system" fn Java_com_radixdlt_db_RocksDbMigrationStore_isMigrated( - env: JNIEnv, - _class: JClass, - j_rust_global_context: JObject, - migration_id: jbyteArray, -) -> jbyteArray { - jni_sbor_coded_call(&env, migration_id, |migration_store_id: MigrationId| -> bool { - JNINodeRustEnvironment::get_migration_database(&env, j_rust_global_context) - .is_migration_done(migration_store_id) - }) -} - -pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/p2p/mod.rs b/core-rust/state-manager/src/jni/p2p/mod.rs index 45c5fbaa61..50ddee9600 100644 --- a/core-rust/state-manager/src/jni/p2p/mod.rs +++ b/core-rust/state-manager/src/jni/p2p/mod.rs @@ -2,5 +2,4 @@ pub mod address_book; pub mod high_priority_peers; -pub mod migration_store; pub mod safety_store; \ No newline at end of file diff --git a/core-rust/state-manager/src/jni/p2p/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs index 616250fe43..d4ed46d221 100644 --- a/core-rust/state-manager/src/jni/p2p/safety_store.rs +++ b/core-rust/state-manager/src/jni/p2p/safety_store.rs @@ -97,4 +97,30 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_get( }) } +#[no_mangle] +extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_markAsMigrated( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_: ()| { + JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) + .mark_as_migrated(); + }) +} + +#[no_mangle] +extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_isMigrated( + env: JNIEnv, + _class: JClass, + j_rust_global_context: JObject, + node_id: jbyteArray, +) -> jbyteArray { + jni_sbor_coded_call(&env, node_id, |_: ()| -> bool { + JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) + .is_migrated() + }) +} + pub fn export_extern_functions() {} diff --git a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java index 0dbc0cb177..7f68f00e94 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java @@ -100,6 +100,14 @@ public Optional get() { return rocksDbSafetyStore.get().map(RocksSafetyStateStore::fromDTO).toOptional(); } + public boolean isMigrated() { + return rocksDbSafetyStore.isMigrated(); + } + + public void markAsMigrated() { + rocksDbSafetyStore.markAsMigrated(); + } + // ------------------------------------------------------------------------------------- // Conversion to DTO // ------------------------------------------------------------------------------------- diff --git a/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java index 8a186447ed..f423873db0 100644 --- a/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java +++ b/core/src/main/java/com/radixdlt/p2p/addressbook/RocksAddressBookStore.java @@ -124,6 +124,14 @@ public List getHighPriorityPeers() { .collect(ImmutableList.toImmutableList()); } + public boolean isMigrated() { + return addressBookStore.isMigrated(); + } + + public void markAsMigrated() { + addressBookStore.markAsMigrated(); + } + private static AddressBookEntryDTO toDTO(AddressBookEntry entry) { return new AddressBookEntryDTO( new NodeIdDTO(entry.getNodeId().getPublicKey()), diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java index 602a5cc59a..1fafcc0914 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java @@ -70,8 +70,6 @@ import com.radixdlt.consensus.safety.BerkeleySafetyStateStore; import com.radixdlt.consensus.safety.PersistentSafetyStateStore; import com.radixdlt.consensus.safety.RocksSafetyStateStore; -import com.radixdlt.db.RocksDbMigrationStore; -import com.radixdlt.db.StoreId; import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; import com.radixdlt.p2p.RocksDbAddressBookStore; @@ -91,70 +89,29 @@ protected void configure() {} @Provides @Singleton - BerkeleySafetyStateStore bdbSafetyStateStore( + AddressBookPersistence addressBookPersistence( RuntimeProperties properties, Serialization serialization, Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - } + @NodeStorageLocation String nodeStorageLocation, + NodeRustEnvironment environment) { + var berkeleyAddressBookStore = + new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - @Provides - @Singleton - BerkeleyAddressBookStore bdbAddressBookStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleyAddressBookStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - } - - @Provides - @Singleton - RocksDbSafetyStore rocksDbSafetyStore(Metrics metrics, NodeRustEnvironment environment) { - return RocksDbSafetyStore.create(metrics, environment); - } - - @Provides - @Singleton - RocksDbAddressBookStore rocksDbAddressBookStore( - Metrics metrics, NodeRustEnvironment environment) { - return RocksDbAddressBookStore.create(metrics, environment); - } - - @Provides - @Singleton - RocksDbHighPriorityPeersStore rocksDbHighPriorityPeersStore( - Metrics metrics, NodeRustEnvironment environment) { - return RocksDbHighPriorityPeersStore.create(metrics, environment); - } - - @Provides - @Singleton - RocksDbMigrationStore rocksDbMigrationStore(Metrics metrics, NodeRustEnvironment environment) { - return RocksDbMigrationStore.create(metrics, environment); - } - - @Provides - @Singleton - AddressBookPersistence addressBookPersistence( - RocksDbMigrationStore migrationStore, - BerkeleyAddressBookStore berkeleyAddressBookStore, - RocksAddressBookStore addressBookStore) { + var addressBookStore = + new RocksAddressBookStore( + RocksDbAddressBookStore.create(metrics, environment), + RocksDbHighPriorityPeersStore.create(metrics, environment)); try (berkeleyAddressBookStore) { - if (!migrationStore.isMigrated(StoreId.ADDRESS_BOOK)) { + if (!addressBookStore.isMigrated()) { berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); - migrationStore.migrationDone(StoreId.ADDRESS_BOOK); + addressBookStore.markAsMigrated(); } } @@ -164,14 +121,26 @@ AddressBookPersistence addressBookPersistence( @Provides @Singleton PersistentSafetyStateStore persistentSafetyStateStore( - RocksDbMigrationStore migrationStore, - BerkeleySafetyStateStore berkeleySafetyStateStore, - RocksSafetyStateStore rocksSafetyStateStore) { + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @NodeStorageLocation String nodeStorageLocation, + NodeRustEnvironment environment) { + + var berkeleySafetyStateStore = + new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + + var rocksSafetyStateStore = + new RocksSafetyStateStore(RocksDbSafetyStore.create(metrics, environment)); try (berkeleySafetyStateStore) { - if (!migrationStore.isMigrated(StoreId.SAFETY_STORE)) { + if (!rocksSafetyStateStore.isMigrated()) { berkeleySafetyStateStore.get().ifPresent(rocksSafetyStateStore::commitState); - migrationStore.migrationDone(StoreId.SAFETY_STORE); + rocksSafetyStateStore.markAsMigrated(); } } From fdf3a75cb59aed2fd7e1e7f4347b99003a5ea4ef Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Fri, 16 Aug 2024 12:45:43 +0200 Subject: [PATCH 22/40] Remove base class from tests --- .../helper/NodeRustEnvironmentBuilder.java | 13 ++++++------ .../p2p/RocksDbAddressBookStoreTest.java | 20 +++++++++++++------ .../RocksDbHighPriorityPeersStoreTest.java | 16 ++++++++++++--- .../safety/RocksDbSafetyStoreTest.java | 14 +++++++++---- .../consensus/EpochTimeoutCertTest.java | 2 +- .../rev2/NodeRustEnvironmentBuilder.java | 20 ++++--------------- .../safety/RocksSafetyStateStoreTest.java | 11 +++++++--- .../RocksAddressBookStoreTest.java | 11 +++++++--- 8 files changed, 64 insertions(+), 43 deletions(-) rename core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java => core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentBuilder.java (93%) rename core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java => core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java (89%) diff --git a/core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java b/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentBuilder.java similarity index 93% rename from core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java rename to core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentBuilder.java index ff5365dddb..5e7eeca4d6 100644 --- a/core/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentBuilder.java @@ -70,17 +70,16 @@ import com.radixdlt.protocol.ProtocolConfig; import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -public abstract class NodeRustEnvironmentTestBase { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +public final class NodeRustEnvironmentBuilder { + private NodeRustEnvironmentBuilder() { + throw new IllegalStateException("Can't construct"); + } - protected NodeRustEnvironment createNodeRustEnvironment() throws IOException { + public static NodeRustEnvironment createNodeRustEnvironment(String dbPath) { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var stateManagerDbConfig = new DatabaseBackendConfig(dbPath); final var config = new StateManagerConfig( diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java index 64e8056fdd..7e697f381c 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbAddressBookStoreTest.java @@ -69,7 +69,7 @@ import static org.junit.Assert.*; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.helper.NodeRustEnvironmentTestBase; +import com.radixdlt.helper.NodeRustEnvironmentBuilder; import com.radixdlt.lang.Option; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.p2p.PeerAddressEntryDTO.ConnectionStatus; @@ -77,14 +77,19 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbAddressBookStoreTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); -public class RocksDbAddressBookStoreTest extends NodeRustEnvironmentTestBase { private static final Random RANDOM = new Random(); @Test public void test_address_book_can_be_marked_as_migrated() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBookStore = RocksDbAddressBookStore.create( new MetricsInitializer().initialize(), nodeRustEnvironment); @@ -99,7 +104,8 @@ public void test_address_book_can_be_marked_as_migrated() throws Exception { @Test public void test_address_book_entries_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBookStore = RocksDbAddressBookStore.create( new MetricsInitializer().initialize(), nodeRustEnvironment); @@ -133,7 +139,8 @@ public void test_address_book_entries_can_be_saved_and_restored() throws Excepti @Test public void test_address_book_entry_can_be_added_and_removed() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBookStore = RocksDbAddressBookStore.create( new MetricsInitializer().initialize(), nodeRustEnvironment); @@ -168,7 +175,8 @@ public void test_address_book_entry_can_be_added_and_removed() throws Exception @Test public void test_address_book_can_be_reset() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBookStore = RocksDbAddressBookStore.create( new MetricsInitializer().initialize(), nodeRustEnvironment); diff --git a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java index dd32f410fd..80a00c4235 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStoreTest.java @@ -66,15 +66,21 @@ import static org.junit.Assert.*; -import com.radixdlt.helper.NodeRustEnvironmentTestBase; +import com.radixdlt.crypto.ECKeyPair; +import com.radixdlt.helper.NodeRustEnvironmentBuilder; import com.radixdlt.monitoring.MetricsInitializer; import java.util.List; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbHighPriorityPeersStoreTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); -public class RocksDbHighPriorityPeersStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_high_priority_peers_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var highPriorityPeersStore = RocksDbHighPriorityPeersStore.create( new MetricsInitializer().initialize(), nodeRustEnvironment); @@ -112,4 +118,8 @@ public void test_high_priority_peers_can_be_saved_and_restored() throws Exceptio assertTrue(empty.isEmpty()); } } + + private static NodeIdDTO newNodeId() { + return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); + } } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java index 84dfdf2b5f..ae28393224 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java +++ b/core-rust-bridge/src/test/java/com/radixdlt/safety/RocksDbSafetyStoreTest.java @@ -68,17 +68,22 @@ import com.google.common.primitives.Bytes; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.helper.NodeRustEnvironmentTestBase; +import com.radixdlt.helper.NodeRustEnvironmentBuilder; import com.radixdlt.lang.Option; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.rev2.ComponentAddress; import java.util.Collections; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksDbSafetyStoreTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); -public class RocksDbSafetyStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_safety_store_can_be_marked_as_migrated() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var safetyStore = RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); @@ -92,7 +97,8 @@ public void test_safety_store_can_be_marked_as_migrated() throws Exception { @Test public void test_safety_state_can_be_saved_and_restored() throws Exception { - try (var nodeRustEnvironment = createNodeRustEnvironment()) { + try (var nodeRustEnvironment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var safetyStore = RocksDbSafetyStore.create(new MetricsInitializer().initialize(), nodeRustEnvironment); var address = diff --git a/core/src/integration/java/com/radixdlt/integration/targeted/consensus/EpochTimeoutCertTest.java b/core/src/integration/java/com/radixdlt/integration/targeted/consensus/EpochTimeoutCertTest.java index d382319319..08ab1bb9fd 100644 --- a/core/src/integration/java/com/radixdlt/integration/targeted/consensus/EpochTimeoutCertTest.java +++ b/core/src/integration/java/com/radixdlt/integration/targeted/consensus/EpochTimeoutCertTest.java @@ -132,7 +132,7 @@ public void no_byzantine_event_occurs_on_epoch_tc_event() { test.runUntilMessage( proposalAtRound(ROUNDS_PER_EPOCH + 2), true, - 10 * NUM_NODES * NUM_NODES * ((int) ROUNDS_PER_EPOCH)); + 10 * NUM_NODES * NUM_NODES * ROUNDS_PER_EPOCH); // Run for a while more and verify that no byzantine issues occur test.runForCount(40000); } diff --git a/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java b/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java similarity index 89% rename from core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java rename to core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java index 3768ab9e08..1739666981 100644 --- a/core-rust-bridge/src/test/java/com/radixdlt/helper/NodeRustEnvironmentTestBase.java +++ b/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java @@ -62,27 +62,19 @@ * permissions under this License. */ -package com.radixdlt.helper; +package com.radixdlt.rev2; -import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.environment.*; import com.radixdlt.lang.Option; import com.radixdlt.mempool.RustMempoolConfig; -import com.radixdlt.p2p.NodeIdDTO; import com.radixdlt.protocol.ProtocolConfig; -import com.radixdlt.rev2.NetworkDefinition; import com.radixdlt.transaction.LedgerSyncLimitsConfig; -import java.io.IOException; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -public abstract class NodeRustEnvironmentTestBase { - @Rule public TemporaryFolder folder = new TemporaryFolder(); - - protected NodeRustEnvironment createNodeRustEnvironment() throws IOException { +public class NodeRustEnvironmentBuilder { + public static NodeRustEnvironment createNodeRustEnvironment(String dbPath) { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; - final var stateManagerDbConfig = new DatabaseBackendConfig(folder.newFolder().getPath()); + final var stateManagerDbConfig = new DatabaseBackendConfig(dbPath); final var config = new StateManagerConfig( @@ -109,8 +101,4 @@ protected NodeRustEnvironment createNodeRustEnvironment() throws IOException { // gracefully anyway. config); } - - protected static NodeIdDTO newNodeId() { - return new NodeIdDTO(ECKeyPair.generateNew().getPublicKey()); - } } diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index 3a956c1805..08d060ec9f 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -67,15 +67,19 @@ import static org.junit.Assert.*; import com.radixdlt.consensus.bft.BFTValidatorId; -import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.rev2.NodeRustEnvironmentBuilder; import com.radixdlt.safety.RocksDbSafetyStore; import java.io.IOException; import java.util.function.Consumer; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksSafetyStateStoreTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); -public class RocksSafetyStateStoreTest extends NodeRustEnvironmentTestBase { @Test public void test_address_book_entries_can_be_saved_and_restored() { runTest( @@ -95,7 +99,8 @@ public void test_address_book_entries_can_be_saved_and_restored() { } private void runTest(Consumer test) { - try (var environment = createNodeRustEnvironment()) { + try (var environment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var safetyStore = RocksDbSafetyStore.create(newMetrics(), environment); try (var underTest = new RocksSafetyStateStore(safetyStore)) { test.accept(underTest); diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index 03fd4a7e4d..bc4aab960f 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -69,22 +69,26 @@ import com.google.common.collect.ImmutableSet; import com.radixdlt.crypto.ECDSASecp256k1PublicKey; import com.radixdlt.crypto.ECKeyPair; -import com.radixdlt.helper.NodeRustEnvironmentTestBase; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.p2p.*; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.FailedHandshake; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.LatestConnectionStatus; +import com.radixdlt.rev2.NodeRustEnvironmentBuilder; import java.io.IOException; import java.time.Instant; import java.util.Optional; import java.util.Random; import java.util.function.Consumer; import java.util.stream.IntStream; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksAddressBookStoreTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); -public class RocksAddressBookStoreTest extends NodeRustEnvironmentTestBase { private static final Random RANDOM = new Random(); @Test @@ -128,7 +132,8 @@ public void test_address_book_entries_can_be_saved_and_restored() { } private void runTest(Consumer test) { - try (var environment = createNodeRustEnvironment()) { + try (var environment = + NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBook = RocksDbAddressBookStore.create(newMetrics(), environment); var peersStore = RocksDbHighPriorityPeersStore.create(newMetrics(), environment); try (var underTest = new RocksAddressBookStore(addressBook, peersStore)) { From a41b6c9d3413b88f5938c7e707ffa46dacf1c441 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 21 Aug 2024 12:31:48 +0200 Subject: [PATCH 23/40] Fix conversion from DTO to preserve set ordering. --- .../com/radixdlt/consensus/safety/RocksSafetyStateStore.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java index 7f68f00e94..49e009e45b 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java @@ -64,6 +64,7 @@ package com.radixdlt.consensus.safety; +import com.google.common.collect.ImmutableSet; import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.BFTValidator; @@ -194,7 +195,7 @@ private static NextEpochDTO toDTO(NextEpoch epoch) { epoch.getEpoch(), epoch.getValidators().stream() .map(RocksSafetyStateStore::toDTO) - .collect(Collectors.toSet())); + .collect(ImmutableSet.toImmutableSet())); } private static BFTValidatorDTO toDTO(BFTValidator bftValidator) { @@ -278,7 +279,7 @@ private static LedgerHashes fromDTO(com.radixdlt.statecomputer.commit.LedgerHash private static NextEpoch fromDTO(NextEpochDTO dto) { return NextEpoch.create( dto.epoch(), - dto.validators().stream().map(RocksSafetyStateStore::fromDTO).collect(Collectors.toSet())); + dto.validators().stream().map(RocksSafetyStateStore::fromDTO).collect(ImmutableSet.toImmutableSet())); } private static BFTValidator fromDTO(BFTValidatorDTO dto) { From fe1fad4fe7a7d1a3ddd299ef50771572d92f5c81 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 21 Aug 2024 12:55:07 +0200 Subject: [PATCH 24/40] Fix formatting --- .../com/radixdlt/consensus/safety/RocksSafetyStateStore.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java index 49e009e45b..8e88acecd6 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java @@ -279,7 +279,9 @@ private static LedgerHashes fromDTO(com.radixdlt.statecomputer.commit.LedgerHash private static NextEpoch fromDTO(NextEpochDTO dto) { return NextEpoch.create( dto.epoch(), - dto.validators().stream().map(RocksSafetyStateStore::fromDTO).collect(ImmutableSet.toImmutableSet())); + dto.validators().stream() + .map(RocksSafetyStateStore::fromDTO) + .collect(ImmutableSet.toImmutableSet())); } private static BFTValidator fromDTO(BFTValidatorDTO dto) { From 25c13ef3a332a07c6929e9f7669b73456651669f Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 21 Aug 2024 13:57:39 +0200 Subject: [PATCH 25/40] Reuse NodePersistenceModule in FunctionalRadixNodeModule --- .../rev2/modules/AddressBookModule.java | 113 ++++++++++++++++++ .../rev2/modules/NodePersistenceModule.java | 45 ++----- .../rev2/modules/REv2StateManagerModule.java | 2 +- .../modules/FunctionalRadixNodeModule.java | 26 +--- 4 files changed, 130 insertions(+), 56 deletions(-) create mode 100644 core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java diff --git a/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java b/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java new file mode 100644 index 0000000000..d9fdda50b1 --- /dev/null +++ b/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java @@ -0,0 +1,113 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.rev2.modules; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.p2p.RocksDbAddressBookStore; +import com.radixdlt.p2p.RocksDbHighPriorityPeersStore; +import com.radixdlt.p2p.addressbook.AddressBookPersistence; +import com.radixdlt.p2p.addressbook.BerkeleyAddressBookStore; +import com.radixdlt.p2p.addressbook.RocksAddressBookStore; +import com.radixdlt.serialization.Serialization; +import com.radixdlt.store.BerkeleyDbDefaults; +import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.utils.properties.RuntimeProperties; + +public class AddressBookModule extends AbstractModule { + @Provides + @Singleton + AddressBookPersistence addressBookPersistence( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @NodeStorageLocation String nodeStorageLocation, + NodeRustEnvironment environment) { + var berkeleyAddressBookStore = + new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + + var addressBookStore = + new RocksAddressBookStore( + RocksDbAddressBookStore.create(metrics, environment), + RocksDbHighPriorityPeersStore.create(metrics, environment)); + + try (berkeleyAddressBookStore) { + if (!addressBookStore.isMigrated()) { + berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); + addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); + addressBookStore.markAsMigrated(); + } + } + + return addressBookStore; + } +} diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java index 1fafcc0914..509241b9c8 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java @@ -72,11 +72,6 @@ import com.radixdlt.consensus.safety.RocksSafetyStateStore; import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; -import com.radixdlt.p2p.RocksDbAddressBookStore; -import com.radixdlt.p2p.RocksDbHighPriorityPeersStore; -import com.radixdlt.p2p.addressbook.AddressBookPersistence; -import com.radixdlt.p2p.addressbook.BerkeleyAddressBookStore; -import com.radixdlt.p2p.addressbook.RocksAddressBookStore; import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.serialization.Serialization; import com.radixdlt.store.BerkeleyDbDefaults; @@ -84,38 +79,22 @@ import com.radixdlt.utils.properties.RuntimeProperties; public class NodePersistenceModule extends AbstractModule { - @Override - protected void configure() {} + private final AddressBookPolicy policy; - @Provides - @Singleton - AddressBookPersistence addressBookPersistence( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation, - NodeRustEnvironment environment) { - var berkeleyAddressBookStore = - new BerkeleyAddressBookStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + public enum AddressBookPolicy { + ENABLED, + DISABLED + } - var addressBookStore = - new RocksAddressBookStore( - RocksDbAddressBookStore.create(metrics, environment), - RocksDbHighPriorityPeersStore.create(metrics, environment)); + public NodePersistenceModule(AddressBookPolicy policy) { + this.policy = policy; + } - try (berkeleyAddressBookStore) { - if (!addressBookStore.isMigrated()) { - berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); - addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); - addressBookStore.markAsMigrated(); - } + @Override + protected void configure() { + if (policy == AddressBookPolicy.ENABLED) { + install(new AddressBookModule()); } - - return addressBookStore; } @Provides diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index 47e08b2ceb..f12a359c32 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -321,7 +321,7 @@ EventProcessor onInsertUpdatePersistVertexStore( if (installPersistence) { // Moved here for convenience performing migration - install(new NodePersistenceModule()); + install(new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.ENABLED)); } } diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index d4c05ff2e2..9d093df93e 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -66,16 +66,12 @@ import com.google.inject.AbstractModule; import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; import com.radixdlt.consensus.*; import com.radixdlt.consensus.ProposalLimitsConfig; import com.radixdlt.consensus.bft.*; import com.radixdlt.consensus.epoch.EpochsConsensusModule; import com.radixdlt.consensus.liveness.ProposalGenerator; -import com.radixdlt.consensus.safety.PersistentSafetyStateStore; -import com.radixdlt.consensus.safety.RocksSafetyStateStore; import com.radixdlt.consensus.sync.BFTSyncPatienceMillis; import com.radixdlt.environment.*; import com.radixdlt.genesis.RawGenesisDataWithHash; @@ -84,9 +80,7 @@ import com.radixdlt.ledger.MockedLedgerRecoveryModule; import com.radixdlt.mempool.*; import com.radixdlt.modules.StateComputerConfig.*; -import com.radixdlt.monitoring.Metrics; import com.radixdlt.rev2.modules.*; -import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.statecomputer.MockedMempoolStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerModule; import com.radixdlt.statecomputer.MockedStateComputerWithEpochsModule; @@ -127,6 +121,7 @@ static NodeStorageConfig file(File folder) { public enum SafetyRecoveryConfig { MOCKED, REAL, + REAL_WITH_ADDRESS_BOOK } public static final class ConsensusConfig { @@ -320,22 +315,9 @@ public void configure() { switch (this.safetyRecoveryConfig) { case MOCKED -> install(new MockedSafetyStoreModule()); case REAL -> install( - new AbstractModule() { - @Override - protected void configure() { - bind(PersistentSafetyStateStore.class).to(RocksSafetyStateStore.class); - Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) - .addBinding() - .to(RocksSafetyStateStore.class); - } - - @Provides - @Singleton - RocksDbSafetyStore rocksDbSafetyStore( - Metrics metrics, NodeRustEnvironment environment) { - return RocksDbSafetyStore.create(metrics, environment); - } - }); + new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.DISABLED)); + case REAL_WITH_ADDRESS_BOOK -> install( + new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.ENABLED)); } // Consensus From b99765ba2445e5ad666179e7857664891c243e63 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 21 Aug 2024 16:20:57 +0200 Subject: [PATCH 26/40] Add migration tests --- .../safety/SafetyStoreMigrationTest.java | 171 ++++++++++++++++++ .../addressbook/AddressBookMigrationTest.java | 164 +++++++++++++++++ .../RocksAddressBookStoreTest.java | 2 +- 3 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java create mode 100644 core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java diff --git a/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java new file mode 100644 index 0000000000..eb9080046f --- /dev/null +++ b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java @@ -0,0 +1,171 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.consensus.safety; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.inject.*; +import com.google.inject.Module; +import com.radixdlt.consensus.bft.BFTValidatorId; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.rev2.NodeRustEnvironmentBuilder; +import com.radixdlt.rev2.modules.NodePersistenceModule; +import com.radixdlt.rev2.modules.NodePersistenceModule.AddressBookPolicy; +import com.radixdlt.serialization.Serialization; +import com.radixdlt.serialization.core.ClasspathScanningSerializationPolicy; +import com.radixdlt.serialization.core.ClasspathScanningSerializerIds; +import com.radixdlt.store.BerkeleyDbDefaults; +import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.utils.properties.RuntimeProperties; +import java.io.IOException; +import java.util.Map; +import org.apache.commons.cli.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class SafetyStoreMigrationTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private Module createCommonModule() { + return new AbstractModule() { + @Provides + @Singleton + @NodeStorageLocation + String nodeStorageLocation() { + return folder.getRoot().getPath(); + } + + @Provides + @Singleton + NodeRustEnvironment nodeRustEnvironment() throws IOException { + return NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath()); + } + + @Provides + @Singleton + Metrics metrics() { + return new MetricsInitializer().initialize(); + } + + @Provides + @Singleton + Serialization serialization() { + return Serialization.create( + ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()); + } + + @Provides + @Singleton + RuntimeProperties properties() throws ParseException { + return RuntimeProperties.defaultWithOverrides(Map.of()); + } + }; + } + + private Module createBerkeleyModule() { + return new AbstractModule() { + @Provides + @Singleton + BerkeleySafetyStateStore safetyStore( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @NodeStorageLocation String nodeStorageLocation) { + return new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + }; + } + + @Test + public void address_book_migrates_successfully() { + var injector = + Guice.createInjector( + createCommonModule(), + new NodePersistenceModule(AddressBookPolicy.DISABLED), + createBerkeleyModule()); + var safetyState = SafetyState.initialState(BFTValidatorId.random()); + ; + + // Create some entries in the Berkeley safety store + try (var berkeleySafetyStateStore = injector.getInstance(BerkeleySafetyStateStore.class)) { + berkeleySafetyStateStore.commitState(safetyState); + } + + // Instantiation of the PersistentSafetyStateStore will trigger the migration + var safetyStateStore = injector.getInstance(PersistentSafetyStateStore.class); + var storedState = safetyStateStore.get(); + + assertTrue(storedState.isPresent()); + assertEquals(safetyState, storedState.get()); + } +} diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java new file mode 100644 index 0000000000..8bc71c3155 --- /dev/null +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java @@ -0,0 +1,164 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.p2p.addressbook; + +import static com.radixdlt.p2p.addressbook.RocksAddressBookStoreTest.*; +import static org.junit.Assert.assertEquals; + +import com.google.inject.*; +import com.google.inject.Module; +import com.radixdlt.environment.NodeRustEnvironment; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.monitoring.MetricsInitializer; +import com.radixdlt.rev2.NodeRustEnvironmentBuilder; +import com.radixdlt.rev2.modules.AddressBookModule; +import com.radixdlt.serialization.Serialization; +import com.radixdlt.serialization.core.ClasspathScanningSerializationPolicy; +import com.radixdlt.serialization.core.ClasspathScanningSerializerIds; +import com.radixdlt.store.BerkeleyDbDefaults; +import com.radixdlt.store.NodeStorageLocation; +import com.radixdlt.utils.properties.RuntimeProperties; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.cli.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class AddressBookMigrationTest { + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private Module createCommonModule() { + return new AbstractModule() { + @Provides + @Singleton + @NodeStorageLocation + String nodeStorageLocation() { + return folder.getRoot().getPath(); + } + + @Provides + @Singleton + NodeRustEnvironment nodeRustEnvironment() throws IOException { + return NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath()); + } + + @Provides + @Singleton + Metrics metrics() { + return new MetricsInitializer().initialize(); + } + + @Provides + @Singleton + Serialization serialization() { + return Serialization.create( + ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()); + } + + @Provides + @Singleton + RuntimeProperties properties() throws ParseException { + return RuntimeProperties.defaultWithOverrides(Map.of()); + } + }; + } + + private Module createBerkeleyModule() { + return new AbstractModule() { + @Provides + @Singleton + BerkeleyAddressBookStore addressBookPersistence( + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + @NodeStorageLocation String nodeStorageLocation) { + return new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } + }; + } + + @Test + public void address_book_migrates_successfully() { + var injector = + Guice.createInjector(createCommonModule(), new AddressBookModule(), createBerkeleyModule()); + var addresses = Set.of(newAddressBookEntry(1), newAddressBookEntry(2), newAddressBookEntry(3)); + + // Create some entries in the Berkeley address book + try (var berkeleyAddressBook = injector.getInstance(BerkeleyAddressBookStore.class)) { + addresses.forEach(berkeleyAddressBook::upsertEntry); + } + + // Instantiation of the AddressBook will trigger the migration + var addressBook = injector.getInstance(AddressBookPersistence.class); + assertEquals(new HashSet<>(addressBook.getAllEntries()), addresses); + } +} diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index bc4aab960f..66c25f0fe3 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -148,7 +148,7 @@ private static Metrics newMetrics() { return new MetricsInitializer().initialize(); } - private static AddressBookEntry newAddressBookEntry(int id) { + public static AddressBookEntry newAddressBookEntry(int id) { var pubKey = ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey(); var bannedUntil = newBannedUntil(); From bcc2576372cb6579f6fc98a9a2b59bb417ce7664 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 21 Aug 2024 16:29:38 +0200 Subject: [PATCH 27/40] Minor cleanup --- .../com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java index eb9080046f..4e3bc187b6 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java @@ -154,7 +154,6 @@ public void address_book_migrates_successfully() { new NodePersistenceModule(AddressBookPolicy.DISABLED), createBerkeleyModule()); var safetyState = SafetyState.initialState(BFTValidatorId.random()); - ; // Create some entries in the Berkeley safety store try (var berkeleySafetyStateStore = injector.getInstance(BerkeleySafetyStateStore.class)) { From 1ba58126582f1748e516026c97a90835629b7430 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Tue, 27 Aug 2024 13:56:21 +0200 Subject: [PATCH 28/40] Address review comments --- .../handlers/state_consensus_manager.rs | 2 +- .../core-api-server/src/core_api/helpers.rs | 2 +- core-rust/node-common/src/java/utils.rs | 3 +- core-rust/node-common/src/lib.rs | 2 +- core-rust/node-common/src/store/mod.rs | 2 +- core-rust/node-common/src/store/rocks_db.rs | 15 +- core-rust/p2p/src/address_book_components.rs | 5 +- core-rust/p2p/src/column_families.rs | 6 +- core-rust/p2p/src/rocks_db.rs | 83 ++++++----- core-rust/p2p/src/traits.rs | 6 +- core-rust/p2p/src/typed_cf_api.rs | 2 +- core-rust/state-manager/Cargo.toml | 1 - core-rust/state-manager/src/jni/mod.rs | 2 +- .../src/jni/node_rust_environment.rs | 5 +- .../state-manager/src/jni/p2p/address_book.rs | 11 +- .../src/jni/p2p/high_priority_peers.rs | 7 +- core-rust/state-manager/src/jni/p2p/mod.rs | 2 +- .../state-manager/src/jni/p2p/safety_store.rs | 5 +- .../src/jni/test_state_reader.rs | 2 +- core-rust/state-manager/src/staging/cache.rs | 2 +- core-rust/state-manager/src/staging/mod.rs | 2 +- core-rust/state-manager/src/staging/result.rs | 2 +- core-rust/state-manager/src/state_manager.rs | 2 +- core-rust/state-manager/src/store/codecs.rs | 2 +- .../src/store/column_families.rs | 37 +---- .../src/store/historical_state.rs | 4 +- core-rust/state-manager/src/store/mod.rs | 6 +- .../state-manager/src/store/proofs_gc.rs | 4 +- core-rust/state-manager/src/store/rocks_db.rs | 77 +++++------ .../rev2/modules/AddressBookModule.java | 38 +++-- ... => PersistentSafetyStateStoreModule.java} | 52 +++---- .../rev2/modules/REv2StateManagerModule.java | 3 +- .../modules/FunctionalRadixNodeModule.java | 6 +- ...er.java => NodeRustEnvironmentHelper.java} | 2 +- .../safety/RocksSafetyStateStoreTest.java | 4 +- .../safety/SafetyStoreMigrationTest.java | 110 ++++++--------- .../addressbook/AddressBookMigrationTest.java | 130 +++++++++--------- .../RocksAddressBookStoreTest.java | 35 ++++- .../p2p/test/P2PTestNetworkRunner.java | 4 + 39 files changed, 337 insertions(+), 348 deletions(-) rename core/src/main/java/com/radixdlt/rev2/modules/{NodePersistenceModule.java => PersistentSafetyStateStoreModule.java} (86%) rename core/src/test-core/java/com/radixdlt/rev2/{NodeRustEnvironmentBuilder.java => NodeRustEnvironmentHelper.java} (99%) diff --git a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs index 28b6806ac4..e040ea8fde 100644 --- a/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs +++ b/core-rust/core-api-server/src/core_api/handlers/state_consensus_manager.rs @@ -1,8 +1,8 @@ use crate::core_api::*; use crate::engine_prelude::*; -use state_manager::protocol::ProtocolVersionName; use node_common::store::rocks_db::ReadableRocks; +use state_manager::protocol::ProtocolVersionName; use state_manager::store::rocks_db::StateManagerDatabase; use std::ops::Deref; diff --git a/core-rust/core-api-server/src/core_api/helpers.rs b/core-rust/core-api-server/src/core_api/helpers.rs index aba7c23884..6d68a4e62a 100644 --- a/core-rust/core-api-server/src/core_api/helpers.rs +++ b/core-rust/core-api-server/src/core_api/helpers.rs @@ -1,6 +1,6 @@ use crate::engine_prelude::*; -use serde::Serialize; use node_common::store::rocks_db::ReadableRocks; +use serde::Serialize; use state_manager::store::rocks_db::StateManagerDatabase; use state_manager::store::traits::*; use state_manager::LedgerHeader; diff --git a/core-rust/node-common/src/java/utils.rs b/core-rust/node-common/src/java/utils.rs index a6bcb45874..7a5012ee59 100644 --- a/core-rust/node-common/src/java/utils.rs +++ b/core-rust/node-common/src/java/utils.rs @@ -123,8 +123,7 @@ pub fn jni_raw_sbor_fallible_call( method: impl FnOnce(Vec) -> JavaResult, ) -> jbyteArray { jni_call(env, || { - let result = jni_jbytearray_to_vector(env, encoded_request) - .and_then(method); + let result = jni_jbytearray_to_vector(env, encoded_request).and_then(method); jni_slice_to_jbytearray(env, &result.to_java().unwrap()) }) .unwrap_or_else(std::ptr::null_mut) diff --git a/core-rust/node-common/src/lib.rs b/core-rust/node-common/src/lib.rs index 3a408dcfae..2f898a4b4b 100644 --- a/core-rust/node-common/src/lib.rs +++ b/core-rust/node-common/src/lib.rs @@ -70,8 +70,8 @@ pub mod jni; pub mod locks; pub mod metrics; pub mod scheduler; -pub mod utils; pub mod store; +pub mod utils; pub use rocksdb; pub(crate) mod engine_prelude { diff --git a/core-rust/node-common/src/store/mod.rs b/core-rust/node-common/src/store/mod.rs index 788d25fb73..561e3addfa 100644 --- a/core-rust/node-common/src/store/mod.rs +++ b/core-rust/node-common/src/store/mod.rs @@ -1,2 +1,2 @@ pub mod rocks_db; -pub mod typed_cf_api; \ No newline at end of file +pub mod typed_cf_api; diff --git a/core-rust/node-common/src/store/rocks_db.rs b/core-rust/node-common/src/store/rocks_db.rs index af94113182..9035adb393 100644 --- a/core-rust/node-common/src/store/rocks_db.rs +++ b/core-rust/node-common/src/store/rocks_db.rs @@ -65,8 +65,7 @@ use crate::engine_prelude::*; use rocksdb::checkpoint::Checkpoint; use rocksdb::{ - AsColumnFamilyRef, ColumnFamily, DBPinnableSlice, - IteratorMode, Snapshot, WriteBatch, DB, + AsColumnFamilyRef, ColumnFamily, DBPinnableSlice, IteratorMode, Snapshot, WriteBatch, DB, }; use std::path::PathBuf; @@ -93,7 +92,7 @@ pub trait ReadableRocks { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_>; + ) -> Box + '_>; /// Gets a single value by key. fn get_pinned_cf( @@ -110,7 +109,7 @@ pub trait ReadableRocks { /// TODO(when the rustc feature mentioned above becomes stable): get rid of the `<'a>`. fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>>; } @@ -151,7 +150,7 @@ impl ReadableRocks for DirectRocks { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.db .iterator_cf(cf, mode) @@ -169,7 +168,7 @@ impl ReadableRocks for DirectRocks { fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>> { self.db .multi_get_cf(keys) @@ -237,7 +236,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> { &self, cf: &impl AsColumnFamilyRef, mode: IteratorMode, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.snapshot .iterator_cf(cf, mode) @@ -257,7 +256,7 @@ impl<'db> ReadableRocks for SnapshotRocks<'db> { fn multi_get_cf<'a>( &'a self, - keys: impl IntoIterator)>, + keys: impl IntoIterator)>, ) -> Vec>> { self.snapshot .multi_get_cf(keys) diff --git a/core-rust/p2p/src/address_book_components.rs b/core-rust/p2p/src/address_book_components.rs index 5ad18f0e49..55eea9ae5f 100644 --- a/core-rust/p2p/src/address_book_components.rs +++ b/core-rust/p2p/src/address_book_components.rs @@ -1,6 +1,5 @@ use crate::engine_prelude::Sbor; - /// The ID of the node stored in the Address book (Secp256k1 public key) #[derive(Clone, Copy, Sbor)] pub struct AddressBookNodeId(pub [u8; AddressBookNodeId::LENGTH]); @@ -18,8 +17,8 @@ impl AddressBookNodeId { } /// Timestamp of the various peer-related events -// At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and -// do something like shown below to transparently convert to/from internal representation +// At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and +// do something like shown below to transparently convert to/from internal representation // (once there will be real usage at Rust side). // #[sbor( // as_type = "i64", diff --git a/core-rust/p2p/src/column_families.rs b/core-rust/p2p/src/column_families.rs index b8f92016d8..7b2374175e 100644 --- a/core-rust/p2p/src/column_families.rs +++ b/core-rust/p2p/src/column_families.rs @@ -1,10 +1,8 @@ -use node_common::store::typed_cf_api::SborDbCodec; -use node_common::store::typed_cf_api::{ - DefaultCf, DirectDbCodec, UnitDbCodec, -}; use crate::address_book_components::AddressBookNodeId; use crate::migration::MigrationStatus; use crate::typed_cf_api::AddressBookNodeIdDbCodec; +use node_common::store::typed_cf_api::SborDbCodec; +use node_common::store::typed_cf_api::{DefaultCf, DirectDbCodec, UnitDbCodec}; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; diff --git a/core-rust/p2p/src/rocks_db.rs b/core-rust/p2p/src/rocks_db.rs index de70abcefc..cedc90f478 100644 --- a/core-rust/p2p/src/rocks_db.rs +++ b/core-rust/p2p/src/rocks_db.rs @@ -64,15 +64,15 @@ use std::path::PathBuf; -use node_common::rocksdb::{ - ColumnFamilyDescriptor, DB, Options, -}; +use node_common::rocksdb::{ColumnFamilyDescriptor, Options, DB}; -use crate::engine_prelude::*; use crate::address_book_components::AddressBookNodeId; -use crate::column_families::{AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf}; -use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, SafetyStateStore}; +use crate::column_families::{ + AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf, +}; +use crate::engine_prelude::*; use crate::migration::MigrationStatus; +use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, SafetyStateStore}; use node_common::store::rocks_db::*; use node_common::store::typed_cf_api::*; @@ -101,10 +101,8 @@ const ALL_ADDRESS_BOOK_COLUMN_FAMILIES: [&str; 3] = [ MigrationStatusCf::DEFAULT_NAME, ]; -const ALL_SAFETY_STORE_COLUMN_FAMILIES: [&str; 2] = [ - SafetyStoreCf::DEFAULT_NAME, - MigrationStatusCf::DEFAULT_NAME, -]; +const ALL_SAFETY_STORE_COLUMN_FAMILIES: [&str; 2] = + [SafetyStoreCf::DEFAULT_NAME, MigrationStatusCf::DEFAULT_NAME]; pub type ActualAddressBookDatabase = AddressBookDatabase; pub type ActualSafetyStoreDatabase = SafetyStoreDatabase; @@ -139,21 +137,21 @@ fn open_rw_context(db: &R) -> TypedDbContext ActualAddressBookDatabase { + pub fn new(root_path: PathBuf) -> ActualAddressBookDatabase { AddressBookDatabase { - rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_ADDRESS_BOOK_COLUMN_FAMILIES) }, + rocks: DirectRocks { + db: new_rocks_db(root_path, &ALL_ADDRESS_BOOK_COLUMN_FAMILIES), + }, } } } impl ActualSafetyStoreDatabase { - pub fn new( - root_path: PathBuf, - ) -> ActualSafetyStoreDatabase { + pub fn new(root_path: PathBuf) -> ActualSafetyStoreDatabase { ActualSafetyStoreDatabase { - rocks: DirectRocks { db: new_rocks_db(root_path, &ALL_SAFETY_STORE_COLUMN_FAMILIES) }, + rocks: DirectRocks { + db: new_rocks_db(root_path, &ALL_SAFETY_STORE_COLUMN_FAMILIES), + }, } } } @@ -184,53 +182,64 @@ impl AddressBookStore for AddressBookDatabase { } fn get_all(&self) -> Vec> { - open_rw_context(&self.rocks).cf(AddressBookCf) - .get_all() + open_rw_context(&self.rocks).cf(AddressBookCf).get_all() } - + fn is_migrated(&self) -> bool { - open_rw_context(&self.rocks).cf(MigrationStatusCf) - .get(&()).is_some() + open_rw_context(&self.rocks) + .cf(MigrationStatusCf) + .get(&()) + .is_some() } - + fn mark_as_migrated(&self) { - open_rw_context(&self.rocks).cf(MigrationStatusCf) + open_rw_context(&self.rocks) + .cf(MigrationStatusCf) .put(&(), &MigrationStatus::Completed) } } impl HighPriorityPeersStore for AddressBookDatabase { - fn upsert_all_peers(&self, peers: &[u8]) { - open_rw_context(&self.rocks).cf(HighPriorityPeersCf).put(&(), &peers.to_vec()); + fn upsert_all_high_priority_peers(&self, peers: &[u8]) { + open_rw_context(&self.rocks) + .cf(HighPriorityPeersCf) + .put(&(), &peers.to_vec()); } - fn get_all_peers(&self) -> Option> { - open_rw_context(&self.rocks).cf(HighPriorityPeersCf) + fn get_all_high_priority_peers(&self) -> Option> { + open_rw_context(&self.rocks) + .cf(HighPriorityPeersCf) .get(&()) } fn reset_high_priority_peers(&self) { - open_rw_context(&self.rocks).cf(HighPriorityPeersCf).delete(&()); + open_rw_context(&self.rocks) + .cf(HighPriorityPeersCf) + .delete(&()); } } impl SafetyStateStore for SafetyStoreDatabase { fn upsert_safety_state(&self, safety_state: &[u8]) { - open_rw_context(&self.rocks).cf(SafetyStoreCf).put(&(), &safety_state.to_vec()); + open_rw_context(&self.rocks) + .cf(SafetyStoreCf) + .put(&(), &safety_state.to_vec()); } fn get_safety_state(&self) -> Option> { - open_rw_context(&self.rocks).cf(SafetyStoreCf) - .get(&()) + open_rw_context(&self.rocks).cf(SafetyStoreCf).get(&()) } fn is_migrated(&self) -> bool { - open_rw_context(&self.rocks).cf(MigrationStatusCf) - .get(&()).is_some() + open_rw_context(&self.rocks) + .cf(MigrationStatusCf) + .get(&()) + .is_some() } - + fn mark_as_migrated(&self) { - open_rw_context(&self.rocks).cf(MigrationStatusCf) + open_rw_context(&self.rocks) + .cf(MigrationStatusCf) .put(&(), &MigrationStatus::Completed) } } diff --git a/core-rust/p2p/src/traits.rs b/core-rust/p2p/src/traits.rs index dd5218b9cd..d6bb755d21 100644 --- a/core-rust/p2p/src/traits.rs +++ b/core-rust/p2p/src/traits.rs @@ -73,15 +73,15 @@ pub mod node { fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; fn reset(&self); - fn get_all(&self, ) -> Vec>; + fn get_all(&self) -> Vec>; fn is_migrated(&self) -> bool; fn mark_as_migrated(&self); } // At Java side it is represented as part of AddressBookStore pub trait HighPriorityPeersStore { - fn upsert_all_peers(&self, peers: &[u8]); - fn get_all_peers(&self) -> Option>; + fn upsert_all_high_priority_peers(&self, peers: &[u8]); + fn get_all_high_priority_peers(&self) -> Option>; fn reset_high_priority_peers(&self); } diff --git a/core-rust/p2p/src/typed_cf_api.rs b/core-rust/p2p/src/typed_cf_api.rs index 0325f5bc30..c10f7235da 100644 --- a/core-rust/p2p/src/typed_cf_api.rs +++ b/core-rust/p2p/src/typed_cf_api.rs @@ -64,7 +64,7 @@ use crate::address_book_components::AddressBookNodeId; use crate::engine_prelude::*; -use node_common::store::typed_cf_api::{DbCodec, BoundedDbCodec}; +use node_common::store::typed_cf_api::{BoundedDbCodec, DbCodec}; #[derive(Clone, Default)] pub struct AddressBookNodeIdDbCodec {} diff --git a/core-rust/state-manager/Cargo.toml b/core-rust/state-manager/Cargo.toml index 44959da920..304891533f 100644 --- a/core-rust/state-manager/Cargo.toml +++ b/core-rust/state-manager/Cargo.toml @@ -29,7 +29,6 @@ blake2 = { workspace = true } hex = { workspace = true } rand = { workspace = true } -#rocksdb = { version = "=0.21.0" } lru = { version = "=0.8.1", default-features = false } slotmap = { version = "=1.0.6" } im = { version = "=15.1.0" } diff --git a/core-rust/state-manager/src/jni/mod.rs b/core-rust/state-manager/src/jni/mod.rs index e018db38fc..ccb138dfd9 100644 --- a/core-rust/state-manager/src/jni/mod.rs +++ b/core-rust/state-manager/src/jni/mod.rs @@ -66,6 +66,7 @@ pub mod db_checkpoints; pub mod fatal_panic_handler; pub mod mempool; pub mod node_rust_environment; +pub mod p2p; pub mod prometheus; pub mod protocol_update; pub mod state_computer; @@ -74,7 +75,6 @@ pub mod test_state_reader; pub mod transaction_preparer; pub mod transaction_store; pub mod vertex_store_recovery; -pub mod p2p; use crate::engine_prelude::*; diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index 201ab331e9..b9970861f4 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -129,7 +129,8 @@ pub struct JNINodeRustEnvironment { impl JNINodeRustEnvironment { pub fn init(env: &JNIEnv, j_node_rust_env: JObject, j_config: jbyteArray) { - let (base_path, config) = Self::prepare_config(&jni_jbytearray_to_vector(env, j_config).unwrap()); + let (base_path, config) = + Self::prepare_config(&jni_jbytearray_to_vector(env, j_config).unwrap()); let network = config.network_definition.clone(); let runtime = Arc::new(Runtime::new().unwrap()); @@ -189,7 +190,7 @@ impl JNINodeRustEnvironment { let config = StateManagerConfig { database_backend_config: DatabaseBackendConfig { - rocks_db_path: state_manager_db_path + rocks_db_path: state_manager_db_path, }, ..config }; diff --git a/core-rust/state-manager/src/jni/p2p/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs index 5b8f36eb74..7713aa52b6 100644 --- a/core-rust/state-manager/src/jni/p2p/address_book.rs +++ b/core-rust/state-manager/src/jni/p2p/address_book.rs @@ -64,12 +64,12 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use p2p::traits::node::AddressBookStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; use p2p::address_book_components::{AddressBookEntry, AddressBookNodeId}; +use p2p::traits::node::AddressBookStore; #[no_mangle] extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( @@ -109,8 +109,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_reset( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| { - JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .reset(); + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context).reset(); }) } @@ -122,8 +121,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec> { - JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .get_all() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context).get_all() }) } @@ -148,8 +146,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_isMigrated( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| -> bool { - JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .is_migrated() + JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context).is_migrated() }) } diff --git a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs index 8e408fe497..8474db2584 100644 --- a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs @@ -62,9 +62,9 @@ * permissions under this License. */ -use jni::JNIEnv; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; +use jni::JNIEnv; use node_common::java::*; @@ -81,7 +81,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAll ) -> jbyteArray { jni_raw_sbor_fallible_call(&env, payload, |bytes| { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .upsert_all_peers(&bytes); + .upsert_all_high_priority_peers(&bytes); Ok(()) }) } @@ -95,7 +95,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHig ) -> jbyteArray { jni_sbor_coded_call(&env, payload, |_: ()| -> Option> { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .get_all_peers() + .get_all_high_priority_peers() }) } @@ -112,5 +112,4 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_resetHigh }) } - pub fn export_extern_functions() {} diff --git a/core-rust/state-manager/src/jni/p2p/mod.rs b/core-rust/state-manager/src/jni/p2p/mod.rs index 50ddee9600..e210684976 100644 --- a/core-rust/state-manager/src/jni/p2p/mod.rs +++ b/core-rust/state-manager/src/jni/p2p/mod.rs @@ -2,4 +2,4 @@ pub mod address_book; pub mod high_priority_peers; -pub mod safety_store; \ No newline at end of file +pub mod safety_store; diff --git a/core-rust/state-manager/src/jni/p2p/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs index d4ed46d221..02bc33650b 100644 --- a/core-rust/state-manager/src/jni/p2p/safety_store.rs +++ b/core-rust/state-manager/src/jni/p2p/safety_store.rs @@ -64,11 +64,11 @@ use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; -use p2p::traits::node::SafetyStateStore; use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; +use p2p::traits::node::SafetyStateStore; #[no_mangle] extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_upsert( @@ -118,8 +118,7 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_isMigrated( node_id: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, node_id, |_: ()| -> bool { - JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) - .is_migrated() + JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context).is_migrated() }) } diff --git a/core-rust/state-manager/src/jni/test_state_reader.rs b/core-rust/state-manager/src/jni/test_state_reader.rs index 08f73a3ab3..225e434eb4 100644 --- a/core-rust/state-manager/src/jni/test_state_reader.rs +++ b/core-rust/state-manager/src/jni/test_state_reader.rs @@ -73,11 +73,11 @@ use crate::jni::node_rust_environment::JNINodeRustEnvironment; use crate::query::StateManagerSubstateQueries; use node_common::java::*; +use crate::store::traits::measurement::MeasurableDatabase; use crate::store::traits::{ gc::StateTreeGcStore, IterableProofStore, QueryableProofStore, QueryableTransactionStore, SubstateNodeAncestryStore, }; -use crate::store::traits::measurement::MeasurableDatabase; use crate::transaction::LedgerTransactionHash; // diff --git a/core-rust/state-manager/src/staging/cache.rs b/core-rust/state-manager/src/staging/cache.rs index 8ee25820f9..64b818973c 100644 --- a/core-rust/state-manager/src/staging/cache.rs +++ b/core-rust/state-manager/src/staging/cache.rs @@ -87,8 +87,8 @@ use im::hashmap::HashMap as ImmutableHashMap; use itertools::Itertools; use crate::store::rocks_db::ActualStateManagerDatabase; -use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; use crate::store::traits::{ConfigurableDatabase, QueryableProofStore}; +use crate::store::traits::{SubstateNodeAncestryRecord, SubstateNodeAncestryStore}; use node_common::locks::{DbLock, LockFactory, Mutex}; use slotmap::SecondaryMap; diff --git a/core-rust/state-manager/src/staging/mod.rs b/core-rust/state-manager/src/staging/mod.rs index eec4594d32..02f7a180b5 100644 --- a/core-rust/state-manager/src/staging/mod.rs +++ b/core-rust/state-manager/src/staging/mod.rs @@ -73,8 +73,8 @@ use crate::accumulator_tree::storage::ReadableAccuTreeStore; use crate::engine_prelude::*; use crate::{ReceiptTreeHash, StateVersion, TransactionTreeHash}; -use crate::store::traits::SubstateNodeAncestryStore; use crate::store::traits::ConfigurableDatabase; +use crate::store::traits::SubstateNodeAncestryStore; pub use cache::*; pub use result::*; diff --git a/core-rust/state-manager/src/staging/result.rs b/core-rust/state-manager/src/staging/result.rs index 9ced5cf51c..143dc21111 100644 --- a/core-rust/state-manager/src/staging/result.rs +++ b/core-rust/state-manager/src/staging/result.rs @@ -79,8 +79,8 @@ use crate::staging::ReadableStore; use crate::staging::node_ancestry_resolver::NodeAncestryResolver; use crate::staging::overlays::{MapSubstateNodeAncestryStore, StagedSubstateNodeAncestryStore}; -use crate::store::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; use crate::store::traits::{ConfigurableDatabase, LeafSubstateKeyAssociation}; +use crate::store::traits::{KeyedSubstateNodeAncestryRecord, SubstateNodeAncestryStore}; use node_common::utils::IsAccountExt; pub enum ProcessedTransactionReceipt { diff --git a/core-rust/state-manager/src/state_manager.rs b/core-rust/state-manager/src/state_manager.rs index 6dbbc8bac6..a7ad97b8cd 100644 --- a/core-rust/state-manager/src/state_manager.rs +++ b/core-rust/state-manager/src/state_manager.rs @@ -72,9 +72,9 @@ use crate::jni::LedgerSyncLimitsConfig; use crate::protocol::{ ProtocolConfig, ProtocolManager, ProtocolUpdateExecutor, ProtocolVersionName, }; -use crate::store::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; use crate::store::jmt_gc::StateTreeGcConfig; use crate::store::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; +use crate::store::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; use crate::store::traits::proofs::QueryableProofStore; use crate::store::traits::DatabaseConfigValidationError; use crate::transaction::{ diff --git a/core-rust/state-manager/src/store/codecs.rs b/core-rust/state-manager/src/store/codecs.rs index 121348d73b..c87c2e6aa5 100644 --- a/core-rust/state-manager/src/store/codecs.rs +++ b/core-rust/state-manager/src/store/codecs.rs @@ -69,9 +69,9 @@ use crate::engine_prelude::*; use crate::store::traits::indices::CreationId; use crate::store::traits::scenario::ScenarioSequenceNumber; -use node_common::store::typed_cf_api::*; use crate::transaction::RawLedgerTransaction; use crate::StateVersion; +use node_common::store::typed_cf_api::*; #[derive(Default)] pub struct StateVersionDbCodec {} diff --git a/core-rust/state-manager/src/store/column_families.rs b/core-rust/state-manager/src/store/column_families.rs index ba14a26556..6a5756bb0c 100644 --- a/core-rust/state-manager/src/store/column_families.rs +++ b/core-rust/state-manager/src/store/column_families.rs @@ -2,41 +2,18 @@ use std::fmt; use radix_common::crypto::Hash; use radix_common::prelude::{EntityType, Epoch, GlobalAddress, NodeId, PackageAddress}; -use radix_substate_store_impls::state_tree::tree_store::{ - StoredTreeNodeKey, TreeNode, VersionedTreeNode, -}; +use radix_substate_store_impls::state_tree::tree_store::*; use radix_substate_store_interface::interface::{DbSubstateKey, DbSubstateValue}; use radix_transactions::model::{IntentHash, NotarizedTransactionHash}; -use crate::{ - CommittedTransactionIdentifiers, LedgerProof, LedgerTransactionReceipt, - LocalTransactionExecution, StateVersion, VersionedCommittedTransactionIdentifiers, - VersionedLedgerProof, VersionedLedgerTransactionReceipt, VersionedLocalTransactionExecution, -}; -use crate::store::traits::{ - ReceiptAccuTreeSlice, StaleTreeParts, SubstateNodeAncestryRecord, TransactionAccuTreeSlice, - VersionedReceiptAccuTreeSlice, VersionedStaleTreeParts, VersionedSubstateNodeAncestryRecord, - VersionedTransactionAccuTreeSlice, VersionedVertexStoreBlob, VertexStoreBlob, -}; +use crate::store::codecs::*; use crate::store::traits::gc::{LedgerProofsGcProgress, VersionedLedgerProofsGcProgress}; -use crate::store::traits::indices::{ - CreationId, EntityBlueprintId, ObjectBlueprintName, VersionedEntityBlueprintId, - VersionedObjectBlueprintName, -}; -use crate::store::traits::scenario::{ - ExecutedScenario, ScenarioSequenceNumber, VersionedExecutedScenario, -}; -use crate::store::codecs::{ - BlueprintAndCreationIndexKeyDbCodec, EpochDbCodec, HashDbCodec, NodeIdDbCodec, - PrefixGlobalAddressDbCodec, RawLedgerTransactionDbCodec, ScenarioSequenceNumberDbCodec, - StateVersionDbCodec, StoredTreeNodeKeyDbCodec, SubstateKeyDbCodec, - TypeAndCreationIndexKeyDbCodec, -}; -use node_common::store::typed_cf_api::{ - DefaultCf, DirectDbCodec, - PredefinedDbCodec, TypedCf, UnitDbCodec, VersionedCf, -}; +use crate::store::traits::indices::*; +use crate::store::traits::scenario::*; +use crate::store::traits::*; use crate::transaction::{LedgerTransactionHash, RawLedgerTransaction}; +use crate::*; +use node_common::store::typed_cf_api::*; /// Committed transactions. /// Schema: `StateVersion.to_bytes()` -> `RawLedgerTransaction.as_ref::<[u8]>()` diff --git a/core-rust/state-manager/src/store/historical_state.rs b/core-rust/state-manager/src/store/historical_state.rs index dcc4ef791d..6a3accc1db 100644 --- a/core-rust/state-manager/src/store/historical_state.rs +++ b/core-rust/state-manager/src/store/historical_state.rs @@ -67,11 +67,11 @@ use std::ops::Deref; use crate::engine_prelude::entity_tier::EntityTier; use crate::engine_prelude::*; use crate::query::StateManagerSubstateQueries; -use node_common::store::rocks_db::ReadableRocks; use crate::store::rocks_db::StateManagerDatabase; -use crate::store::traits::*; use crate::store::traits::indices::{CreationId, EntityBlueprintId, EntityListingIndex}; +use crate::store::traits::*; use crate::{CommittedTransactionIdentifiers, LedgerStateSummary, StateVersion}; +use node_common::store::rocks_db::ReadableRocks; /// An implementation of a [`SubstateDatabase`] viewed at a specific [`StateVersion`]. /// diff --git a/core-rust/state-manager/src/store/mod.rs b/core-rust/state-manager/src/store/mod.rs index 495fa7c86d..5263dee145 100644 --- a/core-rust/state-manager/src/store/mod.rs +++ b/core-rust/state-manager/src/store/mod.rs @@ -71,14 +71,14 @@ use node_common::locks::DbLock; use rocks_db::ActualStateManagerDatabase; use crate::RawDbMetrics; -pub use traits::DatabaseConfig; use traits::measurement::MeasurableDatabase; +pub use traits::DatabaseConfig; +mod codecs; +pub mod column_families; pub mod historical_state; pub mod jmt_gc; pub mod proofs_gc; -mod codecs; -pub mod column_families; pub mod rocks_db; pub mod traits; diff --git a/core-rust/state-manager/src/store/proofs_gc.rs b/core-rust/state-manager/src/store/proofs_gc.rs index 0194971d40..e08201d3e4 100644 --- a/core-rust/state-manager/src/store/proofs_gc.rs +++ b/core-rust/state-manager/src/store/proofs_gc.rs @@ -76,12 +76,12 @@ use crate::store::traits::proofs::QueryableProofStore; use crate::jni::LedgerSyncLimitsConfig; use crate::store::rocks_db::{ActualStateManagerDatabase, StateManagerDatabase}; -use node_common::store::rocks_db::ReadableRocks; use crate::store::traits::GetSyncableTxnsAndProofError::{ FailedToPrepareAResponseWithinLimits, NothingToServeAtTheGivenStateVersion, RefusedToServeGenesis, RefusedToServeProtocolUpdate, }; use crate::{LedgerProof, StateVersion}; +use node_common::store::rocks_db::ReadableRocks; /// A configuration for [`LedgerProofsGc`]. #[derive(Debug, Categorize, Encode, Decode, Clone, Default)] @@ -313,8 +313,8 @@ mod tests { use crate::proofs_gc::{LedgerProofsGc, LedgerProofsGcConfig}; use crate::protocol::*; use crate::store::traits::proofs::QueryableProofStore; - use crate::test::{commit_round_updates_until_epoch, create_state_manager}; use crate::store::traits::GetSyncableTxnsAndProofError; + use crate::test::{commit_round_updates_until_epoch, create_state_manager}; use crate::{StateManagerConfig, StateVersion}; use std::time::Duration; diff --git a/core-rust/state-manager/src/store/rocks_db.rs b/core-rust/state-manager/src/store/rocks_db.rs index ad1ebfee2e..d4775a6f4b 100644 --- a/core-rust/state-manager/src/store/rocks_db.rs +++ b/core-rust/state-manager/src/store/rocks_db.rs @@ -65,25 +65,17 @@ use std::collections::HashSet; use std::path::PathBuf; -use node_common::rocksdb::{ - ColumnFamilyDescriptor, DB, Direction, IteratorMode, Options, -}; +use node_common::rocksdb::{ColumnFamilyDescriptor, Direction, IteratorMode, Options, DB}; use tracing::{error, info, warn}; use node_common::locks::Snapshottable; use node_common::utils::IsAccountExt; -use crate::{ - BySubstate, CommittedTransactionIdentifiers, LedgerProof, LedgerProofOrigin, - LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, ReceiptTreeHash, - StateVersion, SubstateChangeAction, TransactionTreeHash, -}; use crate::accumulator_tree::storage::{ReadableAccuTreeStore, TreeSlice}; -use crate::store::column_families::*; use crate::engine_prelude::*; use crate::query::TransactionIdentifierLoader; -use node_common::store::rocks_db::*; -use crate::store::traits::*; +use crate::store::column_families::*; +use crate::store::historical_state::StateTreeBasedSubstateDatabase; use crate::store::traits::extensions::*; use crate::store::traits::gc::{LedgerProofsGcProgress, LedgerProofsGcStore, StateTreeGcStore}; use crate::store::traits::indices::{ @@ -93,11 +85,17 @@ use crate::store::traits::measurement::{CategoryDbVolumeStatistic, MeasurableDat use crate::store::traits::scenario::{ ExecutedScenario, ExecutedScenarioStore, ScenarioSequenceNumber, }; -use crate::store::historical_state::StateTreeBasedSubstateDatabase; -use node_common::store::typed_cf_api::*; +use crate::store::traits::*; use crate::transaction::{ LedgerTransactionHash, RawLedgerTransaction, TypedTransactionIdentifiers, }; +use crate::{ + BySubstate, CommittedTransactionIdentifiers, LedgerProof, LedgerProofOrigin, + LedgerTransactionReceipt, LocalTransactionExecution, LocalTransactionReceipt, ReceiptTreeHash, + StateVersion, SubstateChangeAction, TransactionTreeHash, +}; +use node_common::store::rocks_db::*; +use node_common::store::typed_cf_api::*; /// A listing of all column family names used by the Node. /// @@ -166,7 +164,6 @@ impl<'db> Snapshottable<'db> for StateManagerDatabase { } } - /// A RocksDB-backed persistence layer for state manager. pub struct StateManagerDatabase { /// Database config. @@ -237,7 +234,7 @@ impl ActualStateManagerDatabase { column_families, false, ) - .unwrap(); + .unwrap(); StateManagerDatabase { config: DatabaseConfig { @@ -272,7 +269,7 @@ impl ActualStateManagerDatabase { temp_path.as_path(), column_families, ) - .unwrap(); + .unwrap(); StateManagerDatabase { config: DatabaseConfig { @@ -752,11 +749,11 @@ impl ExecutedScenarioStore for StateManagerDatabase { pub struct RocksDBCommittedTransactionBundleIterator<'r> { state_version: StateVersion, - txns_iter: Box + 'r>, - ledger_receipts_iter: Box + 'r>, - local_executions_iter: Box + 'r>, + txns_iter: Box + 'r>, + ledger_receipts_iter: Box + 'r>, + local_executions_iter: Box + 'r>, identifiers_iter: - Box + 'r>, + Box + 'r>, } impl<'r> RocksDBCommittedTransactionBundleIterator<'r> { @@ -834,7 +831,7 @@ impl IterableTransactionStore for StateManagerDatabase { fn get_committed_transaction_bundle_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { // This should not happen. This interface should be used after checking (e.g. `core-api-server/src/core-api/handlers/`). // However, with or without this debug_assert there would still be a panic if LocalTransactionExecution is missing. debug_assert!(self.is_local_transaction_execution_index_enabled()); @@ -953,7 +950,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(LedgerProofsCf) @@ -965,7 +962,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_next_epoch_proof_iter( &self, from_epoch: Epoch, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(EpochLedgerProofsCf) @@ -977,7 +974,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_protocol_update_init_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(ProtocolUpdateInitLedgerProofsCf) @@ -989,7 +986,7 @@ impl IterableProofStore for StateManagerDatabase { fn get_protocol_update_execution_proof_iter( &self, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(ProtocolUpdateExecutionLedgerProofsCf) @@ -1070,8 +1067,8 @@ impl QueryableProofStore for StateManagerDatabase { 'proof_txns_loop: while payload_size_including_next_proof_txns <= max_payload_size_in_bytes && (latest_usable_proof.is_none() - || txns.len() + next_proof_txns.len() - <= (max_number_of_txns_if_more_than_one_proof as usize)) + || txns.len() + next_proof_txns.len() + <= (max_number_of_txns_if_more_than_one_proof as usize)) { match txns_iter.next() { Some((next_txn_state_version, next_txn)) => { @@ -1097,8 +1094,8 @@ impl QueryableProofStore for StateManagerDatabase { // that they can all fit in the response (the last txn could have crossed the limit) if payload_size_including_next_proof_txns <= max_payload_size_in_bytes && (latest_usable_proof.is_none() - || txns.len() + next_proof_txns.len() - <= (max_number_of_txns_if_more_than_one_proof as usize)) + || txns.len() + next_proof_txns.len() + <= (max_number_of_txns_if_more_than_one_proof as usize)) { // Yup, all good, use next_proof as the result and add its txns let next_proof_is_a_protocol_update = @@ -1233,7 +1230,7 @@ impl SubstateDatabase for StateManagerDatabase { &self, partition_key: &DbPartitionKey, from_sort_key: Option<&DbSortKey>, - ) -> Box + '_> { + ) -> Box + '_> { let partition_key = partition_key.clone(); let from_sort_key = from_sort_key.cloned().unwrap_or(DbSortKey(vec![])); Box::new( @@ -1246,7 +1243,7 @@ impl SubstateDatabase for StateManagerDatabase { } impl ListableSubstateDatabase for StateManagerDatabase { - fn list_partition_keys(&self) -> Box + '_> { + fn list_partition_keys(&self) -> Box + '_> { self.open_read_context() .cf(SubstatesCf) .iterate_key_groups() @@ -1256,7 +1253,7 @@ impl ListableSubstateDatabase for StateManagerDatabase { impl SubstateNodeAncestryStore for StateManagerDatabase { fn batch_get_ancestry<'a>( &self, - node_ids: impl IntoIterator, + node_ids: impl IntoIterator, ) -> Vec> { self.open_read_context() .cf(SubstateNodeAncestryRecordsCf) @@ -1273,7 +1270,7 @@ impl ReadableTreeStore for StateManagerDatabase { impl StateTreeGcStore for StateManagerDatabase { fn get_stale_tree_parts_iter( &self, - ) -> Box + '_> { + ) -> Box + '_> { self.open_read_context() .cf(StaleStateTreePartsCf) .iterate(Direction::Forward) @@ -1304,7 +1301,7 @@ impl StateTreeGcStore for StateManagerDatabase { ); } - fn batch_delete_node<'a>(&self, keys: impl IntoIterator) { + fn batch_delete_node<'a>(&self, keys: impl IntoIterator) { let db_context = self.open_rw_context(); let tree_nodes_cf = db_context.cf(StateTreeNodesCf); let associated_values_cf = db_context.cf(AssociatedStateTreeValuesCf); @@ -1346,7 +1343,7 @@ impl LedgerProofsGcStore for StateManagerDatabase { } impl ReadableAccuTreeStore -for StateManagerDatabase + for StateManagerDatabase { fn get_tree_slice( &self, @@ -1360,7 +1357,7 @@ for StateManagerDatabase } impl ReadableAccuTreeStore -for StateManagerDatabase + for StateManagerDatabase { fn get_tree_slice(&self, state_version: &StateVersion) -> Option> { self.open_read_context() @@ -1676,7 +1673,7 @@ impl RestoreDecember2023LostSubstates for StateManagerDatabas let substates_cf = db_context.cf(SubstatesCf); - let receipts_iter: Box> = + let receipts_iter: Box> = db_context .cf(TransactionReceiptsCf) .iterate_from(&StateVersion::of(1u64), Direction::Forward); @@ -1730,7 +1727,7 @@ impl IterableAccountChangeIndex for StateManagerDatabase { &self, account: GlobalAddress, from_state_version: StateVersion, - ) -> Box + '_> { + ) -> Box + '_> { Box::new( self.open_read_context() .cf(AccountChangeStateVersionsCf) @@ -1746,7 +1743,7 @@ impl EntityListingIndex for StateManagerDatabase { &self, entity_type: EntityType, from_creation_id: Option<&CreationId>, - ) -> Box + '_> { + ) -> Box + '_> { let from_creation_id = from_creation_id.cloned().unwrap_or_else(CreationId::zero); Box::new( self.open_read_context() @@ -1760,7 +1757,7 @@ impl EntityListingIndex for StateManagerDatabase { &self, blueprint_id: &BlueprintId, from_creation_id: Option<&CreationId>, - ) -> Box + '_> { + ) -> Box + '_> { let BlueprintId { package_address, blueprint_name, diff --git a/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java b/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java index d9fdda50b1..de2365efd8 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/AddressBookModule.java @@ -64,6 +64,7 @@ package com.radixdlt.rev2.modules; +import com.google.common.annotations.VisibleForTesting; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -88,6 +89,28 @@ AddressBookPersistence addressBookPersistence( Metrics metrics, @NodeStorageLocation String nodeStorageLocation, NodeRustEnvironment environment) { + + var addressBookStore = + new RocksAddressBookStore( + RocksDbAddressBookStore.create(metrics, environment), + RocksDbHighPriorityPeersStore.create(metrics, environment)); + + ensureMigrated(addressBookStore, properties, serialization, metrics, nodeStorageLocation); + + return addressBookStore; + } + + @VisibleForTesting + public static void ensureMigrated( + RocksAddressBookStore addressBookStore, + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + String nodeStorageLocation) { + if (addressBookStore.isMigrated()) { + return; + } + var berkeleyAddressBookStore = new BerkeleyAddressBookStore( serialization, @@ -95,19 +118,10 @@ AddressBookPersistence addressBookPersistence( nodeStorageLocation, BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - var addressBookStore = - new RocksAddressBookStore( - RocksDbAddressBookStore.create(metrics, environment), - RocksDbHighPriorityPeersStore.create(metrics, environment)); - try (berkeleyAddressBookStore) { - if (!addressBookStore.isMigrated()) { - berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); - addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); - addressBookStore.markAsMigrated(); - } + berkeleyAddressBookStore.getAllEntries().forEach(addressBookStore::upsertEntry); + addressBookStore.storeHighPriorityPeers(berkeleyAddressBookStore.getHighPriorityPeers()); + addressBookStore.markAsMigrated(); } - - return addressBookStore; } } diff --git a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java b/core/src/main/java/com/radixdlt/rev2/modules/PersistentSafetyStateStoreModule.java similarity index 86% rename from core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java rename to core/src/main/java/com/radixdlt/rev2/modules/PersistentSafetyStateStoreModule.java index 509241b9c8..e4a6feabe4 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/NodePersistenceModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/PersistentSafetyStateStoreModule.java @@ -64,6 +64,7 @@ package com.radixdlt.rev2.modules; +import com.google.common.annotations.VisibleForTesting; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -78,25 +79,7 @@ import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; -public class NodePersistenceModule extends AbstractModule { - private final AddressBookPolicy policy; - - public enum AddressBookPolicy { - ENABLED, - DISABLED - } - - public NodePersistenceModule(AddressBookPolicy policy) { - this.policy = policy; - } - - @Override - protected void configure() { - if (policy == AddressBookPolicy.ENABLED) { - install(new AddressBookModule()); - } - } - +public class PersistentSafetyStateStoreModule extends AbstractModule { @Provides @Singleton PersistentSafetyStateStore persistentSafetyStateStore( @@ -106,23 +89,32 @@ PersistentSafetyStateStore persistentSafetyStateStore( @NodeStorageLocation String nodeStorageLocation, NodeRustEnvironment environment) { - var berkeleySafetyStateStore = - new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - var rocksSafetyStateStore = new RocksSafetyStateStore(RocksDbSafetyStore.create(metrics, environment)); - try (berkeleySafetyStateStore) { - if (!rocksSafetyStateStore.isMigrated()) { + ensureMigrated(rocksSafetyStateStore, properties, serialization, metrics, nodeStorageLocation); + + return rocksSafetyStateStore; + } + + @VisibleForTesting + public static void ensureMigrated( + RocksSafetyStateStore rocksSafetyStateStore, + RuntimeProperties properties, + Serialization serialization, + Metrics metrics, + String nodeStorageLocation) { + if (!rocksSafetyStateStore.isMigrated()) { + var berkeleySafetyStateStore = + new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + try (berkeleySafetyStateStore) { berkeleySafetyStateStore.get().ifPresent(rocksSafetyStateStore::commitState); rocksSafetyStateStore.markAsMigrated(); } } - - return rocksSafetyStateStore; } } diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index f12a359c32..63f84507f3 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -321,7 +321,8 @@ EventProcessor onInsertUpdatePersistVertexStore( if (installPersistence) { // Moved here for convenience performing migration - install(new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.ENABLED)); + install(new PersistentSafetyStateStoreModule()); + install(new AddressBookModule()); } } diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index 9d093df93e..4aa500a38e 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -121,7 +121,6 @@ static NodeStorageConfig file(File folder) { public enum SafetyRecoveryConfig { MOCKED, REAL, - REAL_WITH_ADDRESS_BOOK } public static final class ConsensusConfig { @@ -314,10 +313,7 @@ public void configure() { switch (this.safetyRecoveryConfig) { case MOCKED -> install(new MockedSafetyStoreModule()); - case REAL -> install( - new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.DISABLED)); - case REAL_WITH_ADDRESS_BOOK -> install( - new NodePersistenceModule(NodePersistenceModule.AddressBookPolicy.ENABLED)); + case REAL -> install(new PersistentSafetyStateStoreModule()); } // Consensus diff --git a/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java b/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentHelper.java similarity index 99% rename from core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java rename to core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentHelper.java index 1739666981..5b9a73ad9f 100644 --- a/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentBuilder.java +++ b/core/src/test-core/java/com/radixdlt/rev2/NodeRustEnvironmentHelper.java @@ -70,7 +70,7 @@ import com.radixdlt.protocol.ProtocolConfig; import com.radixdlt.transaction.LedgerSyncLimitsConfig; -public class NodeRustEnvironmentBuilder { +public class NodeRustEnvironmentHelper { public static NodeRustEnvironment createNodeRustEnvironment(String dbPath) { final var mempoolMaxTotalTransactionsSize = 10 * 1024 * 1024; final var mempoolMaxTransactionCount = 20; diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index 08d060ec9f..d97bfe9fb2 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -69,7 +69,7 @@ import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.rev2.NodeRustEnvironmentBuilder; +import com.radixdlt.rev2.NodeRustEnvironmentHelper; import com.radixdlt.safety.RocksDbSafetyStore; import java.io.IOException; import java.util.function.Consumer; @@ -100,7 +100,7 @@ public void test_address_book_entries_can_be_saved_and_restored() { private void runTest(Consumer test) { try (var environment = - NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { + NodeRustEnvironmentHelper.createNodeRustEnvironment(folder.newFolder().getPath())) { var safetyStore = RocksDbSafetyStore.create(newMetrics(), environment); try (var underTest = new RocksSafetyStateStore(safetyStore)) { test.accept(underTest); diff --git a/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java index 4e3bc187b6..9244907dad 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/SafetyStoreMigrationTest.java @@ -67,20 +67,17 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.google.inject.*; -import com.google.inject.Module; import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.rev2.NodeRustEnvironmentBuilder; -import com.radixdlt.rev2.modules.NodePersistenceModule; -import com.radixdlt.rev2.modules.NodePersistenceModule.AddressBookPolicy; +import com.radixdlt.rev2.NodeRustEnvironmentHelper; +import com.radixdlt.rev2.modules.PersistentSafetyStateStoreModule; +import com.radixdlt.safety.RocksDbSafetyStore; import com.radixdlt.serialization.Serialization; import com.radixdlt.serialization.core.ClasspathScanningSerializationPolicy; import com.radixdlt.serialization.core.ClasspathScanningSerializerIds; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.IOException; import java.util.Map; @@ -92,79 +89,58 @@ public class SafetyStoreMigrationTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private Module createCommonModule() { - return new AbstractModule() { - @Provides - @Singleton - @NodeStorageLocation - String nodeStorageLocation() { - return folder.getRoot().getPath(); - } - - @Provides - @Singleton - NodeRustEnvironment nodeRustEnvironment() throws IOException { - return NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath()); - } - - @Provides - @Singleton - Metrics metrics() { - return new MetricsInitializer().initialize(); - } - - @Provides - @Singleton - Serialization serialization() { - return Serialization.create( - ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()); - } + record TestEnvironment( + String nodeStorageLocation, + NodeRustEnvironment nodeRustEnvironment, + Metrics metrics, + Serialization serialization, + RuntimeProperties properties) { + public RocksSafetyStateStore rocksSafetyStateStore() { + return new RocksSafetyStateStore(RocksDbSafetyStore.create(metrics, nodeRustEnvironment)); + } - @Provides - @Singleton - RuntimeProperties properties() throws ParseException { - return RuntimeProperties.defaultWithOverrides(Map.of()); - } - }; + public BerkeleySafetyStateStore berkeleySafetyStateStore() { + return new BerkeleySafetyStateStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } } - private Module createBerkeleyModule() { - return new AbstractModule() { - @Provides - @Singleton - BerkeleySafetyStateStore safetyStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - } - }; + private TestEnvironment createTestEnvironment() throws IOException, ParseException { + return new TestEnvironment( + folder.getRoot().getPath(), + NodeRustEnvironmentHelper.createNodeRustEnvironment(folder.newFolder().getPath()), + new MetricsInitializer().initialize(), + Serialization.create( + ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()), + RuntimeProperties.defaultWithOverrides(Map.of())); } @Test - public void address_book_migrates_successfully() { - var injector = - Guice.createInjector( - createCommonModule(), - new NodePersistenceModule(AddressBookPolicy.DISABLED), - createBerkeleyModule()); + public void safety_store_migrates_successfully() throws IOException, ParseException { + var injector = createTestEnvironment(); var safetyState = SafetyState.initialState(BFTValidatorId.random()); // Create some entries in the Berkeley safety store - try (var berkeleySafetyStateStore = injector.getInstance(BerkeleySafetyStateStore.class)) { + try (var berkeleySafetyStateStore = injector.berkeleySafetyStateStore()) { berkeleySafetyStateStore.commitState(safetyState); } - // Instantiation of the PersistentSafetyStateStore will trigger the migration - var safetyStateStore = injector.getInstance(PersistentSafetyStateStore.class); - var storedState = safetyStateStore.get(); + // Perform the migration and check content + try (var safetyStateStore = injector.rocksSafetyStateStore()) { + PersistentSafetyStateStoreModule.ensureMigrated( + safetyStateStore, + injector.properties, + injector.serialization, + injector.metrics, + injector.nodeStorageLocation); + + var storedState = safetyStateStore.get(); - assertTrue(storedState.isPresent()); - assertEquals(safetyState, storedState.get()); + assertTrue(storedState.isPresent()); + assertEquals(safetyState, storedState.get()); + } } } diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java index 8bc71c3155..4811d19ac8 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/AddressBookMigrationTest.java @@ -67,23 +67,20 @@ import static com.radixdlt.p2p.addressbook.RocksAddressBookStoreTest.*; import static org.junit.Assert.assertEquals; -import com.google.inject.*; -import com.google.inject.Module; import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; -import com.radixdlt.rev2.NodeRustEnvironmentBuilder; +import com.radixdlt.p2p.RocksDbAddressBookStore; +import com.radixdlt.p2p.RocksDbHighPriorityPeersStore; +import com.radixdlt.rev2.NodeRustEnvironmentHelper; import com.radixdlt.rev2.modules.AddressBookModule; import com.radixdlt.serialization.Serialization; import com.radixdlt.serialization.core.ClasspathScanningSerializationPolicy; import com.radixdlt.serialization.core.ClasspathScanningSerializerIds; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.ParseException; import org.junit.Rule; import org.junit.Test; @@ -92,73 +89,80 @@ public class AddressBookMigrationTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private Module createCommonModule() { - return new AbstractModule() { - @Provides - @Singleton - @NodeStorageLocation - String nodeStorageLocation() { - return folder.getRoot().getPath(); - } - - @Provides - @Singleton - NodeRustEnvironment nodeRustEnvironment() throws IOException { - return NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath()); - } - - @Provides - @Singleton - Metrics metrics() { - return new MetricsInitializer().initialize(); - } - - @Provides - @Singleton - Serialization serialization() { - return Serialization.create( - ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()); - } + record TestEnvironment( + String nodeStorageLocation, + NodeRustEnvironment nodeRustEnvironment, + Metrics metrics, + Serialization serialization, + RuntimeProperties properties) { + public RocksAddressBookStore rocksAddressBookPersistence() { + return new RocksAddressBookStore( + RocksDbAddressBookStore.create(metrics, nodeRustEnvironment), + RocksDbHighPriorityPeersStore.create(metrics, nodeRustEnvironment)); + } - @Provides - @Singleton - RuntimeProperties properties() throws ParseException { - return RuntimeProperties.defaultWithOverrides(Map.of()); - } - }; + public BerkeleyAddressBookStore berkeleyAddressBookPersistence() { + return new BerkeleyAddressBookStore( + serialization, + metrics, + nodeStorageLocation, + BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + } } - private Module createBerkeleyModule() { - return new AbstractModule() { - @Provides - @Singleton - BerkeleyAddressBookStore addressBookPersistence( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleyAddressBookStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); - } - }; + private TestEnvironment createTestEnvironment() throws IOException, ParseException { + return new TestEnvironment( + folder.getRoot().getPath(), + NodeRustEnvironmentHelper.createNodeRustEnvironment(folder.newFolder().getPath()), + new MetricsInitializer().initialize(), + Serialization.create( + ClasspathScanningSerializerIds.create(), ClasspathScanningSerializationPolicy.create()), + RuntimeProperties.defaultWithOverrides(Map.of())); } @Test - public void address_book_migrates_successfully() { - var injector = - Guice.createInjector(createCommonModule(), new AddressBookModule(), createBerkeleyModule()); + public void address_book_migrates_successfully() throws IOException, ParseException { + var environment = createTestEnvironment(); var addresses = Set.of(newAddressBookEntry(1), newAddressBookEntry(2), newAddressBookEntry(3)); // Create some entries in the Berkeley address book - try (var berkeleyAddressBook = injector.getInstance(BerkeleyAddressBookStore.class)) { + try (var berkeleyAddressBook = environment.berkeleyAddressBookPersistence()) { addresses.forEach(berkeleyAddressBook::upsertEntry); } - // Instantiation of the AddressBook will trigger the migration - var addressBook = injector.getInstance(AddressBookPersistence.class); - assertEquals(new HashSet<>(addressBook.getAllEntries()), addresses); + // Perform the migration and check content + try (var addressBook = environment.rocksAddressBookPersistence()) { + AddressBookModule.ensureMigrated( + addressBook, + environment.properties, + environment.serialization, + environment.metrics, + environment.nodeStorageLocation); + + assertEquals(new HashSet<>(addressBook.getAllEntries()), addresses); + } + } + + @Test + public void high_priority_peers_migrate_successfully() throws IOException, ParseException { + var environment = createTestEnvironment(); + var highPriorityPeers = List.of(newNodeId(21), newNodeId(32), newNodeId(43)); + + // Create some entries in the Berkeley address book + try (var berkeleyAddressBook = environment.berkeleyAddressBookPersistence()) { + berkeleyAddressBook.storeHighPriorityPeers(highPriorityPeers); + } + + // Perform the migration and check content + try (var addressBook = environment.rocksAddressBookPersistence()) { + AddressBookModule.ensureMigrated( + addressBook, + environment.properties, + environment.serialization, + environment.metrics, + environment.nodeStorageLocation); + + assertEquals(highPriorityPeers, addressBook.getHighPriorityPeers()); + } } } diff --git a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java index 66c25f0fe3..b28c5cdcbf 100644 --- a/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java +++ b/core/src/test/java/com/radixdlt/p2p/addressbook/RocksAddressBookStoreTest.java @@ -75,9 +75,10 @@ import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.FailedHandshake; import com.radixdlt.p2p.addressbook.AddressBookEntry.PeerAddressEntry.LatestConnectionStatus; -import com.radixdlt.rev2.NodeRustEnvironmentBuilder; +import com.radixdlt.rev2.NodeRustEnvironmentHelper; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.Random; import java.util.function.Consumer; @@ -131,9 +132,29 @@ public void test_address_book_entries_can_be_saved_and_restored() { }); } + @Test + public void test_high_priority_peers_can_be_saved_and_restored() { + runTest( + rocksAddressBookStore -> { + var peers = List.of(newNodeId(11), newNodeId(12), newNodeId(13)); + + // New store is empty + var empty = rocksAddressBookStore.getHighPriorityPeers(); + assertTrue(empty.isEmpty()); + + // Peers can be stored and restored + rocksAddressBookStore.storeHighPriorityPeers(peers); + + var allPeers = rocksAddressBookStore.getHighPriorityPeers(); + + assertEquals(3L, allPeers.size()); + assertEquals(peers, allPeers); + }); + } + private void runTest(Consumer test) { try (var environment = - NodeRustEnvironmentBuilder.createNodeRustEnvironment(folder.newFolder().getPath())) { + NodeRustEnvironmentHelper.createNodeRustEnvironment(folder.newFolder().getPath())) { var addressBook = RocksDbAddressBookStore.create(newMetrics(), environment); var peersStore = RocksDbHighPriorityPeersStore.create(newMetrics(), environment); try (var underTest = new RocksAddressBookStore(addressBook, peersStore)) { @@ -149,13 +170,21 @@ private static Metrics newMetrics() { } public static AddressBookEntry newAddressBookEntry(int id) { - var pubKey = ECKeyPair.fromSeed(new byte[] {(byte) id}).getPublicKey(); + var pubKey = newPubKey((byte) id); var bannedUntil = newBannedUntil(); return new AddressBookEntry( NodeId.fromPublicKey(pubKey), bannedUntil, newKnownAddresses(pubKey)); } + public static NodeId newNodeId(int id) { + return NodeId.fromPublicKey(newPubKey((byte) id)); + } + + private static ECDSASecp256k1PublicKey newPubKey(byte id) { + return ECKeyPair.fromSeed(new byte[] {id}).getPublicKey(); + } + private static ImmutableSet newKnownAddresses(ECDSASecp256k1PublicKey pubKey) { return IntStream.range(0, RANDOM.nextInt(3)) .mapToObj(__ -> newPeerAddressEntry(pubKey)) diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index df9e129191..0b08d49326 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -195,6 +195,10 @@ private static Injector createInjector( final var properties = RuntimeProperties.fromCommandLineArgs(new String[] {}); return Guice.createInjector( new AbstractModule() { + + // FIXME: Replacing implementation with the RocksDB version does not work out of the box + // because NodeRustEnvironment can't + // be properly instantiated (error on Rust side). @Override protected void configure() { bind(AddressBookPersistence.class).to(BerkeleyAddressBookStore.class); From d3925ee8a0c948b9793e407a790d36185e3ea272 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Tue, 27 Aug 2024 14:06:31 +0200 Subject: [PATCH 29/40] Fix docker-related CI part --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index be0db21497..49125cdada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,7 @@ RUN apt-get update \ wget=${WGET_VERSION} \ software-properties-common=0.99.30-4.1~deb12u1 \ && apt-get install -y --no-install-recommends \ - openjdk-17-jdk=17.0.11+9-1~deb12u1 \ + openjdk-17-jdk=17.0.12+7-2~deb12u1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -256,7 +256,7 @@ LABEL org.opencontainers.image.authors="devops@radixdlt.com" # - https://packages.debian.org/bookworm/libc6 RUN apt-get update -y \ && apt-get -y --no-install-recommends install \ - openjdk-17-jre-headless=17.0.11+9-1~deb12u1 \ + openjdk-17-jre-headless=17.0.12+7-2~deb12u1 \ # https://security-tracker.debian.org/tracker/CVE-2023-38545 curl=7.88.1-10+deb12u6 \ gettext-base=0.21-12 \ From fb4a81429e2922e67c9d790cc6ad19df8d9f159b Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Tue, 27 Aug 2024 14:29:58 +0200 Subject: [PATCH 30/40] Fix docker-related CI part --- .github/workflows/ci.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39293fff88..98dcbe6cac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,8 +111,7 @@ jobs: fetch-depth: 0 - uses: RDXWorks-actions/rust-toolchain@master with: - toolchain: 1.77.2 - default: true + toolchain: stable - name: Set up JDK 17 uses: RDXWorks-actions/setup-java@main with: @@ -191,8 +190,7 @@ jobs: fetch-depth: 0 - uses: RDXWorks-actions/rust-toolchain@master with: - toolchain: 1.77.2 - default: true + toolchain: stable - name: Set up JDK 17 uses: RDXWorks-actions/setup-java@main with: @@ -216,8 +214,7 @@ jobs: fetch-depth: 0 - uses: RDXWorks-actions/rust-toolchain@master with: - toolchain: 1.77.2 - default: true + toolchain: stable - name: Set up JDK 17 uses: RDXWorks-actions/setup-java@main with: @@ -246,8 +243,7 @@ jobs: fetch-depth: 0 - uses: RDXWorks-actions/rust-toolchain@master with: - toolchain: 1.77.2 - default: true + toolchain: stable - name: Set up JDK 17 uses: RDXWorks-actions/setup-java@main with: @@ -275,8 +271,7 @@ jobs: fetch-depth: 1 - uses: RDXWorks-actions/rust-toolchain@master with: - toolchain: 1.77.2 - default: true + toolchain: stable targets: x86_64-pc-windows-msvc - name: Update clang version to 17 run: sudo apt remove clang-14 && sudo apt autoclean && sudo apt autoremove && wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh 17 && sudo ls /usr/bin/ | grep clang && sudo ln -sf /usr/bin/clang-17 /usr/bin/clang && sudo ln -sf /usr/bin/clang++-17 /usr/bin/clang++ && sudo apt-get install -y libclang-dev llvm llvm-dev From 2eddb0209a65b68424756c9e0a002dec579c2795 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 28 Aug 2024 10:37:18 +0200 Subject: [PATCH 31/40] Address Sonar issues --- .../main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java | 9 +++++++++ .../main/java/com/radixdlt/safety/BFTValidatorDTO.java | 8 ++++++++ .../src/main/java/com/radixdlt/safety/VertexIdDTO.java | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java index 4cdf5b1863..e245079f14 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java @@ -109,4 +109,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(Arrays.hashCode(address), latestConnectionStatus, maybeFailedHandshake); } + + @Override + public String toString() { + return "PeerAddressEntryDTO{" + + "address=" + Arrays.toString(address) + + ", latestConnectionStatus=" + latestConnectionStatus + + ", maybeFailedHandshake=" + maybeFailedHandshake + + '}'; + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java index efc0465059..53c53b2228 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java @@ -90,4 +90,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(Arrays.hashCode(power), validatorId); } + + @Override + public String toString() { + return "BFTValidatorDTO{" + + "power=" + Arrays.toString(power) + + ", validatorId=" + validatorId + + '}'; + } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java index 3948b5b852..af44127cc4 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java @@ -92,6 +92,13 @@ public int hashCode() { return Arrays.hashCode(idBytes); } + @Override + public String toString() { + return "VertexIdDTO{" + + "idBytes=" + Arrays.toString(idBytes) + + '}'; + } + public static void registerCodec(CodecMap codecMap) { codecMap.register( VertexIdDTO.class, From 464af48a8c85a7b9e67f6683f467266358e49f50 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 28 Aug 2024 10:54:39 +0200 Subject: [PATCH 32/40] Attempt to fix Docker builds --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 49125cdada..6e8e7aac5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,11 +177,13 @@ RUN USER=root "$HOME/.cargo/bin/cargo" init --lib --name dummy --vcs none . \ && mkdir -p ./jni-export/src \ && mkdir -p ./node-common/src \ && mkdir -p ./state-manager/src \ + && mkdir -p ./p2p/src \ && touch ./core-api-server/src/lib.rs \ && touch ./engine-state-api-server/src/lib.rs \ && touch ./jni-export/src/lib.rs \ && touch ./node-common/src/lib.rs \ && touch ./state-manager/src/lib.rs + && touch ./p2p/src/lib.rs COPY core-rust/Cargo.toml ./ COPY core-rust/Cargo.lock ./ COPY core-rust/core-api-server/Cargo.toml ./core-api-server @@ -189,6 +191,7 @@ COPY core-rust/engine-state-api-server/Cargo.toml ./engine-state-api-server COPY core-rust/jni-export/Cargo.toml ./jni-export COPY core-rust/node-common/Cargo.toml ./node-common COPY core-rust/state-manager/Cargo.toml ./state-manager +COPY core-rust/p2p/Cargo.toml ./p2p COPY docker/build_scripts/cargo_build_by_platform.sh /opt/radixdlt/cargo_build_by_platform.sh From 0ab42ea6d37add2b1b066464ce198729cdbcf215 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 28 Aug 2024 10:54:56 +0200 Subject: [PATCH 33/40] Attempt to fix Docker builds --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e8e7aac5a..1237759f9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -182,7 +182,7 @@ RUN USER=root "$HOME/.cargo/bin/cargo" init --lib --name dummy --vcs none . \ && touch ./engine-state-api-server/src/lib.rs \ && touch ./jni-export/src/lib.rs \ && touch ./node-common/src/lib.rs \ - && touch ./state-manager/src/lib.rs + && touch ./state-manager/src/lib.rs \ && touch ./p2p/src/lib.rs COPY core-rust/Cargo.toml ./ COPY core-rust/Cargo.lock ./ From ffad413bb7247e07cf501e88e24ff120b3fbf46a Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Wed, 28 Aug 2024 11:15:41 +0200 Subject: [PATCH 34/40] Fix formatting --- .../java/com/radixdlt/p2p/PeerAddressEntryDTO.java | 13 ++++++++----- .../java/com/radixdlt/safety/BFTValidatorDTO.java | 10 ++++++---- .../main/java/com/radixdlt/safety/VertexIdDTO.java | 4 +--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java index e245079f14..7b2432412b 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/PeerAddressEntryDTO.java @@ -112,10 +112,13 @@ public int hashCode() { @Override public String toString() { - return "PeerAddressEntryDTO{" + - "address=" + Arrays.toString(address) + - ", latestConnectionStatus=" + latestConnectionStatus + - ", maybeFailedHandshake=" + maybeFailedHandshake + - '}'; + return "PeerAddressEntryDTO{" + + "address=" + + Arrays.toString(address) + + ", latestConnectionStatus=" + + latestConnectionStatus + + ", maybeFailedHandshake=" + + maybeFailedHandshake + + '}'; } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java index 53c53b2228..8ec08391f2 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/BFTValidatorDTO.java @@ -93,9 +93,11 @@ public int hashCode() { @Override public String toString() { - return "BFTValidatorDTO{" + - "power=" + Arrays.toString(power) + - ", validatorId=" + validatorId + - '}'; + return "BFTValidatorDTO{" + + "power=" + + Arrays.toString(power) + + ", validatorId=" + + validatorId + + '}'; } } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java index af44127cc4..74a7e9e1d5 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/VertexIdDTO.java @@ -94,9 +94,7 @@ public int hashCode() { @Override public String toString() { - return "VertexIdDTO{" + - "idBytes=" + Arrays.toString(idBytes) + - '}'; + return "VertexIdDTO{" + "idBytes=" + Arrays.toString(idBytes) + '}'; } public static void registerCodec(CodecMap codecMap) { From 993078317fac00e84b4c339105a1f5bb3ece8c0a Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Fri, 30 Aug 2024 14:13:24 +0200 Subject: [PATCH 35/40] Add versioning to DB data format. --- .../radixdlt/p2p/RocksDbAddressBookStore.java | 12 +- .../p2p/RocksDbHighPriorityPeersStore.java | 27 +-- .../radixdlt/safety/RocksDbSafetyStore.java | 10 +- .../com/radixdlt/sbor/NodeSborCodecs.java | 2 + core-rust/node-common/src/java/utils.rs | 14 -- core-rust/p2p/src/address_book_components.rs | 111 ++++++++-- core-rust/p2p/src/column_families.rs | 102 +++++++-- core-rust/p2p/src/components.rs | 36 ++++ core-rust/p2p/src/lib.rs | 2 + core-rust/p2p/src/migration.rs | 64 ++++++ core-rust/p2p/src/rocks_db.rs | 32 +-- core-rust/p2p/src/safety_store_components.rs | 203 ++++++++++++++++++ core-rust/p2p/src/traits.rs | 19 +- core-rust/p2p/src/typed_cf_api.rs | 17 +- .../state-manager/src/jni/p2p/address_book.rs | 14 +- .../src/jni/p2p/high_priority_peers.rs | 11 +- .../state-manager/src/jni/p2p/safety_store.rs | 8 +- .../safety/RocksSafetyStateStoreTest.java | 2 +- 18 files changed, 548 insertions(+), 138 deletions(-) create mode 100644 core-rust/p2p/src/components.rs create mode 100644 core-rust/p2p/src/safety_store_components.rs diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java index 9daee86d94..29bdce7f49 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbAddressBookStore.java @@ -71,8 +71,6 @@ import com.radixdlt.environment.NodeRustEnvironment; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; -import com.radixdlt.sbor.NodeSborCodecs; -import com.radixdlt.sbor.codec.Codec; import java.util.List; public class RocksDbAddressBookStore { @@ -93,8 +91,6 @@ private static native byte[] markAsMigrated( private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private final Codec dtoCodec; - public static RocksDbAddressBookStore create( Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { return new RocksDbAddressBookStore(metrics, nodeRustEnvironment); @@ -127,8 +123,6 @@ private RocksDbAddressBookStore(Metrics metrics, NodeRustEnvironment nodeRustEnv Natives.builder(nodeRustEnvironment, RocksDbAddressBookStore::isMigrated) .measure(timer.label(new Metrics.MethodId(RocksDbAddressBookStore.class, "isMigrated"))) .build(new TypeToken<>() {}); - - dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } public boolean upsertEntry(AddressBookEntryDTO entry) { @@ -144,9 +138,7 @@ public void reset() { } public ImmutableList getAllEntries() { - return this.getAllFunc.call(tuple()).stream() - .map(value -> NodeSborCodecs.decode(value, dtoCodec)) - .collect(ImmutableList.toImmutableList()); + return this.getAllFunc.call(tuple()).stream().collect(ImmutableList.toImmutableList()); } public boolean isMigrated() { @@ -160,7 +152,7 @@ public void markAsMigrated() { private final Natives.Call1 removeOneFunc; private final Natives.Call1 upsertOneFunc; private final Natives.Call1 resetFunc; - private final Natives.Call1> getAllFunc; + private final Natives.Call1> getAllFunc; private final Natives.Call1 isMigratedFunc; private final Natives.Call1 markAsMigratedFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java index 689297f9d0..905638144a 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/p2p/RocksDbHighPriorityPeersStore.java @@ -71,8 +71,8 @@ import com.radixdlt.lang.Option; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; -import com.radixdlt.sbor.NodeSborCodecs; -import com.radixdlt.sbor.codec.Codec; +import com.radixdlt.sbor.codec.CodecMap; +import com.radixdlt.sbor.codec.StructCodec; import java.util.List; public class RocksDbHighPriorityPeersStore { @@ -80,6 +80,14 @@ public class RocksDbHighPriorityPeersStore { System.loadLibrary("corerust"); } + public record HighPriorityPeers(List ids) { + public static void registerCodec(CodecMap codecMap) { + codecMap.register( + HighPriorityPeers.class, + codecs -> StructCodec.fromRecordComponents(HighPriorityPeers.class, codecs)); + } + } + private static native byte[] upsertAllHighPriorityPeers( NodeRustEnvironment nodeRustEnvironment, byte[] payload); @@ -89,8 +97,6 @@ private static native byte[] getAllHighPriorityPeers( private static native byte[] resetHighPriorityPeers( NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private final Codec> dtoCodec; - public static RocksDbHighPriorityPeersStore create( Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { return new RocksDbHighPriorityPeersStore(metrics, nodeRustEnvironment); @@ -120,26 +126,21 @@ private RocksDbHighPriorityPeersStore(Metrics metrics, NodeRustEnvironment nodeR new Metrics.MethodId( RocksDbHighPriorityPeersStore.class, "resetHighPriorityPeers"))) .build(new TypeToken<>() {}); - - dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } public void storeHighPriorityPeers(List ids) { - this.upsertAllHighPriorityPeersFunc.call(ids); + this.upsertAllHighPriorityPeersFunc.call(new HighPriorityPeers(ids)); } public List getHighPriorityPeers() { - return this.getAllHighPriorityPeersFunc - .call(tuple()) - .map(value -> NodeSborCodecs.decode(value, dtoCodec)) - .or(List.of()); + return this.getAllHighPriorityPeersFunc.call(tuple()).map(HighPriorityPeers::ids).or(List.of()); } public void reset() { this.resetFunc.call(tuple()); } - private final Natives.Call1, Tuple0> upsertAllHighPriorityPeersFunc; - private final Natives.Call1> getAllHighPriorityPeersFunc; + private final Natives.Call1 upsertAllHighPriorityPeersFunc; + private final Natives.Call1> getAllHighPriorityPeersFunc; private final Natives.Call1 resetFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java index b0bbdbd6f6..4a08c57135 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/RocksDbSafetyStore.java @@ -71,8 +71,6 @@ import com.radixdlt.lang.Option; import com.radixdlt.monitoring.Metrics; import com.radixdlt.sbor.Natives; -import com.radixdlt.sbor.NodeSborCodecs; -import com.radixdlt.sbor.codec.Codec; public class RocksDbSafetyStore { static { @@ -88,8 +86,6 @@ private static native byte[] markAsMigrated( private static native byte[] isMigrated(NodeRustEnvironment nodeRustEnvironment, byte[] payload); - private final Codec dtoCodec; - public static RocksDbSafetyStore create( Metrics metrics, NodeRustEnvironment nodeRustEnvironment) { return new RocksDbSafetyStore(metrics, nodeRustEnvironment); @@ -113,8 +109,6 @@ private RocksDbSafetyStore(Metrics metrics, NodeRustEnvironment nodeRustEnvironm Natives.builder(nodeRustEnvironment, RocksDbSafetyStore::isMigrated) .measure(timer.label(new Metrics.MethodId(RocksDbSafetyStore.class, "isMigrated"))) .build(new TypeToken<>() {}); - - dtoCodec = NodeSborCodecs.resolveCodec(new TypeToken<>() {}); } public void upsert(SafetyStateDTO state) { @@ -122,7 +116,7 @@ public void upsert(SafetyStateDTO state) { } public Option get() { - return this.getFunc.call(tuple()).map(value -> NodeSborCodecs.decode(value, dtoCodec)); + return this.getFunc.call(tuple()); } public boolean isMigrated() { @@ -134,7 +128,7 @@ public void markAsMigrated() { } private final Natives.Call1 upsertFunc; - private final Natives.Call1> getFunc; + private final Natives.Call1> getFunc; private final Natives.Call1 isMigratedFunc; private final Natives.Call1 markAsMigratedFunc; } diff --git a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java index 002bd87484..aa93a9e209 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java @@ -78,6 +78,7 @@ import com.radixdlt.p2p.AddressBookEntryDTO; import com.radixdlt.p2p.NodeIdDTO; import com.radixdlt.p2p.PeerAddressEntryDTO; +import com.radixdlt.p2p.RocksDbHighPriorityPeersStore.HighPriorityPeers; import com.radixdlt.prometheus.LedgerStatus; import com.radixdlt.prometheus.RecentSelfProposalMissStatistic; import com.radixdlt.protocol.*; @@ -222,6 +223,7 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) { BFTValidatorDTO.registerCodec(codecMap); BFTValidatorIdDTO.registerCodec(codecMap); HighQCDTO.registerCodec(codecMap); + HighPriorityPeers.registerCodec(codecMap); LedgerHeaderDTO.registerCodec(codecMap); NextEpochDTO.registerCodec(codecMap); QuorumCertificateDTO.registerCodec(codecMap); diff --git a/core-rust/node-common/src/java/utils.rs b/core-rust/node-common/src/java/utils.rs index 7a5012ee59..5af40d96c1 100644 --- a/core-rust/node-common/src/java/utils.rs +++ b/core-rust/node-common/src/java/utils.rs @@ -115,20 +115,6 @@ pub fn jni_sbor_coded_fallible_call( - env: &JNIEnv, - encoded_request: jbyteArray, - method: impl FnOnce(Vec) -> JavaResult, -) -> jbyteArray { - jni_call(env, || { - let result = jni_jbytearray_to_vector(env, encoded_request).and_then(method); - jni_slice_to_jbytearray(env, &result.to_java().unwrap()) - }) - .unwrap_or_else(std::ptr::null_mut) -} - /// Executes the given function in a way that is panic-safe on a JNI-originating stack. /// This is achieved by intercepting any panic (i.e. `catch_unwind()`) and throwing a Java-side /// `RustPanicException` instead. diff --git a/core-rust/p2p/src/address_book_components.rs b/core-rust/p2p/src/address_book_components.rs index 55eea9ae5f..b9847a133b 100644 --- a/core-rust/p2p/src/address_book_components.rs +++ b/core-rust/p2p/src/address_book_components.rs @@ -1,20 +1,71 @@ -use crate::engine_prelude::Sbor; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ -/// The ID of the node stored in the Address book (Secp256k1 public key) -#[derive(Clone, Copy, Sbor)] -pub struct AddressBookNodeId(pub [u8; AddressBookNodeId::LENGTH]); - -impl AddressBookNodeId { - pub const LENGTH: usize = 33; - - pub fn new(id: [u8; Self::LENGTH]) -> Self { - Self(id) - } - - pub fn as_bytes(&self) -> &[u8; Self::LENGTH] { - &self.0 - } -} +use crate::components; +use crate::engine_prelude::*; +use crate::engine_prelude::{ScryptoCategorize, ScryptoDecode, ScryptoEncode}; +use sbor::define_single_versioned; /// Timestamp of the various peer-related events // At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and @@ -28,23 +79,41 @@ impl AddressBookNodeId { type PeerTimestamp = i64; /// Peer address entry with all components -#[derive(Clone, Sbor)] +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub struct PeerAddress { pub encoded_uri: Vec, pub latest_connection_status: Option, pub last_seen: Option, } -#[derive(Clone, Copy, Sbor)] +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub enum ConnectionStatus { Connected, Disconnected, } +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct NodeIdDTO { + pub key: components::RawPublicKey, +} + /// Address book entry -#[derive(Clone, Sbor)] -pub struct AddressBookEntry { - pub node_id: AddressBookNodeId, +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct AddressBookEntryV1 { + pub node_id: NodeIdDTO, pub banned_until: Option, pub known_addresses: Vec, } + +define_single_versioned! { + #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] + pub VersionedAddressBookEntry(AddressBookEntryVersions) => AddressBookEntry = AddressBookEntryV1 +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct HighPriorityPeersV1(Vec); + +define_single_versioned! { + #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] + pub VersionedHighPriorityPeers(HighPriorityPeersVersions) => HighPriorityPeers = HighPriorityPeersV1 +} diff --git a/core-rust/p2p/src/column_families.rs b/core-rust/p2p/src/column_families.rs index 7b2374175e..53a4bdbf57 100644 --- a/core-rust/p2p/src/column_families.rs +++ b/core-rust/p2p/src/column_families.rs @@ -1,8 +1,73 @@ -use crate::address_book_components::AddressBookNodeId; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::address_book_components::*; +use crate::components::RawPublicKey; use crate::migration::MigrationStatus; -use crate::typed_cf_api::AddressBookNodeIdDbCodec; -use node_common::store::typed_cf_api::SborDbCodec; -use node_common::store::typed_cf_api::{DefaultCf, DirectDbCodec, UnitDbCodec}; +use crate::safety_store_components::{SafetyState, VersionedSafetyState}; +use crate::typed_cf_api::RawPublicKeyDbCodec; +use node_common::store::typed_cf_api::{DefaultCf, SborDbCodec, UnitDbCodec, VersionedCf}; /// Address book and safety state store migration status. Filled once during the migration. pub struct MigrationStatusCf; @@ -17,32 +82,33 @@ impl DefaultCf for MigrationStatusCf { /// Address book pub struct AddressBookCf; -impl DefaultCf for AddressBookCf { - type Key = AddressBookNodeId; - type Value = Vec; +impl VersionedCf for AddressBookCf { + type Key = RawPublicKey; + type Value = AddressBookEntry; - const DEFAULT_NAME: &'static str = "address_book"; - type KeyCodec = AddressBookNodeIdDbCodec; - type ValueCodec = DirectDbCodec; + const VERSIONED_NAME: &'static str = "address_book"; + type KeyCodec = RawPublicKeyDbCodec; + type VersionedValue = VersionedAddressBookEntry; } /// Safety store pub struct SafetyStoreCf; -impl DefaultCf for SafetyStoreCf { +impl VersionedCf for SafetyStoreCf { type Key = (); - type Value = Vec; + type Value = SafetyState; - const DEFAULT_NAME: &'static str = "safety_store"; + const VERSIONED_NAME: &'static str = "safety_store"; type KeyCodec = UnitDbCodec; - type ValueCodec = DirectDbCodec; + type VersionedValue = VersionedSafetyState; } +/// High priority peers pub struct HighPriorityPeersCf; -impl DefaultCf for HighPriorityPeersCf { +impl VersionedCf for HighPriorityPeersCf { type Key = (); - type Value = Vec; + type Value = HighPriorityPeers; - const DEFAULT_NAME: &'static str = "high_priority_peers"; + const VERSIONED_NAME: &'static str = "high_priority_peers"; type KeyCodec = UnitDbCodec; - type ValueCodec = DirectDbCodec; + type VersionedValue = VersionedHighPriorityPeers; } diff --git a/core-rust/p2p/src/components.rs b/core-rust/p2p/src/components.rs new file mode 100644 index 0000000000..d6925504a4 --- /dev/null +++ b/core-rust/p2p/src/components.rs @@ -0,0 +1,36 @@ +use crate::engine_prelude::{ScryptoCategorize, ScryptoDecode, ScryptoEncode}; + +/// The ID of the node stored in the Address book (Secp256k1 public key) +#[derive( + Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode, PartialOrd, Ord, PartialEq, Eq, +)] +#[sbor(transparent)] +pub struct RawPublicKey(pub [u8; RawPublicKey::LENGTH]); + +impl RawPublicKey { + pub const LENGTH: usize = 33; + + pub fn new(id: [u8; Self::LENGTH]) -> Self { + Self(id) + } + + pub fn as_bytes(&self) -> &[u8; Self::LENGTH] { + &self.0 + } +} + +/// The Secp256K1 signature +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct Signature(pub [u8; Signature::LENGTH]); + +impl Signature { + pub const LENGTH: usize = 65; // v(1) + r(32) + s(32) + + pub fn new(signature: [u8; Self::LENGTH]) -> Self { + Self(signature) + } + + pub fn as_bytes(&self) -> &[u8; Self::LENGTH] { + &self.0 + } +} diff --git a/core-rust/p2p/src/lib.rs b/core-rust/p2p/src/lib.rs index 5e67756683..40a63c031a 100644 --- a/core-rust/p2p/src/lib.rs +++ b/core-rust/p2p/src/lib.rs @@ -1,7 +1,9 @@ pub mod address_book_components; pub mod column_families; +pub mod components; pub mod migration; pub mod rocks_db; +pub mod safety_store_components; pub mod traits; pub mod typed_cf_api; diff --git a/core-rust/p2p/src/migration.rs b/core-rust/p2p/src/migration.rs index d2894e1811..67edbc1259 100644 --- a/core-rust/p2p/src/migration.rs +++ b/core-rust/p2p/src/migration.rs @@ -1,3 +1,67 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + use sbor::Sbor; /// Status of the migration diff --git a/core-rust/p2p/src/rocks_db.rs b/core-rust/p2p/src/rocks_db.rs index cedc90f478..cd1e703c3a 100644 --- a/core-rust/p2p/src/rocks_db.rs +++ b/core-rust/p2p/src/rocks_db.rs @@ -66,12 +66,12 @@ use std::path::PathBuf; use node_common::rocksdb::{ColumnFamilyDescriptor, Options, DB}; -use crate::address_book_components::AddressBookNodeId; -use crate::column_families::{ - AddressBookCf, HighPriorityPeersCf, MigrationStatusCf, SafetyStoreCf, -}; +use crate::address_book_components::*; +use crate::column_families::*; +use crate::components::RawPublicKey; use crate::engine_prelude::*; use crate::migration::MigrationStatus; +use crate::safety_store_components::SafetyState; use crate::traits::node::{AddressBookStore, HighPriorityPeersStore, SafetyStateStore}; use node_common::store::rocks_db::*; use node_common::store::typed_cf_api::*; @@ -96,8 +96,8 @@ use node_common::store::typed_cf_api::*; /// define them manually (rather than using the `Into`, which is refactor-sensitive). const ALL_ADDRESS_BOOK_COLUMN_FAMILIES: [&str; 3] = [ - AddressBookCf::DEFAULT_NAME, - HighPriorityPeersCf::DEFAULT_NAME, + AddressBookCf::VERSIONED_NAME, + HighPriorityPeersCf::VERSIONED_NAME, MigrationStatusCf::DEFAULT_NAME, ]; @@ -157,7 +157,7 @@ impl ActualSafetyStoreDatabase { } impl AddressBookStore for AddressBookDatabase { - fn remove_one(&self, node_id: &AddressBookNodeId) -> bool { + fn remove_one(&self, node_id: &RawPublicKey) -> bool { let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); @@ -168,11 +168,11 @@ impl AddressBookStore for AddressBookDatabase { true } - fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool { + fn upsert_one(&self, node_id: &RawPublicKey, entry: &AddressBookEntry) -> bool { let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); - context.put(node_id, &entry.to_vec()); + context.put(node_id, &entry); true } @@ -181,7 +181,7 @@ impl AddressBookStore for AddressBookDatabase { open_rw_context(&self.rocks).cf(AddressBookCf).delete_all(); } - fn get_all(&self) -> Vec> { + fn get_all(&self) -> Vec { open_rw_context(&self.rocks).cf(AddressBookCf).get_all() } @@ -200,13 +200,13 @@ impl AddressBookStore for AddressBookDatabase { } impl HighPriorityPeersStore for AddressBookDatabase { - fn upsert_all_high_priority_peers(&self, peers: &[u8]) { + fn upsert_all_high_priority_peers(&self, peers: &HighPriorityPeers) { open_rw_context(&self.rocks) .cf(HighPriorityPeersCf) - .put(&(), &peers.to_vec()); + .put(&(), &peers); } - fn get_all_high_priority_peers(&self) -> Option> { + fn get_all_high_priority_peers(&self) -> Option { open_rw_context(&self.rocks) .cf(HighPriorityPeersCf) .get(&()) @@ -220,13 +220,13 @@ impl HighPriorityPeersStore for AddressBookDatabase { } impl SafetyStateStore for SafetyStoreDatabase { - fn upsert_safety_state(&self, safety_state: &[u8]) { + fn upsert_safety_state(&self, safety_state: &SafetyState) { open_rw_context(&self.rocks) .cf(SafetyStoreCf) - .put(&(), &safety_state.to_vec()); + .put(&(), safety_state); } - fn get_safety_state(&self) -> Option> { + fn get_safety_state(&self) -> Option { open_rw_context(&self.rocks).cf(SafetyStoreCf).get(&()) } diff --git a/core-rust/p2p/src/safety_store_components.rs b/core-rust/p2p/src/safety_store_components.rs new file mode 100644 index 0000000000..daddea9161 --- /dev/null +++ b/core-rust/p2p/src/safety_store_components.rs @@ -0,0 +1,203 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +use crate::components::{RawPublicKey, Signature}; +use crate::engine_prelude::*; +use sbor::define_single_versioned; + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct SafetyStateV1 { + validator_id: BFTValidatorId, + round: Round, + last_vote: Option, +} + +define_single_versioned! { + #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] + pub VersionedSafetyState(SafetyStateVersions) => SafetyState = SafetyStateV1 +} + +/// Safety state components. Note that these structs are intended only for proper encoding/deconding +/// of the safety state. They may repeat existing structs defined elsewhere. + +/// Timestamp of the various safety state components. +// At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and +// do something like shown below to transparently convert to/from internal representation +// (once there will be real usage at Rust side). +// #[sbor( +// as_type = "i64", +// as_ref = "self.timestamp()", +// from_value = "Self(DateTime::from_timestamp(value, 0))" +// )] +type SafetyStateTimestamp = i64; + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct BFTHeader { + round: Round, + vertex_id: VertexId, + ledger_header: LedgerHeader, +} + +#[derive( + Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode, Ord, PartialOrd, Eq, PartialEq, +)] +pub struct BFTValidator { + power: Vec, + validator_id: BFTValidatorId, +} + +#[derive( + Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode, Ord, PartialOrd, Eq, PartialEq, +)] +pub struct BFTValidatorId { + key: RawPublicKey, + validator_address: ComponentAddress, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct HighQC { + highest_quorum_certificate: QuorumCertificate, + highest_committed_quorum_certificate: QuorumCertificate, + highest_timeout_certificate: Option, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct LedgerHeader { + epoch: i64, + round: Round, + state_version: i64, + hashes: LedgerHashes, + consensus_parent_round_timestamp_ms: SafetyStateTimestamp, + proposer_timestamp_ms: SafetyStateTimestamp, + next_epoch: Option, + next_protocol_version: Option, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct LedgerHashes { + pub state_root: RawHash, + pub transaction_root: RawHash, + pub receipt_root: RawHash, +} + +define_wrapped_hash! { + RawHash +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct NextEpoch { + epoch: i64, + validators: BTreeSet, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct QuorumCertificate { + signatures: TimestampedECDSASignatures, + vote_data: VoteData, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct Round { + round: i64, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct TimeoutCertificate { + epoch: i64, + round: Round, + signatures: TimestampedECDSASignatures, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct TimestampedECDSASignature { + timestamp: SafetyStateTimestamp, + signature: Signature, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct TimestampedECDSASignatures { + node_to_timestamped_signature: BTreeMap, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct VertexId { + id_bytes: Vec, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct Vote { + author: BFTValidatorId, + high_quorum_certificate: HighQC, + vote_data: VoteData, + timestamp: SafetyStateTimestamp, + signature: Signature, + timeout_signature: Option, +} + +#[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] +pub struct VoteData { + proposed: BFTHeader, + parent: BFTHeader, + committed: Option, +} diff --git a/core-rust/p2p/src/traits.rs b/core-rust/p2p/src/traits.rs index d6bb755d21..f2e2aff914 100644 --- a/core-rust/p2p/src/traits.rs +++ b/core-rust/p2p/src/traits.rs @@ -65,29 +65,30 @@ use crate::engine_prelude::*; pub mod node { - use crate::address_book_components::AddressBookNodeId; - use super::*; + use crate::address_book_components::*; + use crate::components::RawPublicKey; + use crate::safety_store_components::SafetyState; pub trait AddressBookStore { - fn remove_one(&self, node_id: &AddressBookNodeId) -> bool; - fn upsert_one(&self, node_id: &AddressBookNodeId, entry: &[u8]) -> bool; + fn remove_one(&self, node_id: &RawPublicKey) -> bool; + fn upsert_one(&self, node_id: &RawPublicKey, entry: &AddressBookEntry) -> bool; fn reset(&self); - fn get_all(&self) -> Vec>; + fn get_all(&self) -> Vec; fn is_migrated(&self) -> bool; fn mark_as_migrated(&self); } // At Java side it is represented as part of AddressBookStore pub trait HighPriorityPeersStore { - fn upsert_all_high_priority_peers(&self, peers: &[u8]); - fn get_all_high_priority_peers(&self) -> Option>; + fn upsert_all_high_priority_peers(&self, peers: &HighPriorityPeers); + fn get_all_high_priority_peers(&self) -> Option; fn reset_high_priority_peers(&self); } pub trait SafetyStateStore { - fn upsert_safety_state(&self, safety_state: &[u8]); - fn get_safety_state(&self) -> Option>; + fn upsert_safety_state(&self, safety_state: &SafetyState); + fn get_safety_state(&self) -> Option; fn is_migrated(&self) -> bool; fn mark_as_migrated(&self); } diff --git a/core-rust/p2p/src/typed_cf_api.rs b/core-rust/p2p/src/typed_cf_api.rs index c10f7235da..3ad01a8aed 100644 --- a/core-rust/p2p/src/typed_cf_api.rs +++ b/core-rust/p2p/src/typed_cf_api.rs @@ -62,25 +62,24 @@ * permissions under this License. */ -use crate::address_book_components::AddressBookNodeId; -use crate::engine_prelude::*; +use crate::components::RawPublicKey; use node_common::store::typed_cf_api::{BoundedDbCodec, DbCodec}; #[derive(Clone, Default)] -pub struct AddressBookNodeIdDbCodec {} +pub struct RawPublicKeyDbCodec {} -impl DbCodec for AddressBookNodeIdDbCodec { - fn encode(&self, value: &AddressBookNodeId) -> Vec { +impl DbCodec for RawPublicKeyDbCodec { + fn encode(&self, value: &RawPublicKey) -> Vec { value.as_bytes().to_vec() } - fn decode(&self, bytes: &[u8]) -> AddressBookNodeId { - AddressBookNodeId::new(bytes.try_into().expect("Invalid NodeId")) + fn decode(&self, bytes: &[u8]) -> RawPublicKey { + RawPublicKey::new(bytes.try_into().expect("Invalid NodeId")) } } -impl BoundedDbCodec for AddressBookNodeIdDbCodec { +impl BoundedDbCodec for RawPublicKeyDbCodec { fn upper_bound_encoding(&self) -> Vec { - vec![0xFF; AddressBookNodeId::LENGTH] + vec![0xFF; RawPublicKey::LENGTH] } } diff --git a/core-rust/state-manager/src/jni/p2p/address_book.rs b/core-rust/state-manager/src/jni/p2p/address_book.rs index 7713aa52b6..99ac69cbb4 100644 --- a/core-rust/state-manager/src/jni/p2p/address_book.rs +++ b/core-rust/state-manager/src/jni/p2p/address_book.rs @@ -68,7 +68,7 @@ use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; -use p2p::address_book_components::{AddressBookEntry, AddressBookNodeId}; +use p2p::address_book_components::{AddressBookEntry, NodeIdDTO}; use p2p::traits::node::AddressBookStore; #[no_mangle] @@ -78,9 +78,9 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_removeOne( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |request: AddressBookNodeId| -> bool { + jni_sbor_coded_call(&env, node_id, |request: NodeIdDTO| -> bool { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .remove_one(&request) + .remove_one(&request.key) }) } @@ -92,12 +92,8 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_upsertOne( address_entry: jbyteArray, ) -> jbyteArray { jni_sbor_coded_call(&env, address_entry, |request: AddressBookEntry| -> bool { - // It should be safe to unwrap here, as this operation was already successful inside - // jni_sbor_coded_call. - let entity = jni_jbytearray_to_vector(&env, address_entry).unwrap(); - JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .upsert_one(&request.node_id, &entity) + .upsert_one(&request.node_id.key, &request) }) } @@ -120,7 +116,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbAddressBookStore_getAll( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec> { + jni_sbor_coded_call(&env, node_id, |_: ()| -> Vec { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context).get_all() }) } diff --git a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs index 8474db2584..f97eada408 100644 --- a/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs +++ b/core-rust/state-manager/src/jni/p2p/high_priority_peers.rs @@ -66,10 +66,10 @@ use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; -use node_common::java::*; - use crate::engine_prelude::*; use crate::jni::node_rust_environment::JNINodeRustEnvironment; +use node_common::java::*; +use p2p::address_book_components::HighPriorityPeers; use p2p::traits::node::HighPriorityPeersStore; #[no_mangle] @@ -79,10 +79,9 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_upsertAll j_rust_global_context: JObject, payload: jbyteArray, ) -> jbyteArray { - jni_raw_sbor_fallible_call(&env, payload, |bytes| { + jni_sbor_coded_call(&env, payload, |bytes| { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) - .upsert_all_high_priority_peers(&bytes); - Ok(()) + .upsert_all_high_priority_peers(&bytes) }) } @@ -93,7 +92,7 @@ extern "system" fn Java_com_radixdlt_p2p_RocksDbHighPriorityPeersStore_getAllHig j_rust_global_context: JObject, payload: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, payload, |_: ()| -> Option> { + jni_sbor_coded_call(&env, payload, |_: ()| -> Option { JNINodeRustEnvironment::get_address_book_database(&env, j_rust_global_context) .get_all_high_priority_peers() }) diff --git a/core-rust/state-manager/src/jni/p2p/safety_store.rs b/core-rust/state-manager/src/jni/p2p/safety_store.rs index 02bc33650b..da1e375192 100644 --- a/core-rust/state-manager/src/jni/p2p/safety_store.rs +++ b/core-rust/state-manager/src/jni/p2p/safety_store.rs @@ -68,6 +68,7 @@ use jni::objects::{JClass, JObject}; use jni::sys::jbyteArray; use jni::JNIEnv; use node_common::java::*; +use p2p::safety_store_components::SafetyState; use p2p::traits::node::SafetyStateStore; #[no_mangle] @@ -77,10 +78,9 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_upsert( j_rust_global_context: JObject, request: jbyteArray, ) -> jbyteArray { - jni_raw_sbor_fallible_call(&env, request, |bytes| { + jni_sbor_coded_call(&env, request, |safety_state| { JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) - .upsert_safety_state(&bytes); - Ok(()) + .upsert_safety_state(&safety_state) }) } @@ -91,7 +91,7 @@ extern "system" fn Java_com_radixdlt_safety_RocksDbSafetyStore_get( j_rust_global_context: JObject, node_id: jbyteArray, ) -> jbyteArray { - jni_sbor_coded_call(&env, node_id, |_: ()| -> Option> { + jni_sbor_coded_call(&env, node_id, |_: ()| -> Option { JNINodeRustEnvironment::get_safety_store_database(&env, j_rust_global_context) .get_safety_state() }) diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index d97bfe9fb2..7f0cfbf470 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -81,7 +81,7 @@ public class RocksSafetyStateStoreTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test - public void test_address_book_entries_can_be_saved_and_restored() { + public void test_safety_state_can_be_saved_and_restored() { runTest( safetyStore -> { // Fresh store is empty From d9c609d5de5c34c5118709fbcd3ce795abbebd67 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 2 Sep 2024 14:45:48 +0200 Subject: [PATCH 36/40] Fix serialization for the NextEpoch. --- .../main/java/com/radixdlt/safety/NextEpochDTO.java | 4 ++-- core-rust/p2p/src/safety_store_components.rs | 2 +- .../consensus/safety/RocksSafetyStateStore.java | 4 +--- .../consensus/safety/RocksSafetyStateStoreTest.java | 10 ++++++++-- .../radixdlt/utils/SerializerTestDataGenerator.java | 3 +-- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java index 793aacfb1d..34968060fd 100644 --- a/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java +++ b/core-rust-bridge/src/main/java/com/radixdlt/safety/NextEpochDTO.java @@ -66,9 +66,9 @@ import com.radixdlt.sbor.codec.CodecMap; import com.radixdlt.sbor.codec.StructCodec; -import java.util.Set; +import java.util.List; -public record NextEpochDTO(long epoch, Set validators) { +public record NextEpochDTO(long epoch, List validators) { public static void registerCodec(CodecMap codecMap) { codecMap.register( NextEpochDTO.class, codecs -> StructCodec.fromRecordComponents(NextEpochDTO.class, codecs)); diff --git a/core-rust/p2p/src/safety_store_components.rs b/core-rust/p2p/src/safety_store_components.rs index daddea9161..4aef20179a 100644 --- a/core-rust/p2p/src/safety_store_components.rs +++ b/core-rust/p2p/src/safety_store_components.rs @@ -148,7 +148,7 @@ define_wrapped_hash! { #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub struct NextEpoch { epoch: i64, - validators: BTreeSet, + validators: Vec, } #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] diff --git a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java index 8e88acecd6..41405866de 100644 --- a/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java +++ b/core/src/main/java/com/radixdlt/consensus/safety/RocksSafetyStateStore.java @@ -193,9 +193,7 @@ private static com.radixdlt.statecomputer.commit.LedgerHashes toDTO(LedgerHashes private static NextEpochDTO toDTO(NextEpoch epoch) { return new NextEpochDTO( epoch.getEpoch(), - epoch.getValidators().stream() - .map(RocksSafetyStateStore::toDTO) - .collect(ImmutableSet.toImmutableSet())); + epoch.getValidators().stream().map(RocksSafetyStateStore::toDTO).toList()); } private static BFTValidatorDTO toDTO(BFTValidator bftValidator) { diff --git a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java index 7f0cfbf470..e3f01b54d6 100644 --- a/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/safety/RocksSafetyStateStoreTest.java @@ -66,12 +66,13 @@ import static org.junit.Assert.*; -import com.radixdlt.consensus.bft.BFTValidatorId; import com.radixdlt.monitoring.Metrics; import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.rev2.NodeRustEnvironmentHelper; import com.radixdlt.safety.RocksDbSafetyStore; +import com.radixdlt.utils.SerializerTestDataGenerator; import java.io.IOException; +import java.util.Optional; import java.util.function.Consumer; import org.junit.Rule; import org.junit.Test; @@ -88,7 +89,7 @@ public void test_safety_state_can_be_saved_and_restored() { assertTrue(safetyStore.get().isEmpty()); // Save a safety state - var safetyState = SafetyState.initialState(BFTValidatorId.random()); + var safetyState = createSafetyState(); safetyStore.commitState(safetyState); // Retrieve the saved safety state @@ -98,6 +99,11 @@ public void test_safety_state_can_be_saved_and_restored() { }); } + private static SafetyState createSafetyState() { + var vote = SerializerTestDataGenerator.randomVote(); + return SafetyState.create(vote.getAuthor(), vote.getRound(), Optional.of(vote)); + } + private void runTest(Consumer test) { try (var environment = NodeRustEnvironmentHelper.createNodeRustEnvironment(folder.newFolder().getPath())) { diff --git a/core/src/test/java/com/radixdlt/utils/SerializerTestDataGenerator.java b/core/src/test/java/com/radixdlt/utils/SerializerTestDataGenerator.java index 4940f96eeb..f6945e547b 100644 --- a/core/src/test/java/com/radixdlt/utils/SerializerTestDataGenerator.java +++ b/core/src/test/java/com/radixdlt/utils/SerializerTestDataGenerator.java @@ -66,7 +66,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.BFTValidator; import com.radixdlt.consensus.bft.BFTValidatorId; @@ -118,7 +117,7 @@ public static VoteData randomVoteData() { public static BFTHeader randomBFTHeader() { return new BFTHeader( randomRound(), - HashCode.fromLong(random.nextLong()), + HashUtils.random256(), LedgerHeader.create( Math.abs(random.nextLong()), randomRound(), From 6668337c2343dc010fcb60f3e58572c46d1143dc Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 2 Sep 2024 14:55:42 +0200 Subject: [PATCH 37/40] Bump curl version to available one --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1237759f9e..7948b1bbe5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,7 +126,7 @@ RUN apt-get update \ ca-certificates \ build-essential=12.9 \ # https://security-tracker.debian.org/tracker/CVE-2023-38545 - curl=7.88.1-10+deb12u6 \ + curl=7.88.1-10+deb12u7 \ g++-aarch64-linux-gnu \ g++-x86-64-linux-gnu \ libc6-dev-arm64-cross=2.36-8cross1 \ @@ -261,7 +261,7 @@ RUN apt-get update -y \ && apt-get -y --no-install-recommends install \ openjdk-17-jre-headless=17.0.12+7-2~deb12u1 \ # https://security-tracker.debian.org/tracker/CVE-2023-38545 - curl=7.88.1-10+deb12u6 \ + curl=7.88.1-10+deb12u7 \ gettext-base=0.21-12 \ daemontools=1:0.76-8.1 \ # https://security-tracker.debian.org/tracker/CVE-2023-4911 From 3056773df89ca535e16a067f9cc4df790dc584ce Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 2 Sep 2024 15:05:47 +0200 Subject: [PATCH 38/40] Bump libssl-dev version to available one --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7948b1bbe5..3e24172098 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ENV VERSION_LAST_TAG=$VERSION_LAST_TAG RUN apt-get update \ && apt-get install -y --no-install-recommends \ docker.io=20.10.24+dfsg1-1+b3 \ - libssl-dev=3.0.13-1~deb12u1 \ + libssl-dev=3.0.14-1~deb12u1 \ pkg-config=1.8.1-1 \ unzip=6.0-28 \ wget=${WGET_VERSION} \ @@ -131,7 +131,7 @@ RUN apt-get update \ g++-x86-64-linux-gnu \ libc6-dev-arm64-cross=2.36-8cross1 \ libclang-dev=1:14.0-55.7~deb12u1 \ - libssl-dev=3.0.13-1~deb12u1 \ + libssl-dev=3.0.14-1~deb12u1 \ pkg-config=1.8.1-1 \ && rm -rf /var/lib/apt/lists/* From a2829da30de883d62f87be7d0272357318e84527 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 9 Sep 2024 11:13:52 +0200 Subject: [PATCH 39/40] Address PR comments. Signed-off-by: Sergiy Yevtushenko Signed-off-by: Sergiy Yevtushenko --- core-rust/p2p/src/address_book_components.rs | 4 +-- core-rust/p2p/src/column_families.rs | 4 +-- core-rust/p2p/src/components.rs | 8 ++--- core-rust/p2p/src/rocks_db.rs | 6 ++-- core-rust/p2p/src/safety_store_components.rs | 13 +++++--- core-rust/p2p/src/traits.rs | 6 ++-- core-rust/p2p/src/typed_cf_api.rs | 12 +++---- .../src/jni/node_rust_environment.rs | 6 +--- .../java/com/radixdlt/RadixNodeModule.java | 4 +++ .../rev2/modules/REv2StateManagerModule.java | 17 ++-------- .../p2p/test/P2PTestNetworkRunner.java | 33 ++++--------------- 11 files changed, 42 insertions(+), 71 deletions(-) diff --git a/core-rust/p2p/src/address_book_components.rs b/core-rust/p2p/src/address_book_components.rs index b9847a133b..1bec0b8497 100644 --- a/core-rust/p2p/src/address_book_components.rs +++ b/core-rust/p2p/src/address_book_components.rs @@ -68,7 +68,7 @@ use crate::engine_prelude::{ScryptoCategorize, ScryptoDecode, ScryptoEncode}; use sbor::define_single_versioned; /// Timestamp of the various peer-related events -// At present it's just an alias for i64. Later we may want to replace it with struct using crono crate and +// At present, it's just an alias for i64. Later we may want to replace it with struct using crono crate and // do something like shown below to transparently convert to/from internal representation // (once there will be real usage at Rust side). // #[sbor( @@ -94,7 +94,7 @@ pub enum ConnectionStatus { #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub struct NodeIdDTO { - pub key: components::RawPublicKey, + pub key: components::NodeSecp256k1PublicKey, } /// Address book entry diff --git a/core-rust/p2p/src/column_families.rs b/core-rust/p2p/src/column_families.rs index 53a4bdbf57..3c4b247b97 100644 --- a/core-rust/p2p/src/column_families.rs +++ b/core-rust/p2p/src/column_families.rs @@ -63,7 +63,7 @@ */ use crate::address_book_components::*; -use crate::components::RawPublicKey; +use crate::components::NodeSecp256k1PublicKey; use crate::migration::MigrationStatus; use crate::safety_store_components::{SafetyState, VersionedSafetyState}; use crate::typed_cf_api::RawPublicKeyDbCodec; @@ -83,7 +83,7 @@ impl DefaultCf for MigrationStatusCf { /// Address book pub struct AddressBookCf; impl VersionedCf for AddressBookCf { - type Key = RawPublicKey; + type Key = NodeSecp256k1PublicKey; type Value = AddressBookEntry; const VERSIONED_NAME: &'static str = "address_book"; diff --git a/core-rust/p2p/src/components.rs b/core-rust/p2p/src/components.rs index d6925504a4..78c4908c87 100644 --- a/core-rust/p2p/src/components.rs +++ b/core-rust/p2p/src/components.rs @@ -5,9 +5,9 @@ use crate::engine_prelude::{ScryptoCategorize, ScryptoDecode, ScryptoEncode}; Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode, PartialOrd, Ord, PartialEq, Eq, )] #[sbor(transparent)] -pub struct RawPublicKey(pub [u8; RawPublicKey::LENGTH]); +pub struct NodeSecp256k1PublicKey(pub [u8; NodeSecp256k1PublicKey::LENGTH]); -impl RawPublicKey { +impl NodeSecp256k1PublicKey { pub const LENGTH: usize = 33; pub fn new(id: [u8; Self::LENGTH]) -> Self { @@ -21,9 +21,9 @@ impl RawPublicKey { /// The Secp256K1 signature #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] -pub struct Signature(pub [u8; Signature::LENGTH]); +pub struct NodeSignature(pub [u8; NodeSignature::LENGTH]); -impl Signature { +impl NodeSignature { pub const LENGTH: usize = 65; // v(1) + r(32) + s(32) pub fn new(signature: [u8; Self::LENGTH]) -> Self { diff --git a/core-rust/p2p/src/rocks_db.rs b/core-rust/p2p/src/rocks_db.rs index cd1e703c3a..11271eb031 100644 --- a/core-rust/p2p/src/rocks_db.rs +++ b/core-rust/p2p/src/rocks_db.rs @@ -68,7 +68,7 @@ use node_common::rocksdb::{ColumnFamilyDescriptor, Options, DB}; use crate::address_book_components::*; use crate::column_families::*; -use crate::components::RawPublicKey; +use crate::components::NodeSecp256k1PublicKey; use crate::engine_prelude::*; use crate::migration::MigrationStatus; use crate::safety_store_components::SafetyState; @@ -157,7 +157,7 @@ impl ActualSafetyStoreDatabase { } impl AddressBookStore for AddressBookDatabase { - fn remove_one(&self, node_id: &RawPublicKey) -> bool { + fn remove_one(&self, node_id: &NodeSecp256k1PublicKey) -> bool { let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); @@ -168,7 +168,7 @@ impl AddressBookStore for AddressBookDatabase { true } - fn upsert_one(&self, node_id: &RawPublicKey, entry: &AddressBookEntry) -> bool { + fn upsert_one(&self, node_id: &NodeSecp256k1PublicKey, entry: &AddressBookEntry) -> bool { let binding = open_rw_context(&self.rocks); let context = binding.cf(AddressBookCf); diff --git a/core-rust/p2p/src/safety_store_components.rs b/core-rust/p2p/src/safety_store_components.rs index 4aef20179a..cabb80254c 100644 --- a/core-rust/p2p/src/safety_store_components.rs +++ b/core-rust/p2p/src/safety_store_components.rs @@ -62,7 +62,7 @@ * permissions under this License. */ -use crate::components::{RawPublicKey, Signature}; +use crate::components::{NodeSecp256k1PublicKey, NodeSignature}; use crate::engine_prelude::*; use sbor::define_single_versioned; @@ -111,7 +111,7 @@ pub struct BFTValidator { Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode, Ord, PartialOrd, Eq, PartialEq, )] pub struct BFTValidatorId { - key: RawPublicKey, + key: NodeSecp256k1PublicKey, validator_address: ComponentAddress, } @@ -122,6 +122,9 @@ pub struct HighQC { highest_timeout_certificate: Option, } +// FIXME: A duplicate of LedgerHeader from StateManager. +// Made separate te reference only types within this module. De-duplication requires +// careful merging of the other referenced types as well. #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub struct LedgerHeader { epoch: i64, @@ -172,7 +175,7 @@ pub struct TimeoutCertificate { #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] pub struct TimestampedECDSASignature { timestamp: SafetyStateTimestamp, - signature: Signature, + signature: NodeSignature, } #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] @@ -191,8 +194,8 @@ pub struct Vote { high_quorum_certificate: HighQC, vote_data: VoteData, timestamp: SafetyStateTimestamp, - signature: Signature, - timeout_signature: Option, + signature: NodeSignature, + timeout_signature: Option, } #[derive(Debug, Clone, ScryptoCategorize, ScryptoEncode, ScryptoDecode)] diff --git a/core-rust/p2p/src/traits.rs b/core-rust/p2p/src/traits.rs index f2e2aff914..ac1136d973 100644 --- a/core-rust/p2p/src/traits.rs +++ b/core-rust/p2p/src/traits.rs @@ -67,12 +67,12 @@ use crate::engine_prelude::*; pub mod node { use super::*; use crate::address_book_components::*; - use crate::components::RawPublicKey; + use crate::components::NodeSecp256k1PublicKey; use crate::safety_store_components::SafetyState; pub trait AddressBookStore { - fn remove_one(&self, node_id: &RawPublicKey) -> bool; - fn upsert_one(&self, node_id: &RawPublicKey, entry: &AddressBookEntry) -> bool; + fn remove_one(&self, node_id: &NodeSecp256k1PublicKey) -> bool; + fn upsert_one(&self, node_id: &NodeSecp256k1PublicKey, entry: &AddressBookEntry) -> bool; fn reset(&self); fn get_all(&self) -> Vec; fn is_migrated(&self) -> bool; diff --git a/core-rust/p2p/src/typed_cf_api.rs b/core-rust/p2p/src/typed_cf_api.rs index 3ad01a8aed..9bf833e20c 100644 --- a/core-rust/p2p/src/typed_cf_api.rs +++ b/core-rust/p2p/src/typed_cf_api.rs @@ -62,24 +62,24 @@ * permissions under this License. */ -use crate::components::RawPublicKey; +use crate::components::NodeSecp256k1PublicKey; use node_common::store::typed_cf_api::{BoundedDbCodec, DbCodec}; #[derive(Clone, Default)] pub struct RawPublicKeyDbCodec {} -impl DbCodec for RawPublicKeyDbCodec { - fn encode(&self, value: &RawPublicKey) -> Vec { +impl DbCodec for RawPublicKeyDbCodec { + fn encode(&self, value: &NodeSecp256k1PublicKey) -> Vec { value.as_bytes().to_vec() } - fn decode(&self, bytes: &[u8]) -> RawPublicKey { - RawPublicKey::new(bytes.try_into().expect("Invalid NodeId")) + fn decode(&self, bytes: &[u8]) -> NodeSecp256k1PublicKey { + NodeSecp256k1PublicKey::new(bytes.try_into().expect("Invalid NodeId")) } } impl BoundedDbCodec for RawPublicKeyDbCodec { fn upper_bound_encoding(&self) -> Vec { - vec![0xFF; RawPublicKey::LENGTH] + vec![0xFF; NodeSecp256k1PublicKey::LENGTH] } } diff --git a/core-rust/state-manager/src/jni/node_rust_environment.rs b/core-rust/state-manager/src/jni/node_rust_environment.rs index b9970861f4..fb52268a51 100644 --- a/core-rust/state-manager/src/jni/node_rust_environment.rs +++ b/core-rust/state-manager/src/jni/node_rust_environment.rs @@ -199,11 +199,7 @@ impl JNINodeRustEnvironment { } fn combine(base: &String, ext: &str) -> PathBuf { - let mut path = base.clone(); - path.push(MAIN_SEPARATOR); - path.push_str(ext); - - PathBuf::from(path) + [base, ext].iter().collect() } pub fn cleanup(env: &JNIEnv, j_node_rust_env: JObject) { diff --git a/core/src/main/java/com/radixdlt/RadixNodeModule.java b/core/src/main/java/com/radixdlt/RadixNodeModule.java index 2aeb488636..8f42b60bf1 100644 --- a/core/src/main/java/com/radixdlt/RadixNodeModule.java +++ b/core/src/main/java/com/radixdlt/RadixNodeModule.java @@ -390,6 +390,10 @@ protected void configure() { protocolConfig, ScenariosExecutionConfig.resolveForNetwork(network))); + // Persistence + install(new PersistentSafetyStateStoreModule()); + install(new AddressBookModule()); + // Recovery install(new EpochsSafetyRecoveryModule()); install(new REv2LedgerRecoveryModule()); diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index 63f84507f3..53d7f6f51d 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -115,7 +115,6 @@ public final class REv2StateManagerModule extends AbstractModule { private final ProtocolConfig protocolConfig; private final boolean noFees; private final ScenariosExecutionConfig scenariosExecutionConfig; - private final boolean installPersistence; private REv2StateManagerModule( ProposalLimitsConfig proposalLimitsConfig, @@ -128,8 +127,7 @@ private REv2StateManagerModule( LedgerSyncLimitsConfig ledgerSyncLimitsConfig, ProtocolConfig protocolConfig, boolean noFees, - ScenariosExecutionConfig scenariosExecutionConfig, - boolean installPersistence) { + ScenariosExecutionConfig scenariosExecutionConfig) { this.proposalLimitsConfig = proposalLimitsConfig; this.vertexLimitsConfigOpt = vertexLimitsConfigOpt; this.databaseConfig = databaseConfig; @@ -141,7 +139,6 @@ private REv2StateManagerModule( this.protocolConfig = protocolConfig; this.noFees = noFees; this.scenariosExecutionConfig = scenariosExecutionConfig; - this.installPersistence = installPersistence; } public static REv2StateManagerModule create( @@ -165,8 +162,7 @@ public static REv2StateManagerModule create( ledgerSyncLimitsConfig, protocolConfig, false, - scenariosExecutionConfig, - true); + scenariosExecutionConfig); } public static REv2StateManagerModule createForTesting( @@ -191,8 +187,7 @@ public static REv2StateManagerModule createForTesting( ledgerSyncLimitsConfig, protocolConfig, noFees, - scenariosExecutionConfig, - false); + scenariosExecutionConfig); } @Override @@ -318,12 +313,6 @@ EventProcessor onInsertUpdatePersistVertexStore( bind(new Key>() {}).to(RustMempool.class); bind(MempoolReevaluator.class).to(RustMempool.class); } - - if (installPersistence) { - // Moved here for convenience performing migration - install(new PersistentSafetyStateStoreModule()); - install(new AddressBookModule()); - } } @ProvidesIntoSet diff --git a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java index 0b08d49326..9d6f95860f 100644 --- a/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java +++ b/core/src/test/java/com/radixdlt/p2p/test/P2PTestNetworkRunner.java @@ -106,8 +106,8 @@ import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.ProtocolState; import com.radixdlt.store.BerkeleyDbDefaults; -import com.radixdlt.store.NodeStorageLocation; import com.radixdlt.utils.properties.RuntimeProperties; +import com.sleepycat.je.EnvironmentConfig; import java.io.IOException; import java.util.Objects; import java.util.function.Function; @@ -203,37 +203,16 @@ private static Injector createInjector( protected void configure() { bind(AddressBookPersistence.class).to(BerkeleyAddressBookStore.class); bind(PersistentSafetyStateStore.class).to(BerkeleySafetyStateStore.class); - Multibinder.newSetBinder(binder(), NodeAutoCloseable.class) - .addBinding() - .to(BerkeleySafetyStateStore.class); - } - @Provides - @Singleton - BerkeleySafetyStateStore safetyStateStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleySafetyStateStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + var binder = Multibinder.newSetBinder(binder(), NodeAutoCloseable.class); + binder.addBinding().to(PersistentSafetyStateStore.class); + binder.addBinding().to(AddressBookPersistence.class); } @Provides @Singleton - BerkeleyAddressBookStore bdbAddressBookStore( - RuntimeProperties properties, - Serialization serialization, - Metrics metrics, - @NodeStorageLocation String nodeStorageLocation) { - return new BerkeleyAddressBookStore( - serialization, - metrics, - nodeStorageLocation, - BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties)); + EnvironmentConfig environmentConfig(RuntimeProperties properties) { + return BerkeleyDbDefaults.createDefaultEnvConfigFromProperties(properties); } }, Modules.override(new P2PModule(properties)) From 4f621f720e11236dbc7bee025f7cb90d5b440c91 Mon Sep 17 00:00:00 2001 From: Sergiy Yevtushenko Date: Mon, 9 Sep 2024 12:32:33 +0200 Subject: [PATCH 40/40] Bump libssl-dev version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e24172098..4ec5e57be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ENV VERSION_LAST_TAG=$VERSION_LAST_TAG RUN apt-get update \ && apt-get install -y --no-install-recommends \ docker.io=20.10.24+dfsg1-1+b3 \ - libssl-dev=3.0.14-1~deb12u1 \ + libssl-dev=3.0.14-1~deb12u2 \ pkg-config=1.8.1-1 \ unzip=6.0-28 \ wget=${WGET_VERSION} \ @@ -131,7 +131,7 @@ RUN apt-get update \ g++-x86-64-linux-gnu \ libc6-dev-arm64-cross=2.36-8cross1 \ libclang-dev=1:14.0-55.7~deb12u1 \ - libssl-dev=3.0.14-1~deb12u1 \ + libssl-dev=3.0.14-1~deb12u2 \ pkg-config=1.8.1-1 \ && rm -rf /var/lib/apt/lists/*