Merge pull request #2063 from OffchainLabs/fork-node

Fork node

.circleci/config.yml

@@ -57,7 +57,7 @@ workflows:
 jobs:
   go-deps:
     docker:
-      - image: offchainlabs/backend-base:0.4.1
+      - image: offchainlabs/backend-base:0.4.3
     steps:
       - checkout
       - restore_cache: *restore_go_cache
@@ -264,7 +264,7 @@ jobs:
           path: *test-path
   backend:
     docker:
-      - image: offchainlabs/backend-base:0.4.1
+      - image: offchainlabs/backend-base:0.4.3
     environment: # environment variables for the build itself
       TEST_RESULTS: *test-path # path to where test results will be saved
     steps:
@@ -305,7 +305,7 @@ jobs:
 
   backend-rpc:
     docker:
-      - image: offchainlabs/backend-base:0.4.1
+      - image: offchainlabs/backend-base:0.4.3
     resource_class: large
     environment: # environment variables for the build itself
       TEST_RESULTS: *test-path # path to where test results will be saved
@@ -339,7 +339,7 @@ jobs:
 
   arb-bridge-peripherals:
     docker:
-      - image: offchainlabs/integration-base:0.4.1
+      - image: offchainlabs/integration-base:0.4.3
     environment: # environment variables for the build itself
       TEST_RESULTS: *test-path # path to where test results will be saved
     steps:

packages/arb-avm-cpp/avm/CMakeLists.txt

@@ -22,7 +22,6 @@ project (avm LANGUAGES CXX)
 set(LIB_HEADERS
   include/avm/machine.hpp
   include/avm/inboxmessage.hpp
-  include/avm/valueloader.hpp
   include/avm/machinestate/blockreason.hpp
   include/avm/machinestate/datastack.hpp
   include/avm/machinestate/ecops.hpp

packages/arb-avm-cpp/avm/include/avm/machinestate/machinestate.hpp

@@ -21,7 +21,7 @@
 #include <avm/machinestate/blockreason.hpp>
 #include <avm/machinestate/datastack.hpp>
 #include <avm/machinestate/status.hpp>
-#include <avm/valueloader.hpp>
+#include <avm_values/valueloader.hpp>
 
 #include <avm_values/value.hpp>
 #include <avm_values/vmValueParser.hpp>

packages/arb-avm-cpp/avm_values/CMakeLists.txt

@@ -24,15 +24,16 @@ set(LIB_HEADERS
   include/avm_values/tuple.hpp
   include/avm_values/tuplestub.hpp
   include/avm_values/unloadedvalue.hpp
-  include/avm_values/value.hpp
-  include/avm_values/valuetype.hpp
   include/avm_values/bigint.hpp
   include/avm_values/buffer.hpp
   include/avm_values/code.hpp
   include/avm_values/codepoint.hpp
   include/avm_values/codepointstub.hpp
   include/avm_values/pool.hpp
   include/avm_values/opcodes.hpp
+  include/avm_values/value.hpp
+  include/avm_values/valueloader.hpp
+  include/avm_values/valuetype.hpp
   include/avm_values/vmValueParser.hpp
 )
 

packages/arb-avm-cpp/avm_values/include/avm_values/value.hpp

@@ -45,6 +45,8 @@ constexpr uint64_t value_tuple_tag = value_tagged_bit | 1;
 constexpr uint64_t value_hash_pre_image_tag = value_tagged_bit | 2;
 constexpr uint64_t value_buffer_tag = value_tagged_bit | 3;
 
+class ValueLoader;
+
 union TaggedValueContents {
     uint256_t num;
     Tuple tuple;
@@ -372,7 +374,9 @@ Value deserialize_value(const char*& srccode);
 
 void marshal_uint64_t(uint64_t val, std::vector<unsigned char>& buf);
 
-void marshal_value(const Value& val, std::vector<unsigned char>& buf);
+void marshal_value(const Value& val,
+                   std::vector<unsigned char>& buf,
+                   ValueLoader* value_loader);
 
 class Code;
 

packages/arb-avm-cpp/avm_values/src/value.cpp

@@ -19,6 +19,7 @@
 #include <avm_values/pool.hpp>
 #include <avm_values/tuple.hpp>
 #include <avm_values/value.hpp>
+#include <avm_values/valueloader.hpp>
 
 #include <boost/endian/conversion.hpp>
 
@@ -235,6 +236,7 @@ namespace {
 struct Marshaller {
     std::vector<Value>& values;
     std::vector<unsigned char>& buf;
+    ValueLoader* value_loader;
 
     void operator()(const std::shared_ptr<HashPreImage>& val) const {
         buf.push_back(HASH_PRE_IMAGE);
@@ -267,15 +269,20 @@ struct Marshaller {
         val.marshal(buf);
     }
 
-    void operator()(const UnloadedValue&) const {
-        throw std::runtime_error("Cannot marshal unloaded value");
+    void operator()(const UnloadedValue& uv) const {
+        if (value_loader == nullptr) {
+            throw std::runtime_error("Cannot marshal unloaded value");
+        }
+        return visit(*this, value_loader->loadValue(uv.hash()));
     }
 };
 }  // namespace
 
-void marshal_value(const Value& full_val, std::vector<unsigned char>& buf) {
+void marshal_value(const Value& full_val,
+                   std::vector<unsigned char>& buf,
+                   ValueLoader* value_loader) {
     std::vector<Value> values{full_val};
-    Marshaller marshaller{values, buf};
+    Marshaller marshaller{values, buf, value_loader};
     while (!values.empty()) {
         const auto val = std::move(values.back());
         values.pop_back();

packages/arb-avm-cpp/cavm/carbcore.cpp

@@ -114,10 +114,11 @@ ByteSliceArrayResult arbCoreGetLogs(CArbCore* arbcore_ptr,
                                     const void* start_index_ptr,
                                     const void* count_ptr) {
     try {
+        auto arbcore = static_cast<ArbCore*>(arbcore_ptr);
+        auto value_loader = arbcore->makeValueLoader();
         ValueCache cache{1, 0};
-        auto logs = static_cast<ArbCore*>(arbcore_ptr)
-                        ->getLogs(receiveUint256(start_index_ptr),
-                                  receiveUint256(count_ptr), cache);
+        auto logs = arbcore->getLogs(receiveUint256(start_index_ptr),
+                                     receiveUint256(count_ptr), cache);
         if (!logs.status.ok()) {
             return {{}, false};
         }
@@ -127,7 +128,7 @@ ByteSliceArrayResult arbCoreGetLogs(CArbCore* arbcore_ptr,
             std::vector<unsigned char> marshalled_log;
             marshal_uint256_t(log.inbox.count, marshalled_log);
             marshal_uint256_t(log.inbox.accumulator, marshalled_log);
-            marshal_value(log.val, marshalled_log);
+            marshal_value(log.val, marshalled_log, &value_loader);
             data.push_back(move(marshalled_log));
         }
         return {returnCharVectorVector(data), true};
@@ -402,6 +403,8 @@ IndexedDoubleByteSliceArrayResult arbCoreLogsCursorGetLogs(
     auto arbcore = static_cast<ArbCore*>(arbcore_ptr);
     auto cursor_index = receiveUint256(index_ptr);
 
+    auto value_loader = arbcore->makeValueLoader();
+
     try {
         auto result =
             arbcore->logsCursorGetLogs(intx::narrow_cast<size_t>(cursor_index));
@@ -424,7 +427,7 @@ IndexedDoubleByteSliceArrayResult arbCoreLogsCursorGetLogs(
             std::vector<unsigned char> marshalled_value;
             marshal_uint256_t(log.inbox.count, marshalled_value);
             marshal_uint256_t(log.inbox.accumulator, marshalled_value);
-            marshal_value(log.val, marshalled_value);
+            marshal_value(log.val, marshalled_value, &value_loader);
             marshalled_logs.push_back(move(marshalled_value));
         }
 
@@ -434,7 +437,7 @@ IndexedDoubleByteSliceArrayResult arbCoreLogsCursorGetLogs(
             std::vector<unsigned char> marshalled_value;
             marshal_uint256_t(log.inbox.count, marshalled_value);
             marshal_uint256_t(log.inbox.accumulator, marshalled_value);
-            marshal_value(log.val, marshalled_value);
+            marshal_value(log.val, marshalled_value, &value_loader);
             marshalled_deleted_logs.push_back(move(marshalled_value));
         }
 

packages/arb-avm-cpp/cavm/carbstorage.cpp

@@ -94,6 +94,27 @@ void printDatabaseMetadata(CArbStorage* storage_ptr) {
     storage->printDatabaseMetadata();
 }
 
+int initializeExistingArbStorage(CArbStorage* storage_ptr) {
+    auto storage = static_cast<ArbStorage*>(storage_ptr);
+    try {
+        auto result = storage->initializeExisting();
+        if (result.finished) {
+            // Gracefully shutdown
+            return false;
+        }
+        if (!result.status.ok()) {
+            std::cerr << "Error initializing storage: "
+                      << result.status.ToString() << std::endl;
+            return false;
+        }
+
+        return true;
+    } catch (const std::exception& e) {
+        std::cerr << "Exception initializing storage:" << e.what() << std::endl;
+        return false;
+    }
+}
+
 int initializeArbStorage(CArbStorage* storage_ptr,
                          const char* executable_path) {
     auto storage = static_cast<ArbStorage*>(storage_ptr);

packages/arb-avm-cpp/cavm/carbstorage.h

@@ -65,6 +65,7 @@ CArbStorage* createArbStorage(const char* db_path,
                               CArbCoreConfig arb_core_config);
 void printDatabaseMetadata(CArbStorage* storage_ptr);
 int initializeArbStorage(CArbStorage* storage_ptr, const char* executable_path);
+int initializeExistingArbStorage(CArbStorage* storage_ptr);
 int arbStorageInitialized(CArbStorage* storage_ptr);
 void destroyArbStorage(CArbStorage* storage);
 int closeArbStorage(CArbStorage* storage_ptr);

packages/arb-avm-cpp/cavm/cmachine.cpp

@@ -233,12 +233,12 @@ RawAssertionResult executeAssertion(CMachine* m,
 
         std::vector<unsigned char> logData;
         for (const auto& log : assertion.logs) {
-            marshal_value(log.val, logData);
+            marshal_value(log.val, logData, nullptr);
         }
 
         std::vector<unsigned char> debugPrintData;
         for (const auto& debugPrint : assertion.debug_prints) {
-            marshal_value(debugPrint.val, debugPrintData);
+            marshal_value(debugPrint.val, debugPrintData, nullptr);
         }
 
         // TODO extend usage of uint256

packages/arb-avm-cpp/cavm/utils.hpp

@@ -128,22 +128,6 @@ inline Uint64Result returnUint64Result(const ValueResult<uint64_t>& val) {
     return {val.data, true};
 }
 
-inline ByteSlice returnValueResult(const DbResult<Value>& res) {
-    if (std::holds_alternative<rocksdb::Status>(res)) {
-        return {nullptr, 0};
-    }
-
-    std::vector<unsigned char> serialized_value;
-    marshal_value(std::get<CountedData<Value>>(res).data, serialized_value);
-
-    auto value_data =
-        reinterpret_cast<unsigned char*>(malloc(serialized_value.size()));
-    std::copy(serialized_value.begin(), serialized_value.end(), value_data);
-
-    auto void_data = reinterpret_cast<void*>(value_data);
-    return {void_data, static_cast<int>(serialized_value.size())};
-}
-
 inline RawAssertion makeEmptyAssertion() {
     return {0, returnCharVector(std::vector<char>{}),
             0, returnCharVector(std::vector<char>{}),

packages/arb-avm-cpp/cmachine/storage.go

@@ -136,6 +136,16 @@ func (s *ArbStorage) Initialize(contractPath string) error {
 	return nil
 }
 
+func (s *ArbStorage) InitializeExisting() error {
+	defer runtime.KeepAlive(s)
+	success := C.initializeExistingArbStorage(s.c)
+
+	if success == 0 {
+		return errors.Errorf("failed to initialize storage, possibly corrupt or uninitialzed database?")
+	}
+	return nil
+}
+
 func (s *ArbStorage) Initialized() bool {
 	defer runtime.KeepAlive(s)
 	return C.arbStorageInitialized(s.c) == 1

packages/arb-avm-cpp/data_storage/include/data_storage/arbcore.hpp

@@ -19,8 +19,8 @@
 
 #include <avm/machine.hpp>
 #include <avm/machinethread.hpp>
-#include <avm/valueloader.hpp>
 #include <avm_values/bigint.hpp>
+#include <avm_values/valueloader.hpp>
 #include <data_storage/combinedmachinecache.hpp>
 #include <data_storage/datacursor.hpp>
 #include <data_storage/datastorage.hpp>
@@ -155,6 +155,8 @@ class ArbCore {
     ~ArbCore() { abortThread(); }
     void printDatabaseMetadata();
     InitializeResult initialize(const LoadedExecutable& executable);
+    InitializeResult initializeExisting();
+
     [[nodiscard]] bool initialized() const;
     void operator()();
 
@@ -230,9 +232,9 @@ class ArbCore {
         ValueCache& value_cache,
         bool lazy_load) const;
 
+   public:
     [[nodiscard]] ValueLoader makeValueLoader() const;
 
-   public:
     // To be deprecated, use checkpoints instead
     template <class T>
     std::unique_ptr<T> getMachine(uint256_t machineHash,

packages/arb-avm-cpp/data_storage/include/data_storage/arbstorage.hpp

@@ -44,6 +44,7 @@ class ArbStorage {
     void printDatabaseMetadata();
     InitializeResult initialize(const LoadedExecutable& executable);
     InitializeResult initialize(const std::string& executable_path);
+    InitializeResult initializeExisting();
     [[nodiscard]] bool initialized() const;
 
     [[nodiscard]] std::unique_ptr<AggregatorStore> getAggregatorStore() const;

packages/arb-avm-cpp/data_storage/src/arbcore.cpp

@@ -195,7 +195,7 @@ void ArbCore::printDatabaseMetadata() {
     }
 }
 
-InitializeResult ArbCore::initialize(const LoadedExecutable& executable) {
+InitializeResult ArbCore::initializeExisting() {
     // Use latest existing checkpoint
     ValueCache cache{1, 0};
 
@@ -335,9 +335,15 @@ InitializeResult ArbCore::initialize(const LoadedExecutable& executable) {
     if (!status.IsNotFound()) {
         std::cerr << "Error with initial reorg: " << status.ToString()
                   << std::endl;
-        return {status, false};
     }
+    return {status, false};
+}
 
+InitializeResult ArbCore::initialize(const LoadedExecutable& executable) {
+    auto res = initializeExisting();
+    if (!res.status.IsNotFound()) {
+        return res;
+    }
     // Need to initialize database from scratch
     core_code->addSegment(executable.code);
     core_machine = std::make_unique<MachineThread>(
@@ -349,7 +355,7 @@ InitializeResult ArbCore::initialize(const LoadedExecutable& executable) {
 
     ReadWriteTransaction tx(data_storage);
 
-    status = updateSchemaVersion(tx, arbcore_schema_version);
+    auto status = updateSchemaVersion(tx, arbcore_schema_version);
     if (!status.ok()) {
         std::cerr << "failed to save schema version into db: "
                   << status.ToString() << std::endl;

packages/arb-avm-cpp/data_storage/src/arbstorage.cpp

@@ -40,6 +40,10 @@ InitializeResult ArbStorage::initialize(const std::string& executable_path) {
     return initialize(executable);
 }
 
+InitializeResult ArbStorage::initializeExisting() {
+    return arb_core->initializeExisting();
+}
+
 InitializeResult ArbStorage::initialize(const LoadedExecutable& executable) {
     return arb_core->initialize(executable);
 }

packages/arb-avm-cpp/data_storage/src/value/corevalueloader.hpp

@@ -17,9 +17,9 @@
 #ifndef corevalueloader_hpp
 #define corevalueloader_hpp
 
-#include <avm/valueloader.hpp>
 #include <avm_values/code.hpp>
 #include <avm_values/value.hpp>
+#include <avm_values/valueloader.hpp>
 #include <data_storage/datastorage.hpp>
 #include <data_storage/value/valuecache.hpp>
 

packages/arb-avm-cpp/tests/value.cpp

@@ -72,7 +72,7 @@ TEST_CASE("Value marshaling") {
             auto valRaw = reinterpret_cast<const char*>(valBytes.data());
             auto val = deserialize_value(valRaw);
             std::vector<unsigned char> buf;
-            marshal_value(val, buf);
+            marshal_value(val, buf, nullptr);
             auto valptr = (const char*)&buf[0];
             auto newval = deserialize_value(valptr);
             REQUIRE(values_equal(val, newval));

packages/arb-evm/arbos/arbowner.go

@@ -43,6 +43,7 @@ var (
 
 var FeesEnabledParamId = hashing.SoliditySHA3([]byte("FeesEnabled"))
 var ChainOwnerParamId = hashing.SoliditySHA3([]byte("ChainOwner"))
+var ChainIDId = hashing.SoliditySHA3([]byte("ChainID"))
 var NetworkFeeRecipientParamId = hashing.SoliditySHA3([]byte("NetworkFeeRecipient"))
 var CongestionFeeRecipientParamId = hashing.SoliditySHA3([]byte("CongestionFeeRecipient"))
 var DefaultAggregatorParamId = hashing.SoliditySHA3([]byte("DefaultAggregator"))