Finalized transitioned anchor

CHANGELOG.md

@@ -13,3 +13,5 @@
 
 ### Bug Fixes
 - Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library
+
+- Fixed a checkpoint sync issue where Teku couldn't start when the finalized state has been transitioned with empty slot(s) 

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/TransitionedAnchorAcceptanceTest.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.google.common.io.Resources;
+import java.net.URL;
+import java.security.SecureRandom;
+import java.util.List;
+import java.util.stream.IntStream;
+import org.apache.tuweni.bytes.Bytes32;
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.teku.infrastructure.bytes.Bytes20;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
+import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator.InitialStateData;
+import tech.pegasys.teku.test.acceptance.dsl.TekuBeaconNode;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfigBuilder;
+import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeys;
+import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeystores;
+
+public class TransitionedAnchorAcceptanceTest extends AcceptanceTestBase {
+  private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
+  // SEED is chosen to have empty slots in the end of 3rd epoch.
+  // TODO: Good to find one with earlier end epoch empty slots
+  private static final byte[] SEED = new byte[] {0x11, 0x03, 0x04};
+
+  @Test
+  @SuppressWarnings("DoNotCreateSecureRandomDirectly")
+  void shouldMaintainValidatorsInMutableClient() throws Exception {
+    final SecureRandom rnd = SecureRandom.getInstance("SHA1PRNG");
+    rnd.setSeed(SEED);
+    final String networkName = "swift";
+
+    final List<BLSKeyPair> node1Validators =
+        IntStream.range(0, 16).mapToObj(__ -> BLSKeyPair.random(rnd)).toList();
+    final List<BLSKeyPair> node2Validators =
+        IntStream.range(0, 3).mapToObj(__ -> BLSKeyPair.random(rnd)).toList();
+
+    final ValidatorKeystores node1ValidatorKeystores =
+        new ValidatorKeystores(
+            node1Validators.stream()
+                .map(keyPair -> new ValidatorKeys(keyPair, Bytes32.random(rnd), false))
+                .toList());
+    // will be non-active in the beginning
+    final ValidatorKeystores node2ValidatorKeystores =
+        new ValidatorKeystores(
+            node2Validators.stream()
+                .map(keyPair -> new ValidatorKeys(keyPair, Bytes32.random(rnd), false))
+                .toList());
+
+    final InitialStateData genesis =
+        createGenesisGenerator()
+            .network(networkName)
+            .withAltairEpoch(UInt64.ZERO)
+            .withBellatrixEpoch(UInt64.ZERO)
+            .validatorKeys(node1ValidatorKeystores, node2ValidatorKeystores)
+            .generate();
+
+    final String node1FeeRecipient = "0xFE3B557E8Fb62b89F4916B721be55cEb828dBd73";
+    final TekuBeaconNode beaconNode1 =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withStubExecutionEngine()
+                .withJwtSecretFile(JWT_FILE)
+                .withNetwork(networkName)
+                .withRealNetwork()
+                .withInitialState(genesis)
+                .withAltairEpoch(UInt64.ZERO)
+                .withBellatrixEpoch(UInt64.ZERO)
+                .withValidatorProposerDefaultFeeRecipient(node1FeeRecipient)
+                .withReadOnlyKeystorePath(node1ValidatorKeystores)
+                .build());
+
+    beaconNode1.start();
+    beaconNode1.waitForFinalizationAfter(UInt64.valueOf(3));
+
+    final BeaconState finalizedState = beaconNode1.fetchFinalizedState();
+    final UInt64 finalizedSlot = finalizedState.getSlot();
+    final UInt64 finalizedEpoch = beaconNode1.getSpec().computeEpochAtSlot(finalizedSlot);
+    final UInt64 finalizedEpochFirstSlot =
+        beaconNode1.getSpec().computeStartSlotAtEpoch(finalizedEpoch);
+    assertNotEquals(finalizedSlot, finalizedEpochFirstSlot);
+    final UInt64 maxFinalizedSlot =
+        beaconNode1.getSpec().computeStartSlotAtEpoch(finalizedEpoch.plus(1));
+    System.out.println(
+        "State slot: "
+            + finalizedState.getSlot()
+            + ", maxSlot: "
+            + maxFinalizedSlot
+            + ", finalized Epoch: "
+            + finalizedEpoch);
+    assertTrue(finalizedState.getSlot().isLessThan(maxFinalizedSlot));
+    final BeaconState transitionedFinalizedState =
+        beaconNode1.getSpec().processSlots(finalizedState, maxFinalizedSlot);
+
+    final String node2FeeRecipient = "0xfe3b557E8Fb62B89F4916B721be55ceB828DBd55";
+    final TekuBeaconNode beaconNode2 =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withStubExecutionEngine()
+                .withJwtSecretFile(JWT_FILE)
+                .withNetwork(networkName)
+                .withRealNetwork()
+                .withInitialState(new InitialStateData(transitionedFinalizedState))
+                .withAltairEpoch(UInt64.ZERO)
+                .withBellatrixEpoch(UInt64.ZERO)
+                .withValidatorProposerDefaultFeeRecipient(node2FeeRecipient)
+                .withReadOnlyKeystorePath(node2ValidatorKeystores)
+                .withPeers(beaconNode1)
+                .build());
+
+    beaconNode2.start();
+    beaconNode2.waitUntilInSyncWith(beaconNode1);
+    beaconNode2.waitForBlockSatisfying(
+        block -> {
+          final Bytes20 blockFeeRecipient =
+              block.getMessage().getBody().getOptionalExecutionPayload().get().getFeeRecipient();
+          assertEquals(Bytes20.fromHexString(node2FeeRecipient), blockFeeRecipient);
+        });
+    beaconNode2.waitForNewFinalization();
+  }
+}

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java

@@ -454,6 +454,16 @@ public void waitForNewFinalization() {
         MINUTES);
   }
 
+  public void waitForFinalizationAfter(final UInt64 newFinalizedEpoch) {
+    LOG.debug("Wait for finalized block");
+    waitFor(
+        () ->
+            assertThat(fetchStateFinalityCheckpointEpoch().orElseThrow())
+                .isGreaterThanOrEqualTo(newFinalizedEpoch),
+        2,
+        MINUTES);
+  }
+
   public void waitForMilestone(final SpecMilestone expectedMilestone) {
     waitForLogMessageContaining("Activating network upgrade: " + expectedMilestone.name());
   }
@@ -584,6 +594,18 @@ private Optional<Pair<Bytes32, Boolean>> fetchBeaconHeadRootData() throws IOExce
     return Optional.of(Pair.of(root, executionOptimistic));
   }
 
+  private Optional<UInt64> fetchStateFinalityCheckpointEpoch() throws IOException {
+    final String result =
+        httpClient.get(getRestApiUrl(), "/eth/v1/beacon/states/head/finality_checkpoints");
+    if (result.isEmpty()) {
+      return Optional.empty();
+    }
+    final JsonNode jsonNode = OBJECT_MAPPER.readTree(result);
+    final UInt64 finalizedEpoch =
+        UInt64.valueOf(jsonNode.get("data").get("finalized").get("epoch").asText());
+    return Optional.of(finalizedEpoch);
+  }
+
   private Optional<Bytes32> fetchBeaconHeadRoot() throws IOException {
     return fetchBeaconHeadRootData().map(Pair::getLeft);
   }
@@ -776,4 +798,13 @@ public void expectElOffline() throws IOException {
   public void expectElOnline() throws IOException {
     assertThat(getStatusElOffline()).isFalse();
   }
+
+  public BeaconState fetchFinalizedState() throws IOException {
+    final Bytes beaconStateBytes =
+        httpClient.getAsBytes(
+            getRestApiUrl(),
+            "eth/v2/debug/beacon/states/finalized",
+            Map.of("Accept", "application/octet-stream"));
+    return spec.deserializeBeaconState(Bytes.wrap(beaconStateBytes));
+  }
 }

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/DepositProviderTest.java

@@ -492,6 +492,33 @@ void whenCallingForFinalizedSnapshotAndSnapshotAvailable_SnapshotReturned() {
         .isEqualTo(UInt64.valueOf(30));
   }
 
+  @Test
+  void whenUsingTransitionedAnchorPoint_FinalizationIsNotFailed() throws Exception {
+    setup(16);
+    Bytes32 finalizedBlockRoot = Bytes32.fromHexString("0x01");
+    updateStateEth1DepositIndex(10);
+    updateStateEth1DataDepositCount(10);
+    mockDepositsFromEth1Block(0, 20);
+    final AnchorPoint anchorPoint = mock(AnchorPoint.class);
+    final UpdatableStore store = mock(UpdatableStore.class);
+    when(recentChainData.getStore()).thenReturn(store);
+    when(store.getLatestFinalized()).thenReturn(anchorPoint);
+    final BeaconState transitionedState = spec.processSlots(state, state.getSlot().plus(2));
+    when(anchorPoint.getState()).thenAnswer(__ -> transitionedState);
+    when(eth1DataCache.getEth1DataAndHeight(eq(transitionedState.getEth1Data())))
+        .thenReturn(Optional.empty());
+
+    assertThat(depositProvider.getDepositMapSize()).isEqualTo(20);
+
+    depositProvider.onNewFinalizedCheckpoint(new Checkpoint(UInt64.ONE, finalizedBlockRoot), false);
+
+    assertThat(depositProvider.getDepositMapSize()).isEqualTo(10);
+    List<DepositWithIndex> availableDeposits = depositProvider.getAvailableDeposits();
+    assertThat(availableDeposits.size()).isEqualTo(10);
+
+    verify(eth1DataCache).getEth1DataAndHeight(eq(transitionedState.getEth1Data()));
+  }
+
   private void checkThatDepositProofIsValid(final List<DepositWithIndex> depositsWithIndex) {
     final SpecVersion genesisSpec = spec.getGenesisSpec();
     depositsWithIndex.forEach(

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -441,6 +441,10 @@ public UInt64 computeStartSlotAtEpoch(final UInt64 epoch) {
     return atEpoch(epoch).miscHelpers().computeStartSlotAtEpoch(epoch);
   }
 
+  public UInt64 computeEndSlotAtEpoch(final UInt64 epoch) {
+    return atEpoch(epoch).miscHelpers().computeEndSlotAtEpoch(epoch);
+  }
+
   public UInt64 computeEpochAtSlot(final UInt64 slot) {
     return atSlot(slot).miscHelpers().computeEpochAtSlot(slot);
   }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blocks/StateAndBlockSummary.java

@@ -33,11 +33,15 @@ public class StateAndBlockSummary implements BeaconBlockSummary {
   protected StateAndBlockSummary(final BeaconBlockSummary blockSummary, final BeaconState state) {
     checkNotNull(blockSummary);
     checkNotNull(state);
+    this.blockSummary = blockSummary;
+    this.state = state;
+    verifyStateAndBlockConsistency();
+  }
+
+  protected void verifyStateAndBlockConsistency() {
     checkArgument(
         blockSummary.getStateRoot().equals(state.hashTreeRoot()),
         "Block state root must match the supplied state");
-    this.blockSummary = blockSummary;
-    this.state = state;
   }
 
   public static StateAndBlockSummary create(final BeaconState state) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/AnchorPoint.java

@@ -17,6 +17,7 @@
 
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSSignature;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
@@ -38,22 +39,19 @@ public class AnchorPoint extends StateAndBlockSummary {
   private final Spec spec;
   private final Checkpoint checkpoint;
   private final boolean isGenesis;
+  private final boolean isTransitioned;
 
   private AnchorPoint(
       final Spec spec,
       final Checkpoint checkpoint,
       final BeaconState state,
       final BeaconBlockSummary blockSummary) {
     super(blockSummary, state);
-    checkArgument(
-        checkpoint.getRoot().equals(blockSummary.getRoot()), "Checkpoint and block must match");
-    checkArgument(
-        checkpoint.getEpochStartSlot(spec).isGreaterThanOrEqualTo(blockSummary.getSlot()),
-        "Block must be at or prior to the start of the checkpoint epoch");
-
     this.spec = spec;
     this.checkpoint = checkpoint;
     this.isGenesis = checkpoint.getEpoch().equals(SpecConfig.GENESIS_EPOCH);
+    this.isTransitioned = state.getSlot().isGreaterThan(blockSummary.getSlot());
+    verifyAnchor();
   }
 
   public static AnchorPoint create(
@@ -62,7 +60,9 @@ public static AnchorPoint create(
       final BeaconState state,
       final Optional<SignedBeaconBlock> block) {
     final BeaconBlockSummary blockSummary =
-        block.<BeaconBlockSummary>map(a -> a).orElseGet(() -> BeaconBlockHeader.fromState(state));
+        block
+            .<BeaconBlockSummary>map(Function.identity())
+            .orElseGet(() -> BeaconBlockHeader.fromState(state));
     return new AnchorPoint(spec, checkpoint, state, blockSummary);
   }
 
@@ -99,7 +99,7 @@ public static AnchorPoint fromInitialState(final Spec spec, final BeaconState st
     } else {
       final BeaconBlockHeader header = BeaconBlockHeader.fromState(state);
 
-      // Calculate closest epoch boundary to use for the checkpoint
+      // Calculate the closest epoch boundary to use for the checkpoint
       final UInt64 epoch = spec.computeNextEpochBoundary(state.getSlot());
       final Checkpoint checkpoint = new Checkpoint(epoch, header.hashTreeRoot());
 
@@ -129,6 +129,42 @@ public static AnchorPoint fromInitialBlockAndState(
     return new AnchorPoint(spec, checkpoint, state, block);
   }
 
+  /**
+   * Skipping verification in the super class. All checks are made in {@link #verifyAnchor()}
+   * instead
+   */
+  @Override
+  protected void verifyStateAndBlockConsistency() {}
+
+  private void verifyAnchor() {
+    final UInt64 blockSlot = blockSummary.getSlot();
+    if (isTransitioned) {
+      // the finalized state is transitioned with empty slot(s)
+      checkArgument(
+          blockSummary.getStateRoot().equals(state.getLatestBlockHeader().getStateRoot()),
+          "Block state root must match the latest block header state root in the state");
+      final int stateAndBlockRootsIndex =
+          blockSlot.mod(spec.getSlotsPerHistoricalRoot(blockSlot)).intValue();
+      checkArgument(
+          blockSummary
+              .getStateRoot()
+              .equals(state.getStateRoots().get(stateAndBlockRootsIndex).get()),
+          "Block state root must match the state root for the block slot %s in the state roots",
+          blockSlot);
+      checkArgument(
+          blockSummary.getRoot().equals(state.getBlockRoots().get(stateAndBlockRootsIndex).get()),
+          "Block root must match the root for the block slot %s in the block roots in the state",
+          blockSlot);
+    } else {
+      super.verifyStateAndBlockConsistency();
+    }
+    checkArgument(
+        checkpoint.getRoot().equals(blockSummary.getRoot()), "Checkpoint and block must match");
+    checkArgument(
+        checkpoint.getEpochStartSlot(spec).isGreaterThanOrEqualTo(blockSlot),
+        "Block must be at or prior to the start of the checkpoint epoch");
+  }
+
   public boolean isGenesis() {
     return isGenesis;
   }
@@ -149,6 +185,10 @@ public UInt64 getEpochStartSlot() {
     return checkpoint.getEpochStartSlot(spec);
   }
 
+  public boolean isTransitioned() {
+    return isTransitioned;
+  }
+
   @Override
   public boolean equals(final Object o) {
     if (this == o) {