Add --set-high-watermark and --remove-high-watermark to watermark-rep…

…air subcommand (#912)

Reuses watermark-repair subcommand slot and epoch options for setting low watermark, switching on the --set-high-watermark boolean option.

When --remove-high-watermark is set to true, all other subcommand options are ignored and high-watermark slot and epoch values in metadata table are set to null.

CHANGELOG.md

@@ -7,6 +7,9 @@
 
 ### Features Added
 - Aws bulk loading for secp256k1 keys in eth1 mode [#889](https://github.com/Consensys/web3signer/pull/889)
+- Add High Watermark functionality [#696](https://github.com/Consensys/web3signer/issues/696) 
+  - Update `watermark-repair` subcommand with new options `--set-high-watermark`, `--remove-high-watermark` [#912](https://github.com/Consensys/web3signer/pull/912)
+  - Add GET `/highWatermark` to eth2 endpoints [#908](https://github.com/Consensys/web3signer/pull/908)
 
 ## 23.9.0
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/WatermarkRepairParameters.java

@@ -12,24 +12,30 @@
  */
 package tech.pegasys.web3signer.dsl.signer;
 
-import java.util.Collections;
-import java.util.List;
-
 public class WatermarkRepairParameters {
 
-  private final long slot;
-  private final long epoch;
-  private final List<String> validators;
+  private final Long slot;
+  private final Long epoch;
+  private final boolean setHighWatermark;
+  private final boolean removeHighWatermark;
 
   public WatermarkRepairParameters(final long slot, final long epoch) {
-    this(slot, epoch, Collections.emptyList());
+    this(slot, epoch, false);
   }
 
   public WatermarkRepairParameters(
-      final long slot, final long epoch, final List<String> validators) {
+      final long slot, final long epoch, final boolean setHighWatermark) {
     this.slot = slot;
     this.epoch = epoch;
-    this.validators = validators;
+    this.setHighWatermark = setHighWatermark;
+    this.removeHighWatermark = false;
+  }
+
+  public WatermarkRepairParameters(final boolean removeHighWatermark) {
+    this.removeHighWatermark = removeHighWatermark;
+    this.slot = null;
+    this.epoch = null;
+    this.setHighWatermark = false;
   }
 
   public long getSlot() {
@@ -40,7 +46,11 @@ public long getEpoch() {
     return epoch;
   }
 
-  public List<String> getValidators() {
-    return validators;
+  public boolean isSetHighWatermark() {
+    return setHighWatermark;
+  }
+
+  public boolean isRemoveHighWatermark() {
+    return removeHighWatermark;
   }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -271,14 +271,29 @@ private CommandArgs createSubCommandArgs() {
       params.add("watermark-repair"); // sub-sub command
       final WatermarkRepairParameters watermarkRepairParameters =
           signerConfig.getWatermarkRepairParameters().get();
-      yamlConfig.append(
-          String.format(
-              YAML_NUMERIC_FMT, "eth2.watermark-repair.slot", watermarkRepairParameters.getSlot()));
-      yamlConfig.append(
-          String.format(
-              YAML_NUMERIC_FMT,
-              "eth2.watermark-repair.epoch",
-              watermarkRepairParameters.getEpoch()));
+      if (watermarkRepairParameters.isRemoveHighWatermark()) {
+        yamlConfig.append(
+            String.format(
+                YAML_BOOLEAN_FMT,
+                "eth2.watermark-repair.remove-high-watermark",
+                watermarkRepairParameters.isRemoveHighWatermark()));
+      } else {
+        yamlConfig.append(
+            String.format(
+                YAML_NUMERIC_FMT,
+                "eth2.watermark-repair.slot",
+                watermarkRepairParameters.getSlot()));
+        yamlConfig.append(
+            String.format(
+                YAML_NUMERIC_FMT,
+                "eth2.watermark-repair.epoch",
+                watermarkRepairParameters.getEpoch()));
+        yamlConfig.append(
+            String.format(
+                YAML_BOOLEAN_FMT,
+                "eth2.watermark-repair.set-high-watermark",
+                watermarkRepairParameters.isSetHighWatermark()));
+      }
     }
 
     return new CommandArgs(params, yamlConfig.toString());

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java

@@ -442,10 +442,17 @@ private List<String> createSubCommandArgs() {
       final WatermarkRepairParameters watermarkRepairParameters =
           signerConfig.getWatermarkRepairParameters().get();
       params.add("watermark-repair");
-      params.add("--epoch");
-      params.add(Long.toString(watermarkRepairParameters.getEpoch()));
-      params.add("--slot");
-      params.add(Long.toString(watermarkRepairParameters.getSlot()));
+      if (watermarkRepairParameters.isRemoveHighWatermark()) {
+        params.add("--remove-high-watermark=true");
+      } else {
+        params.add("--epoch");
+        params.add(Long.toString(watermarkRepairParameters.getEpoch()));
+        params.add("--slot");
+        params.add(Long.toString(watermarkRepairParameters.getSlot()));
+        if (watermarkRepairParameters.isSetHighWatermark()) {
+          params.add("--set-high-watermark=true");
+        }
+      }
     }
 
     return params;

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/WatermarkRepairSubCommandAcceptanceTest.java

@@ -12,14 +12,20 @@
  */
 package tech.pegasys.web3signer.tests;
 
+import static io.restassured.RestAssured.given;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.equalTo;
 import static tech.pegasys.web3signer.dsl.utils.WaitUtils.waitFor;
 
 import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.web3signer.BLSTestUtil;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.eth2.Eth2SigningRequestBody;
 import tech.pegasys.web3signer.dsl.signer.Signer;
 import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
 import tech.pegasys.web3signer.dsl.signer.WatermarkRepairParameters;
+import tech.pegasys.web3signer.dsl.utils.Eth2RequestUtils;
 import tech.pegasys.web3signer.dsl.utils.MetadataFileHelpers;
 import tech.pegasys.web3signer.signing.KeyType;
 
@@ -31,8 +37,11 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.io.Resources;
+import io.restassured.http.ContentType;
 import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
 import org.jdbi.v3.core.Jdbi;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -66,20 +75,10 @@ void setupSigner(final Path testDirectory) {
   void allLowWatermarksAreUpdated(@TempDir final Path testDirectory) throws URISyntaxException {
     setupSigner(testDirectory);
 
-    importSlashingProtectionData(testDirectory);
+    importSlashingProtectionData(testDirectory, "slashing/slashingImport_two_entries.json");
 
-    final SignerConfigurationBuilder repairBuilder = new SignerConfigurationBuilder();
-    repairBuilder.withMode("eth2");
-    repairBuilder.withSlashingEnabled(true);
-    repairBuilder.withSlashingProtectionDbUrl(signer.getSlashingDbUrl());
-    repairBuilder.withSlashingProtectionDbUsername("postgres");
-    repairBuilder.withSlashingProtectionDbPassword("postgres");
-    repairBuilder.withWatermarkRepairParameters(new WatermarkRepairParameters(20000, 30000));
-    repairBuilder.withHttpPort(12345); // prevent wait for Ports file in AT
-
-    final Signer watermarkRepairSigner = new Signer(repairBuilder.build(), null);
-    watermarkRepairSigner.start();
-    waitFor(() -> assertThat(watermarkRepairSigner.isRunning()).isFalse());
+    final SignerConfigurationBuilder commandConfig = commandConfig();
+    executeSubcommand(commandConfig, new WatermarkRepairParameters(20000, 30000));
 
     final Map<Object, List<Map<String, Object>>> watermarks = getWatermarks();
     assertThat(watermarks).hasSize(2);
@@ -106,6 +105,140 @@ void allLowWatermarksAreUpdated(@TempDir final Path testDirectory) throws URISyn
     assertThat(validator2.get("target_epoch")).isEqualTo(epoch);
   }
 
+  @Test
+  void highWatermarkPreventsImportsAndSignatures(@TempDir final Path testDirectory)
+      throws URISyntaxException, JsonProcessingException {
+
+    /*
+       1. Set high watermark
+       2. Importing slashing data beyond high watermark prevents import
+       3. Signing beyond high watermark is prevented
+       4. Resetting high watermark to lower value fails due to low watermark conflict
+       5. Removing high watermark allows slashing import
+       6. Can sign beyond previously removed high watermark
+    */
+
+    setupSigner(testDirectory);
+
+    // Import validator1's data to set the GVR
+    importSlashingProtectionData(testDirectory, "slashing/slashingImport.json");
+
+    final SignerConfigurationBuilder commandConfig = commandConfig();
+
+    // Set high watermark between the two slashing import entries to prevent second entry from being
+    // imported
+    long highWatermarkSlot = 19998L;
+    long highWatermarkEpoch = 7L;
+    executeSubcommand(
+        commandConfig, new WatermarkRepairParameters(highWatermarkSlot, highWatermarkEpoch, true));
+
+    assertGetHighWatermarkEquals(highWatermarkSlot, highWatermarkEpoch);
+
+    // Import with second entry beyond the high watermark
+    importSlashingProtectionData(testDirectory, "slashing/slashingImport_two_entries.json");
+
+    Map<Object, List<Map<String, Object>>> watermarks = getWatermarks();
+    assertThat(watermarks).hasSize(1);
+    Map<String, Object> validator1 =
+        watermarks
+            .get(
+                "0x8f3f44b74d316c3293cced0c48c72e021ef8d145d136f2908931090e7181c3b777498128a348d07b0b9cd3921b5ca537")
+            .get(0);
+    assertThat(validator1.get("slot")).isEqualTo(BigDecimal.valueOf(12345));
+    assertThat(validator1.get("source_epoch")).isEqualTo(BigDecimal.valueOf(5));
+    assertThat(validator1.get("target_epoch")).isEqualTo(BigDecimal.valueOf(6));
+
+    // validator2 is not imported due to high watermark
+    assertThat(
+            watermarks.get(
+                "0x98d083489b3b06b8740da2dfec5cc3c01b2086363fe023a9d7dc1f907633b1ff11f7b99b19e0533e969862270061d884"))
+        .isNull();
+    assertThatAllSignaturesAreFrom(validator1);
+
+    // signing beyond high watermark is prevented
+    Eth2SigningRequestBody blockRequest =
+        Eth2RequestUtils.createBlockRequest(
+            UInt64.valueOf(highWatermarkSlot).increment(), Bytes32.fromHexString("0x"));
+    signer.eth2Sign(keyPair.getPublicKey().toHexString(), blockRequest).then().statusCode(412);
+
+    Eth2SigningRequestBody attestationRequest =
+        Eth2RequestUtils.createAttestationRequest(
+            (int) highWatermarkEpoch + 1,
+            (int) highWatermarkEpoch + 1,
+            UInt64.valueOf(highWatermarkSlot).decrement());
+    signer
+        .eth2Sign(keyPair.getPublicKey().toHexString(), attestationRequest)
+        .then()
+        .statusCode(412);
+
+    // reset high watermark at a lower value fails due to low watermark conflict
+    executeSubcommand(commandConfig, new WatermarkRepairParameters(12344, 6, true));
+    // high watermark is unchanged
+    assertGetHighWatermarkEquals(highWatermarkSlot, highWatermarkEpoch);
+
+    // remove high watermark allows validator2 through
+    executeSubcommand(commandConfig, new WatermarkRepairParameters(true));
+
+    given()
+        .baseUri(signer.getUrl())
+        .get("/api/v1/eth2/highWatermark")
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("", anEmptyMap());
+
+    // With high watermark removed, validator2 is now imported
+    importSlashingProtectionData(testDirectory, "slashing/slashingImport_two_entries.json");
+
+    watermarks = getWatermarks();
+    assertThat(watermarks).hasSize(2);
+
+    validator1 =
+        watermarks
+            .get(
+                "0x8f3f44b74d316c3293cced0c48c72e021ef8d145d136f2908931090e7181c3b777498128a348d07b0b9cd3921b5ca537")
+            .get(0);
+    assertThat(validator1.get("slot")).isEqualTo(BigDecimal.valueOf(12345));
+    assertThat(validator1.get("source_epoch")).isEqualTo(BigDecimal.valueOf(5));
+    assertThat(validator1.get("target_epoch")).isEqualTo(BigDecimal.valueOf(6));
+
+    final Map<String, Object> validator2 =
+        watermarks
+            .get(
+                "0x98d083489b3b06b8740da2dfec5cc3c01b2086363fe023a9d7dc1f907633b1ff11f7b99b19e0533e969862270061d884")
+            .get(0);
+    assertThat(validator2.get("slot")).isEqualTo(BigDecimal.valueOf(19999));
+    assertThat(validator2.get("source_epoch")).isEqualTo(BigDecimal.valueOf(6));
+    assertThat(validator2.get("target_epoch")).isEqualTo(BigDecimal.valueOf(7));
+
+    // signing beyond previously set high watermark is allowed
+    signer.eth2Sign(keyPair.getPublicKey().toHexString(), blockRequest).then().statusCode(200);
+    signer
+        .eth2Sign(keyPair.getPublicKey().toHexString(), attestationRequest)
+        .then()
+        .statusCode(200);
+  }
+
+  private SignerConfigurationBuilder commandConfig() {
+    final SignerConfigurationBuilder repairBuilder = new SignerConfigurationBuilder();
+    repairBuilder.withMode("eth2");
+    repairBuilder.withSlashingEnabled(true);
+    repairBuilder.withSlashingProtectionDbUrl(signer.getSlashingDbUrl());
+    repairBuilder.withSlashingProtectionDbUsername("postgres");
+    repairBuilder.withSlashingProtectionDbPassword("postgres");
+    repairBuilder.withUseConfigFile(true);
+    repairBuilder.withHttpPort(12345); // prevent wait for Ports file in AT
+    return repairBuilder;
+  }
+
+  private void executeSubcommand(
+      final SignerConfigurationBuilder repairBuilder, final WatermarkRepairParameters params) {
+    repairBuilder.withWatermarkRepairParameters(params);
+    final Signer setHighWatermarkSigner = new Signer(repairBuilder.build(), null);
+    setHighWatermarkSigner.start();
+    waitFor(() -> assertThat(setHighWatermarkSigner.isRunning()).isFalse());
+  }
+
   private Map<Object, List<Map<String, Object>>> getWatermarks() {
     final Jdbi jdbi = Jdbi.create(signer.getSlashingDbUrl(), DB_USERNAME, DB_PASSWORD);
     return jdbi.withHandle(
@@ -120,10 +253,9 @@ private Map<Object, List<Map<String, Object>>> getWatermarks() {
                         m -> Bytes.wrap((byte[]) m.get("public_key")).toHexString())));
   }
 
-  private void importSlashingProtectionData(final Path testDirectory) throws URISyntaxException {
-    final Path importFile =
-        new File(Resources.getResource("slashing/slashingImport_two_entries.json").toURI())
-            .toPath();
+  private void importSlashingProtectionData(
+      final Path testDirectory, final String slashingImportPath) throws URISyntaxException {
+    final Path importFile = new File(Resources.getResource(slashingImportPath).toURI()).toPath();
 
     final SignerConfigurationBuilder importBuilder = new SignerConfigurationBuilder();
     importBuilder.withMode("eth2");
@@ -139,4 +271,37 @@ private void importSlashingProtectionData(final Path testDirectory) throws URISy
     importSigner.start();
     waitFor(() -> assertThat(importSigner.isRunning()).isFalse());
   }
+
+  private void assertGetHighWatermarkEquals(
+      final long highWatermarkSlot, final long highWatermarkEpoch) {
+    given()
+        .baseUri(signer.getUrl())
+        .get("/api/v1/eth2/highWatermark")
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("slot", equalTo(String.valueOf(highWatermarkSlot)))
+        .body("epoch", equalTo(String.valueOf(highWatermarkEpoch)));
+  }
+
+  private void assertThatAllSignaturesAreFrom(Map<String, Object> validator1) {
+    final Jdbi jdbi = Jdbi.create(signer.getSlashingDbUrl(), DB_USERNAME, DB_PASSWORD);
+
+    final List<Map<String, Object>> signedBlocks =
+        jdbi.withHandle(h -> h.select("SELECT * from signed_blocks").mapToMap().list());
+    assertThat(signedBlocks).hasSize(1);
+    assertThat(signedBlocks.get(0).get("validator_id")).isEqualTo(validator1.get("validator_id"));
+    assertThat(signedBlocks.get(0).get("slot"))
+        .isEqualTo(new BigDecimal(validator1.get("slot").toString()));
+
+    final List<Map<String, Object>> signedAttestations =
+        jdbi.withHandle(h -> h.select("SELECT * from signed_attestations").mapToMap().list());
+    assertThat(signedAttestations).hasSize(1);
+    assertThat(signedAttestations.get(0).get("validator_id"))
+        .isEqualTo(validator1.get("validator_id"));
+    assertThat(signedAttestations.get(0).get("source_epoch"))
+        .isEqualTo(new BigDecimal(validator1.get("source_epoch").toString()));
+    assertThat(signedAttestations.get(0).get("target_epoch"))
+        .isEqualTo(new BigDecimal(validator1.get("target_epoch").toString()));
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/keymanager/DeleteKeystoresAcceptanceTest.java

@@ -50,12 +50,12 @@ public class DeleteKeystoresAcceptanceTest extends KeyManagerTestBase {
           + "\"data\" : [ {\n"
           + "  \"pubkey\" : \"0x98d083489b3b06b8740da2dfec5cc3c01b2086363fe023a9d7dc1f907633b1ff11f7b99b19e0533e969862270061d884\",\n"
           + "  \"signed_blocks\" : [ {\n"
-          + "    \"slot\" : \"12345\",\n"
+          + "    \"slot\" : \"19999\",\n"
           + "    \"signing_root\" : \"0x4ff6f743a43f3b4f95350831aeaf0a122a1a392922c45d804280284a69eb850b\"\n"
           + "  } ],\n"
           + "  \"signed_attestations\" : [ {\n"
-          + "    \"source_epoch\" : \"5\",\n"
-          + "    \"target_epoch\" : \"6\",\n"
+          + "    \"source_epoch\" : \"6\",\n"
+          + "    \"target_epoch\" : \"7\",\n"
           + "    \"signing_root\" : \"0x30752da173420e64a66f6ca6b97c55a96390a3158a755ecd277812488bb84e57\"\n"
           + "  } ]\n"
           + "} ]\n"