Support for loading validator keys from Hashicorp Vault

src/commands/start.py

@@ -31,7 +31,7 @@
     update_unused_validator_keys_metric,
 )
 from src.validators.tasks import load_genesis_validators, register_validators
-from src.validators.utils import load_deposit_data, load_keystores
+from src.validators.utils import load_deposit_data, load_validator_keys
 
 logger = logging.getLogger(__name__)
 
@@ -151,6 +151,22 @@
     prompt='Enter the vault address',
     help='Address of the vault to register validators for.',
 )
+@click.option(
+    '--hashicorp-vault-addr',
+    callback=validate_eth_address,
+    envvar='HASHICORP_VAULT_ADDR',
+    help='Address of the vault to register validators for.',
+)
+@click.option(
+    '--hashicorp-vault-token',
+    envvar='HASHICORP_VAULT_TOKEN',
+    help='Token for accessing Hashicopr vault.',
+)
+@click.option(
+    '--hashicorp-vault-key',
+    envvar='HASHICORP_VAULT_KEY',
+    help='Key of the secret where signing keys are stored. Compatible with format recognized by Web3Signer',
+)
 @click.command(help='Start operator service')
 # pylint: disable-next=too-many-arguments,too-many-locals
 def start(
@@ -170,6 +186,9 @@ def start(
     hot_wallet_password_file: str | None,
     max_fee_per_gas_gwei: int,
     database_dir: str | None,
+    hashicorp_vault_addr: str | None,
+    hashicorp_vault_token: str | None,
+    hashicorp_vault_key: str | None,
 ) -> None:
     vault_config = VaultConfig(vault, Path(data_dir))
     if network is None:
@@ -193,6 +212,9 @@ def start(
         hot_wallet_password_file=hot_wallet_password_file,
         max_fee_per_gas_gwei=max_fee_per_gas_gwei,
         database_dir=database_dir,
+        hashicorp_vault_addr=hashicorp_vault_addr,
+        hashicorp_vault_key=hashicorp_vault_key,
+        hashicorp_vault_token=hashicorp_vault_token,
     )
 
     try:
@@ -213,9 +235,9 @@ async def main() -> None:
     # load network validators from ipfs dump
     await load_genesis_validators()
 
-    # load keystores
-    keystores = load_keystores()
-    if not keystores:
+    # load validator keys
+    validator_keys = load_validator_keys()
+    if not validator_keys:
         return
 
     # load deposit data
@@ -235,7 +257,7 @@ async def main() -> None:
     await metrics_server()
 
     # process outdated exit signatures
-    asyncio.create_task(update_exit_signatures_periodically(keystores))
+    asyncio.create_task(update_exit_signatures_periodically(validator_keys))
 
     logger.info('Started operator service')
     with InterruptHandler() as interrupt_handler:
@@ -249,8 +271,8 @@ async def main() -> None:
                 # process new network validators
                 await network_validators_scanner.process_new_events(to_block)
                 # check and register new validators
-                await update_unused_validator_keys_metric(keystores, deposit_data)
-                await register_validators(keystores, deposit_data)
+                await update_unused_validator_keys_metric(validator_keys, deposit_data)
+                await register_validators(validator_keys, deposit_data)
 
                 # submit harvest vault transaction
                 if settings.harvest_vault:

src/commands/validators_exit.py

@@ -17,12 +17,12 @@
 from src.common.validators import validate_eth_address
 from src.common.vault_config import VaultConfig
 from src.config.settings import AVAILABLE_NETWORKS, NETWORKS, settings
-from src.validators.typings import BLSPrivkey, Keystores
-from src.validators.utils import load_keystores
+from src.validators.typings import BLSPrivkey, ValidatorKeys
+from src.validators.utils import load_validator_keys
 
 
 @dataclass
-class ExitKeystore:
+class ExitKeyMaterial:
     private_key: BLSPrivkey
     public_key: HexStr
     index: int
@@ -73,6 +73,22 @@ class ExitKeystore:
     envvar='VERBOSE',
     is_flag=True,
 )
+@click.option(
+    '--hashicorp-vault-addr',
+    callback=validate_eth_address,
+    envvar='HASHICORP_VAULT_ADDR',
+    help='Address of the vault to register validators for.',
+)
+@click.option(
+    '--hashicorp-vault-token',
+    envvar='HASHICORP_VAULT_TOKEN',
+    help='Token for accessing Hashicopr vault.',
+)
+@click.option(
+    '--hashicorp-vault-key',
+    envvar='HASHICORP_VAULT_KEY',
+    help='Key of the secret where signing keys are stored. Compatible with format recognized by Web3Signer',
+)
 @click.command(help='Performs a voluntary exit for active vault validators.')
 # pylint: disable-next=too-many-arguments
 def validators_exit(
@@ -82,6 +98,9 @@ def validators_exit(
     consensus_endpoints: str,
     data_dir: str,
     verbose: bool,
+    hashicorp_vault_addr: str | None,
+    hashicorp_vault_token: str | None,
+    hashicorp_vault_key: str | None,
 ) -> None:
     # pylint: disable=duplicate-code
     vault_config = VaultConfig(vault, Path(data_dir))
@@ -95,6 +114,9 @@ def validators_exit(
         vault_dir=vault_config.vault_dir,
         consensus_endpoints=consensus_endpoints,
         verbose=verbose,
+        hashicorp_vault_addr=hashicorp_vault_addr,
+        hashicorp_vault_key=hashicorp_vault_key,
+        hashicorp_vault_token=hashicorp_vault_token,
     )
     try:
         asyncio.run(main(count))
@@ -103,12 +125,12 @@ def validators_exit(
 
 
 async def main(count: int | None) -> None:
-    keystores = load_keystores()
+    keystores = load_validator_keys()
     if not keystores:
         raise click.ClickException('Keystores not found.')
     fork = await consensus_client.get_consensus_fork()
 
-    exit_keystores = await _get_exit_keystores(keystores)
+    exit_keystores = await _get_exit_key_material(keystores)
     if not exit_keystores:
         raise click.ClickException('There are no active validators.')
 
@@ -164,10 +186,10 @@ def _get_exit_signature(
     return exit_signature
 
 
-async def _get_exit_keystores(keystores: Keystores) -> list[ExitKeystore]:
+async def _get_exit_key_material(validator_keys: ValidatorKeys) -> list[ExitKeyMaterial]:
     """Fetches validators consensus info."""
     results = []
-    public_keys = list(keystores.keys())
+    public_keys = list(validator_keys.keys())
     exited_statuses = [x.value for x in EXITING_STATUSES]
 
     for i in range(0, len(public_keys), settings.validators_fetch_chunk_size):
@@ -179,9 +201,9 @@ async def _get_exit_keystores(keystores: Keystores) -> list[ExitKeystore]:
                 continue
 
             results.append(
-                ExitKeystore(
+                ExitKeyMaterial(
                     public_key=beacon_validator['validator']['pubkey'],
-                    private_key=keystores[beacon_validator['validator']['pubkey']],
+                    private_key=validator_keys[beacon_validator['validator']['pubkey']],
                     index=int(beacon_validator['index']),
                 )
             )

src/config/settings.py

@@ -4,8 +4,8 @@
 from decouple import config as decouple_config
 from web3 import Web3
 from web3.types import ChecksumAddress
-
 from src.config.networks import GOERLI, NETWORKS, NetworkConfig
+from src.config.typings import HashicorpVaultSettings
 
 DATA_DIR = Path.home() / '.stakewise'
 
@@ -47,6 +47,7 @@ class Settings(metaclass=Singleton):
     ipfs_fetch_endpoints: list[str]
     validators_fetch_chunk_size: int
     sentry_dsn: str
+    hashicorp_vault: HashicorpVaultSettings | None
 
     # pylint: disable-next=too-many-arguments,too-many-locals
     def set(
@@ -67,6 +68,9 @@ def set(
         hot_wallet_file: str | None = None,
         hot_wallet_password_file: str | None = None,
         database_dir: str | None = None,
+        hashicorp_vault_addr: str | None = None,
+        hashicorp_vault_token: str | None = None,
+        hashicorp_vault_key: str | None = None,
     ):
         self.vault = Web3.to_checksum_address(vault)
         self.vault_dir = vault_dir
@@ -96,6 +100,14 @@ def set(
             else vault_dir / 'keystores' / 'password.txt'
         )
 
+        # vault
+        if hashicorp_vault_addr:
+            self.hashicorp_vault = HashicorpVaultSettings(
+                hashicorp_vault_addr, hashicorp_vault_key, hashicorp_vault_token
+            )
+        else:
+            self.hashicorp_vault = None
+
         # hot wallet
         self.hot_wallet_file = (
             Path(hot_wallet_file) if hot_wallet_file else vault_dir / 'wallet' / 'wallet.json'

src/config/typings.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class HashicorpVaultSettings:
+    addr: str
+    key: str
+    token: str

src/exits/tasks.py

@@ -25,7 +25,7 @@
 from src.exits.typings import OraclesApproval, SignatureRotationRequest
 from src.exits.utils import send_signature_rotation_requests
 from src.validators.signing import get_exit_signature_shards
-from src.validators.typings import Keystores
+from src.validators.typings import ValidatorKeys
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +79,7 @@ async def wait_oracle_signature_update(
 
 
 async def update_exit_signatures(
-    keystores: Keystores, oracles: Oracles, outdated_indexes: list[int]
+    keystores: ValidatorKeys, oracles: Oracles, outdated_indexes: list[int]
 ) -> HexStr:
     """Fetches update signature requests from oracles."""
     exit_rotation_batch_limit = oracles.validators_exit_rotation_batch_limit
@@ -133,7 +133,7 @@ async def _fetch_exit_signature_block(oracle_endpoint: str) -> BlockNumber | Non
 
 
 async def get_oracles_approval(
-    oracles: Oracles, keystores: Keystores, validators: dict[int, HexStr]
+    oracles: Oracles, keystores: ValidatorKeys, validators: dict[int, HexStr]
 ) -> OraclesApproval:
     """Fetches approval from oracles."""
     fork = await consensus_client.get_consensus_fork()
@@ -168,7 +168,7 @@ async def get_oracles_approval(
     )
 
 
-async def update_exit_signatures_periodically(keystores: Keystores):
+async def update_exit_signatures_periodically(keystores: ValidatorKeys):
     # Oracle may have lag if operator was stopped
     # during `update_exit_signatures_periodically` process.
     # Wait all oracles sync.

src/validators/execution.py

@@ -22,7 +22,7 @@
 from src.validators.database import NetworkValidatorCrud
 from src.validators.typings import (
     DepositData,
-    Keystores,
+    ValidatorKeys,
     NetworkValidator,
     OraclesApproval,
     Validator,
@@ -162,7 +162,7 @@ async def check_deposit_data_root(deposit_data_root: str) -> None:
 
 
 async def get_available_validators(
-    keystores: Keystores, deposit_data: DepositData, count: int
+    keystores: ValidatorKeys, deposit_data: DepositData, count: int
 ) -> list[Validator]:
     """Fetches vault's available validators."""
     await check_deposit_data_root(deposit_data.tree.root)
@@ -195,7 +195,7 @@ async def get_available_validators(
 
 
 async def update_unused_validator_keys_metric(
-    keystores: Keystores, deposit_data: DepositData
+    keystores: ValidatorKeys, deposit_data: DepositData
 ) -> int:
     try:
         await check_deposit_data_root(deposit_data.tree.root)

src/validators/tasks.py

@@ -25,7 +25,7 @@
 from src.validators.typings import (
     ApprovalRequest,
     DepositData,
-    Keystores,
+    ValidatorKeys,
     NetworkValidator,
     OraclesApproval,
     Validator,
@@ -36,7 +36,7 @@
 
 
 # pylint: disable-next=too-many-locals
-async def register_validators(keystores: Keystores, deposit_data: DepositData) -> None:
+async def register_validators(keystores: ValidatorKeys, deposit_data: DepositData) -> None:
     """Registers vault validators."""
     vault_balance, update_state_call = await get_withdrawable_assets()
     if settings.network == GNOSIS:
@@ -124,7 +124,7 @@ async def register_validators(keystores: Keystores, deposit_data: DepositData) -
 
 async def create_approval_request(
     oracles: Oracles,
-    keystores: Keystores,
+    keystores: ValidatorKeys,
     validators: list[Validator],
     registry_root: Bytes32,
     multi_proof: MultiProof,

src/validators/typings.py

@@ -6,7 +6,7 @@
 from sw_utils.typings import Bytes32
 
 BLSPrivkey = NewType('BLSPrivkey', bytes)
-Keystores = NewType('Keystores', dict[HexStr, BLSPrivkey])
+ValidatorKeys = NewType('Keystores', dict[HexStr, BLSPrivkey])
 
 
 @dataclass