(WIP) feat: sequencer

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
+default-run = "magi"
+edition = "2021"
+license = "AGPL-3.0-only"
 name = "magi"
 version = "0.1.0"
-license = "AGPL-3.0-only"
-edition = "2021"
-default-run = "magi"
 
 [[bin]]
 name = "magi"
@@ -14,57 +14,63 @@ name = "network"
 path = "./bin/network.rs"
 
 [dependencies]
-tokio = { version = "1.28.0", features = ["full"] }
+again = "0.1"
+arc-swap = "1.6.0"
 async-trait = "0.1.73"
+ethers = {version = "2.0.8", features = ["optimism"]}
 eyre = "0.6.8"
-ethers = { version = "2.0.8", features = ["optimism"] }
+futures = "0.3.28"
+futures-timer = "0.3.0"
 hex = "0.4.3"
+jsonrpsee = {version = "0.17.0", features = ["server", "macros"]}
 libflate = "1.2.0"
-openssl = { version = "0.10", features = ["vendored"] }
 once_cell = "1"
-jsonrpsee = {version = "0.17.0", features = ["server", "macros"]}
-futures = "0.3.28"
-futures-timer = "0.3.0"
-again = "0.1"
+openssl = {version = "0.10", features = ["vendored"]}
+tokio = {version = "1.28.0", features = ["full"]}
+thiserror = "1.0"
 
 # Logging and Metrics
+ansi_term = "0.12.1"
 chrono = "0.4.22"
+lazy_static = "1.4.0"
+prometheus_exporter = "0.8.5"
 tracing = "0.1.36"
-ansi_term = "0.12.1"
 tracing-appender = "0.2.2"
-tracing-subscriber = { version = "0.3.16", features = [
-    "fmt",
-    "env-filter",
-    "ansi",
-    "tracing-log",
-] }
-prometheus_exporter = "0.8.5"
-lazy_static = "1.4.0"
+tracing-subscriber = {version = "0.3.16", features = [
+  "fmt",
+  "env-filter",
+  "ansi",
+  "tracing-log",
+]}
 
 # Serialization
-serde = { version = "1.0.152", features = ["derive"] }
+serde = {version = "1.0.152", features = ["derive", "rc"]}
 serde_json = "1.0.93"
 
 # Backend Crates
-uuid = { version = "1.3.0", features = ["v4"] }
 bytes = "1.4.0"
-reqwest = "0.11.14"
 jsonwebtoken = "8.2.0"
 rand = "0.8.5"
+reqwest = "0.11.14"
+uuid = {version = "1.3.0", features = ["v4"]}
 
 # Networking
 discv5 = "0.2.2"
-libp2p = { version = "0.51.3", features = ["macros", "tokio", "tcp", "mplex", "noise", "gossipsub", "ping"] }
-libp2p-identity = { version = "0.1.2", features = ["secp256k1"] }
-unsigned-varint = "0.7.1"
+libp2p = {version = "0.51.3", features = ["macros", "tokio", "tcp", "mplex", "noise", "gossipsub", "ping"]}
+libp2p-identity = {version = "0.1.2", features = ["secp256k1"]}
 snap = "1"
 ssz_rs = "0.8.0"
+unsigned-varint = "0.7.1"
 
 # CLI
-figment = { version = "0.10.8", features = ["toml", "env"] }
-ctrlc = { version = "3.2.3", features = ["termination"] }
-clap = { version = "3.2.18", features = ["derive", "env"] }
+clap = {version = "4.4.4", features = ["derive", "env", "string"]}
+ctrlc = {version = "3.2.3", features = ["termination"]}
 dirs = "4.0.0"
+figment = {version = "0.10.11", features = ["toml", "env"]}
+toml = "0.8.2"
+
+[dev-dependencies]
+tempfile = "3.8.1"
 
 [features]
 default = ["test-utils"]

bin/magi.rs

@@ -1,16 +1,21 @@
-use std::path::PathBuf;
-use std::{env::current_dir, process};
+use std::{fs, net::SocketAddr, path::PathBuf, process};
 
 use clap::Parser;
 use dirs::home_dir;
-use eyre::Result;
+use discv5::enr::{CombinedKey, Enr};
+use eyre::{anyhow, Result};
+use libp2p_identity::secp256k1::SecretKey;
+use serde::Serialize;
 
 use magi::{
-    config::{ChainConfig, CliConfig, Config, SyncMode},
+    config::{
+        secret_key_from_hex, serialize_secret_key, ChainConfig, CliConfig, Config, ConfigBuilder,
+        SequencerConfig, SyncMode,
+    },
+    network,
     runner::Runner,
     telemetry::{self, metrics},
 };
-use serde::Serialize;
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -20,10 +25,11 @@ async fn main() -> Result<()> {
     let logs_dir = cli.logs_dir.clone();
     let logs_rotation = cli.logs_rotation.clone();
     let checkpoint_hash = cli.checkpoint_hash.clone();
-    let config = cli.to_config();
+    let metrics_listen = cli.metrics_listen;
+    let config = cli.to_config()?;
 
     let _guards = telemetry::init(verbose, logs_dir, logs_rotation);
-    metrics::init()?;
+    metrics::init(metrics_listen)?;
 
     let runner = Runner::from_config(config)
         .with_sync_mode(sync_mode)
@@ -37,7 +43,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-#[derive(Parser, Serialize)]
+#[derive(Debug, Parser, Serialize)]
 pub struct Cli {
     #[clap(short, long, default_value = "optimism")]
     network: String,
@@ -51,9 +57,6 @@ pub struct Cli {
     l2_engine_url: Option<String>,
     #[clap(long)]
     jwt_secret: Option<String>,
-    /// Path to a JWT secret to use for authenticated RPC endpoints
-    #[clap(long)]
-    jwt_file: Option<PathBuf>,
     #[clap(short = 'v', long)]
     verbose: bool,
     #[clap(short = 'p', long)]
@@ -68,65 +71,156 @@ pub struct Cli {
     checkpoint_sync_url: Option<String>,
     #[clap(long)]
     devnet: bool,
+    #[clap(long = "sequencer-enabled")]
+    sequencer_enabled: bool,
+    #[clap(long = "sequencer-max-safe-lag", default_value = "0")]
+    sequencer_max_safe_lag: u64,
+
+    /// P2P listening address
+    #[clap(long, default_value = network::LISTENING_AS_STR)]
+    p2p_listen: SocketAddr,
+
+    /// Secret key Secp256k1 for P2P.
+    /// You can pass both the path to the key and the value of the private key itself
+    /// The private key must be in hexadecimal format with a length of 64 characters.
+    /// Example:
+    ///     fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3
+    ///     /path/to/secret_key
+    #[clap(
+        long,
+        value_parser = parse_secret_key_from_cli,
+        verbatim_doc_comment
+    )]
+    #[serde(
+        serialize_with = "serialize_secret_key",
+        skip_serializing_if = "Option::is_none"
+    )]
+    p2p_secret_key: Option<SecretKey>,
+
+    /// Bootnodes to which you need to connect initially. A list of addresses separated by a space is expected in ENR format
+    ///
+    /// If not specified, the optimism mainnet will be used.
+    ///
+    /// Example:
+    ///     enr:<BASE_64>_1 enr:<BASE_64>_2 ... enr:<BASE_64>_N
+    #[clap(
+        long,
+        verbatim_doc_comment,
+        value_delimiter = ' ',
+        num_args = 1..
+    )]
+    p2p_bootnodes: Option<Vec<Enr<CombinedKey>>>,
+
+    /// Secret key Secp256k1 for Sequencer.
+    /// You can pass both the path to the key and the value of the private key itself
+    /// The private key must be in hexadecimal format with a length of 64 characters.
+    /// Example:
+    ///     fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3
+    ///     /path/to/secret_key
+    #[clap(
+        long,
+        value_parser = parse_secret_key_from_cli,
+        verbatim_doc_comment
+    )]
+    #[serde(
+        serialize_with = "serialize_secret_key",
+        skip_serializing_if = "Option::is_none"
+    )]
+    p2p_sequencer_secret_key: Option<SecretKey>,
+
+    /// Metrics listening address.
+    /// The parameter wouldn't be saved as part as config.
+    #[clap(long, default_value = metrics::LISTENING_AS_STR)]
+    metrics_listen: SocketAddr,
+
+    /// Specify the magi working directory. It will store all the necessary data for the launch.
+    #[clap(long, short = 'd', verbatim_doc_comment, default_value = default_working_dir())]
+    #[serde(skip)]
+    working_dir: PathBuf,
+
+    /// Save the configuration to launch Magi in the future
+    /// The configuration will be saved in the working directory named "magi.toml": <WORK_DIR>/magi.toml
+    #[clap(long = "save", short = 's', verbatim_doc_comment)]
+    #[serde(skip)]
+    save_config: bool,
 }
 
 impl Cli {
-    pub fn to_config(self) -> Config {
-        let chain = match self.network.as_str() {
-            "optimism" => ChainConfig::optimism(),
-            "optimism-goerli" => ChainConfig::optimism_goerli(),
-            "optimism-sepolia" => ChainConfig::optimism_sepolia(),
-            "base" => ChainConfig::base(),
-            "base-goerli" => ChainConfig::base_goerli(),
-            "base-sepolia" => ChainConfig::base_sepolia(),
-            file if file.ends_with(".json") => ChainConfig::from_json(file),
-            _ => panic!(
-                "Invalid network name. \\
-                Please use one of the following: 'optimism', 'optimism-goerli', 'base-goerli'. \\
-                You can also use a JSON file path for custom configuration."
-            ),
-        };
+    pub fn to_config(self) -> eyre::Result<Config> {
+        let chain = ChainConfig::try_from(self.network.as_str())?;
 
-        let config_path = home_dir().unwrap().join(".magi/magi.toml");
-        let cli_config = CliConfig::from(self);
-        Config::new(&config_path, cli_config, chain)
-    }
+        let mut work_dir = self.working_dir.clone();
+        if !work_dir.is_absolute() {
+            work_dir = std::env::current_dir()?.join(work_dir);
+        }
 
-    pub fn jwt_secret(&self) -> Option<String> {
-        self.jwt_secret.clone().or(self.jwt_secret_from_file())
-    }
+        let magi_config_path = work_dir.join("magi.toml");
+        let save = self.save_config;
+        let cli_config = CliConfig::try_from(self)?;
 
-    pub fn jwt_secret_from_file(&self) -> Option<String> {
-        let jwt_file = self.jwt_file.as_ref()?;
-        match std::fs::read_to_string(jwt_file) {
-            Ok(content) => Some(content),
-            Err(_) => Cli::default_jwt_secret(),
-        }
-    }
+        let config = ConfigBuilder::default()
+            .chain(chain)
+            .toml(&magi_config_path)
+            .cli(cli_config)
+            .build();
 
-    pub fn default_jwt_secret() -> Option<String> {
-        let cur_dir = current_dir().ok()?;
-        match std::fs::read_to_string(cur_dir.join("jwt.hex")) {
-            Ok(content) => Some(content),
-            Err(_) => {
-                tracing::error!(target: "magi", "Failed to read JWT secret from file: {:?}", cur_dir);
-                None
-            }
+        if save {
+            config.save(magi_config_path)?;
         }
+
+        Ok(config)
     }
 }
 
-impl From<Cli> for CliConfig {
-    fn from(value: Cli) -> Self {
-        let jwt_secret = value.jwt_secret();
-        Self {
+impl TryFrom<Cli> for CliConfig {
+    type Error = eyre::Report;
+
+    fn try_from(value: Cli) -> Result<Self> {
+        let sequencer = match value.sequencer_enabled {
+            true => Some(SequencerConfig::new(value.sequencer_max_safe_lag)),
+            false => None,
+        };
+
+        Ok(Self {
             l1_rpc_url: value.l1_rpc_url,
             l2_rpc_url: value.l2_rpc_url,
             l2_engine_url: value.l2_engine_url,
-            jwt_secret,
+            jwt_secret: value.jwt_secret,
             checkpoint_sync_url: value.checkpoint_sync_url,
             rpc_port: value.rpc_port,
             devnet: value.devnet,
-        }
+            sequencer,
+            p2p_secret_key: value.p2p_secret_key,
+            p2p_listen: value.p2p_listen,
+            p2p_bootnodes: value.p2p_bootnodes,
+            p2p_sequencer_secret_key: value.p2p_sequencer_secret_key,
+        })
     }
 }
+
+/// The incoming value is the path to the key or a string with the key.
+/// The private key must be in hexadecimal format with a length of 64 characters.
+fn parse_secret_key_from_cli(value: &str) -> Result<SecretKey> {
+    secret_key_from_hex(value).or_else(|_| {
+        let path = PathBuf::from(value);
+        let key_string = fs::read_to_string(&path)
+            .map_err(|_| anyhow!("The key file {path:?} was not found."))?
+            .trim()
+            .to_string();
+
+        let key = secret_key_from_hex(&key_string)?;
+
+        Ok(key)
+    })
+}
+
+fn default_working_dir() -> String {
+    home_dir()
+        .expect(
+            "Could not determine the home directory in the operating system. \
+            Specify the working directory using the \"--work-dir\" parameter",
+        )
+        .join(".magi/")
+        .display()
+        .to_string()
+}