Merge pull request stacks-network#4250 from stacks-network/feat/miner…

…-improvements

Feat: new miner improvements

.cargo/config

@@ -1,5 +1,6 @@
 [alias]
 stacks-node = "run --package stacks-node --"
+fmt-stacks = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Module"
 
 # Needed by perf to generate flamegraphs.
 #[target.x86_64-unknown-linux-gnu]

.github/actions/bitcoin-int-tests/Dockerfile.rustfmt

@@ -4,9 +4,10 @@ WORKDIR /src
 
 COPY ./rust-toolchain .
 COPY ./Cargo.toml .
+COPY ./.cargo .
 
 RUN rustup component add rustfmt
 
 COPY . .
 
-RUN cargo fmt --all -- --check
+RUN cargo fmt-stacks --check

.github/workflows/bitcoin-tests.yml

@@ -68,6 +68,7 @@ jobs:
           - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
           - tests::neon_integrations::test_problematic_txs_are_not_stored
           - tests::neon_integrations::use_latest_tip_integration_test
+          - tests::neon_integrations::min_txs
           - tests::should_succeed_handling_malformed_and_valid_txs
     steps:
       ## Setup test environment

CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
+## [2.4.0.0.5]
+
+This introduces a set of improvements to the Stacks miner behavior.  In
+particular:
+* The VRF public key can be re-used across node restarts.
+* Settings that affect mining are hot-reloaded from the config file.  They take
+  effect once the file is updated; there is no longer a need to restart the
+node.
+* The act of changing the miner settings in the config file automatically
+  triggers a subsequent block-build attempt, allowing the operator to force the
+miner to re-try building blocks.
+* This adds a new tip-selection algorithm that minimizes block orphans within a
+  configurable window of time.
+* When configured, the node will automatically stop mining if it is not achieving a
+  targeted win rate over a configurable window of blocks.
+* When configured, the node will selectively mine transactions from only certain
+  addresses, or only of certain types (STX-transfers, contract-publishes,
+contract-calls).
+* When configured, the node will optionally only RBF block-commits if it can
+  produce a block with strictly more transactions.
+
 ## [2.4.0.0.4]
 
 This is a high-priority hotfix that addresses a bug in transaction processing which

CONTRIBUTING.md

@@ -91,6 +91,24 @@ should reference the issue in the commit message. For example:
 fix: incorporate unlocks in mempool admitter, #3623
 ```
 
+## Recommended developer setup
+### Recommended githooks
+
+It is helpful to set up the pre-commit git hook set up, so that Rust formatting issues are caught before
+you push your code. Follow these instruction to set it up:
+
+1. Rename `.git/hooks/pre-commit.sample` to `.git/hooks/pre-commit`
+2. Change the content of `.git/hooks/pre-commit` to be the following
+```sh
+#!/bin/sh
+git diff --name-only --staged | grep '\.rs$' | xargs -P 8 -I {} rustfmt {} --edition 2021 --check --config group_imports=StdExternalCrate,imports_granularity=Module || (
+  echo 'rustfmt failed: run "cargo fmt-stacks"';
+  exit 1
+)
+```
+3. Make it executable by running `chmod +x .git/hooks/pre-commit`
+   That's it! Now your pre-commit hook should be configured on your local machine.
+
 # Creating and Reviewing PRs
 
 This section describes some best practices on how to create and review PRs in this context.  The target audience is people who have commit access to this repository (reviewers), and people who open PRs (submitters).  This is a living document -- developers can and should document their own additional guidelines here.
@@ -366,19 +384,20 @@ A test should be marked `#[ignore]` if:
 
 ## Formatting
 
-This repository uses the default rustfmt formatting style. PRs will be checked against `rustfmt` and will _fail_ if not
-properly formatted.
+PRs will be checked against `rustfmt` and will _fail_ if not properly formatted.
+Unfortunately, some config options that we require cannot currently be set in `.rustfmt` files, so arguments must be passed via the command line.
+Therefore, we handle `rustfmt` configuration using a Cargo alias: `cargo fmt-stacks`
 
 You can check the formatting locally via:
 
 ```bash
-cargo fmt --all -- --check --config group_imports=StdExternalCrate
+cargo fmt-stacks --check
 ```
 
 You can automatically reformat your commit via:
 
 ```bash
-cargo fmt --all -- --config group_imports=StdExternalCrate
+cargo fmt-stacks
 ```
 
 ## Comments

README.md

@@ -67,7 +67,7 @@ You can observe the state machine in action locally by running:
 
 ```bash
 $ cd testnet/stacks-node
-$ cargo run --bin stacks-node -- start --config=./conf/testnet-follower-conf.toml
+$ cargo run --bin stacks-node -- start --config ./conf/testnet-follower-conf.toml
 ```
 
 _On Windows, many tests will fail if the line endings aren't `LF`. Please ensure that you are have git's `core.autocrlf` set to `input` when you clone the repository to avoid any potential issues. This is due to the Clarity language currently being sensitive to line endings._

contrib/miner-queries/get_unconfirmed_block_commmits.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Usage:
+This script is designed to be run from the command line. It takes one or more Bitcoin addresses
+and outputs the extracted block commit data for these addresses.
+
+Example command line usage:
+python3 get_unconfirmed_block_commits.py [btcAddress1] [btcAddress2] ...
+"""
+
+import requests
+import json
+import sys
+
+def read_api_endpoint(url):
+    """
+    Reads data from the specified API endpoint and returns the response.
+
+    Args:
+        url (str): The API endpoint URL.
+
+    Returns:
+        dict: JSON response from the API if successful, otherwise None.
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an exception for non-200 status codes
+        return response.json()  # Assuming a JSON response
+    except requests.exceptions.RequestException as e:
+        return None
+
+def is_block_commit(txn):
+    """
+    Determines whether a given transaction is a block commit.
+
+    Args:
+        txn (dict): The transaction data.
+
+    Returns:
+        bool: True if the transaction is a block commit, otherwise False.
+    """
+    try:
+        vout = txn['vout']
+
+        # Verify the number of recipients.
+        assert(3 <= len(vout) <= 4)
+        block_commit_txn = vout[0]
+        to_stacker_txns = vout[1::2]
+
+        # Verify block commit.
+        # TODO: Add more verification steps if necessary.
+        assert(block_commit_txn['scriptpubkey_type'] == "op_return")
+
+        # Verify PoX Payouts.
+        for to_stacker_txn in to_stacker_txns:
+            # TODO: Add more verification steps if necessary.
+            assert(to_stacker_txn['scriptpubkey_type'] != "op_return")
+
+    except (Exception, AssertionError):
+        return False
+    return True
+
+MEMPOOL_TXN_API = "https://mempool.space/api/address/{btcAddress}/txs/mempool"
+def unconfirmed_block_commit_from_address(btcAddress):
+    """
+    Fetches the first unconfirmed block commit for a given Bitcoin address.
+
+    Args:
+        btcAddress (str): Bitcoin address.
+
+    Returns:
+        dict: The first transaction that is a block commit.
+    """
+    url = MEMPOOL_TXN_API.format(btcAddress=btcAddress)
+    txns = read_api_endpoint(url)
+
+    # Return only the first block commit transaction. This is good enough for now.
+    for txn in txns:
+        if is_block_commit(txn):
+            return txn
+
+def extracted_block_commit_data(txn):
+    """
+    Extracts data from a block commit transaction.
+
+    Args:
+        txn (dict): Block commit transaction.
+
+    Returns:
+        dict: Extracted data from the transaction, or None if extraction fails.
+    """
+    try:
+        vout_start = 1
+        vout_end = len(txn['vout']) - 1
+        spent_utxo = txn['vin'][0]
+        return {
+            'txid': txn['txid'],
+            'burn': sum(pox_payout['value'] for pox_payout in txn['vout'][vout_start:vout_end]),
+            'address': spent_utxo['prevout']['scriptpubkey_address'],
+            'pox_addrs': [txn['vout'][i]['scriptpubkey'] for i in range(vout_start,vout_end)],
+            'input_txid': spent_utxo['txid'],
+            'input_index': spent_utxo['vout'],
+        }
+    except Exception as e:
+        return None
+
+def block_commit_data(btcAddresses):
+    """
+    Fetches and extracts block commit data for a list of Bitcoin addresses.
+
+    Args:
+        btcAddresses (list): List of Bitcoin addresses.
+
+    Returns:
+        list: Extracted block commit data for each address.
+    """
+    return [extracted_block_commit_data(unconfirmed_block_commit_from_address(btcAddress)) \
+            for btcAddress in btcAddresses]
+
+def main():
+    """
+    Main function to run the script. Takes command line arguments as Bitcoin addresses.
+    """
+    btc_addresses = sys.argv[1:]
+    if not btc_addresses:
+        print("No Bitcoin addresses provided. Please provide at least one address.")
+        return
+
+    # Return the data by printing it to stdout.
+    data = block_commit_data(btc_addresses)
+    print(json.dumps([datum for datum in data if datum is not None], indent=1))
+
+if __name__ == "__main__":
+    main()

src/burnchains/burnchain.rs

@@ -316,7 +316,7 @@ impl BurnchainStateTransition {
 }
 
 impl BurnchainSigner {
-    #[cfg(test)]
+    #[cfg(any(test, feature = "testing"))]
     pub fn mock_parts(
         hash_mode: AddressHashMode,
         num_sigs: usize,
@@ -330,7 +330,7 @@ impl BurnchainSigner {
         BurnchainSigner(repr)
     }
 
-    #[cfg(test)]
+    #[cfg(any(test, feature = "testing"))]
     pub fn new_p2pkh(pubk: &StacksPublicKey) -> BurnchainSigner {
         BurnchainSigner::mock_parts(AddressHashMode::SerializeP2PKH, 1, vec![pubk.clone()])
     }

src/chainstate/stacks/db/blocks.rs

@@ -7042,6 +7042,28 @@ impl StacksChainState {
         query_row(&self.db(), sql, args).map_err(Error::DBError)
     }
 
+    /// Get all possible canonical chain tips
+    pub fn get_stacks_chain_tips(&self, sortdb: &SortitionDB) -> Result<Vec<StagingBlock>, Error> {
+        let (consensus_hash, block_bhh) =
+            SortitionDB::get_canonical_stacks_chain_tip_hash(sortdb.conn())?;
+        let sql = "SELECT * FROM staging_blocks WHERE processed = 1 AND orphaned = 0 AND consensus_hash = ?1 AND anchored_block_hash = ?2";
+        let args: &[&dyn ToSql] = &[&consensus_hash, &block_bhh];
+        let Some(staging_block): Option<StagingBlock> =
+            query_row(&self.db(), sql, args).map_err(Error::DBError)?
+        else {
+            return Ok(vec![]);
+        };
+        self.get_stacks_chain_tips_at_height(staging_block.height)
+    }
+
+    /// Get all Stacks blocks at a given height
+    pub fn get_stacks_chain_tips_at_height(&self, height: u64) -> Result<Vec<StagingBlock>, Error> {
+        let sql =
+            "SELECT * FROM staging_blocks WHERE processed = 1 AND orphaned = 0 AND height = ?1";
+        let args: &[&dyn ToSql] = &[&u64_to_sql(height)?];
+        query_rows(&self.db(), sql, args).map_err(Error::DBError)
+    }
+
     /// Get the parent block of `staging_block`.
     pub fn get_stacks_block_parent(
         &self,

src/chainstate/stacks/miner.rs

@@ -114,6 +114,7 @@ impl MinerStatus {
     pub fn get_spend_amount(&self) -> u64 {
         return self.spend_amount;
     }
+
     pub fn set_spend_amount(&mut self, amt: u64) {
         self.spend_amount = amt;
     }

src/chainstate/stacks/mod.rs

@@ -84,7 +84,7 @@ pub use stacks_common::address::{
     C32_ADDRESS_VERSION_TESTNET_MULTISIG, C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
 };
 
-pub const STACKS_BLOCK_VERSION: u8 = 6;
+pub const STACKS_BLOCK_VERSION: u8 = 7;
 pub const STACKS_BLOCK_VERSION_AST_PRECHECK_SIZE: u8 = 1;
 
 pub const MAX_BLOCK_LEN: u32 = 2 * 1024 * 1024;

src/chainstate/stacks/tests/block_construction.rs

@@ -4708,7 +4708,6 @@ fn paramaterized_mempool_walk_test(
     let b_2 = make_block(&mut chainstate, ConsensusHash([0x2; 20]), &b_1, 2, 2);
 
     let mut mempool_settings = MemPoolWalkSettings::default();
-    mempool_settings.min_tx_fee = 10;
     let mut tx_events = Vec::new();
 
     let txs = codec_all_transactions(