From 9fa9d9fceb1108d378edba3231d16406e381431f Mon Sep 17 00:00:00 2001 From: bengtlofgren Date: Mon, 6 Nov 2023 14:18:02 +0100 Subject: [PATCH] new python script for building localnets with new genesis flow can now edit parameters.toml freely --- scripts/build_network.sh | 254 --------------------------- scripts/gen_localnet.py | 171 ++++++++++++++++++ scripts/utils/add_validator_shard.py | 28 --- scripts/utils/clean_config.py | 25 --- 4 files changed, 171 insertions(+), 307 deletions(-) delete mode 100755 scripts/build_network.sh create mode 100644 scripts/gen_localnet.py delete mode 100755 scripts/utils/add_validator_shard.py delete mode 100644 scripts/utils/clean_config.py diff --git a/scripts/build_network.sh b/scripts/build_network.sh deleted file mode 100755 index d5ead866c3..0000000000 --- a/scripts/build_network.sh +++ /dev/null @@ -1,254 +0,0 @@ -#!/bin/sh -# -# Script for initializing a Namada chain for local development and joining it. -# Note that this script trashes any existing local chain directories! -# -# ## Prerequisites -# - Python 3 -# - toml Python pip library (this is the -# `python3-toml` package on Ubuntu distributions) -# -# ## How to run -# This script should be run from the root of a Namada source code repo -# (https://github.com/anoma/namada). *.wasm files must already have been built -# and be present under the `wasm/` directory of the Namada repo. This can be -# achieved by running `make build wasm-scripts`. - -# The shell script takes two required arguments (and one optional argument): -# The first argument is the path to -# a network config toml compatible with the version of Namada being used. -# You can find example network config tomls in the `templates/` directory -# of the anoma-network-configs repo (https://github.com/heliaxdev/anoma-network-configs)` -# The second argument is the path to the directory containing the Namada binaries -# The third OPTIONAL argument is the BASE_DIR of the chain. This will depend on your setup. -# -# Example command: -# ```shell -# ./scripts/build_network.sh anoma-network-configs/templates/devnet-0.17.toml ./target/debug -# ```` -# -# Once the script is finished, it should be possible to straight away start -# running a ledger node e.g. the command assuming binaries have been built is: -# -# ```shell -# target/debug/namadan ledger run -# ```` - -# After running the ledger, you can run the following command to kill an underlying process -# ```shell -# pkill -f ".hack/chains" -# ``` -# and also delete the chain data by running -# ```shell -# rm -r .hack/chains -# ``` - -set +x -# set -eoux pipefail - -show_help() { - echo "Usage: script.sh , OPTIONAL:, OPTIONAL:" - echo "" - echo "Arguments:" - echo " namada_dir - The path to the directory containing the Namada binaries" - echo " base_dir - The path to the base directory (BASE_DIR), which is the directory where the chain's information will be stored. If the default base dir has not been changed, this does not need to be provided." - echo " genesis_file - The path to the genesis file. Otherwise, it will default to the genesis file in the genesis/e2e-tests-single-node.toml." - echo " The script should be smart enough to recognise if the second argument is a base dir or a genesis file, so order of last two arguments does not matter." - echo "" -} - -check_toml_file() { - toml_file="$NETWORK_CONFIG_PATH" # Get the file path from the first argument - section_prefix="validator.validator" - # Search for the section name in the TOML file - section_count=$(awk -F'[][]' -v prefix="$section_prefix" '/^\[.*\]$/ && $2 ~ "^" prefix { count++ } END { print count }' "$toml_file") - if [ ! "$(expr "$section_count" : '^[0-9]*$')" -eq 0 ]; then - echo "At least one validator ($section_count, in fact) has been found in the toml file. Please delete all occurrences of the section '[$section_prefix]' in the TOML file and try again." - exit 1 - fi -} - - - -check_wasm_files() { - wasm_files=$(find wasm -type f -name "*.wasm") - - count=$(echo "$wasm_files" | wc -l) - - if [ ! "$count" -ge 5 ]; then - echo "You must run make build-wasm-scripts in the namada directory before running this script." - exit 1 - fi -} -cleanup() { - # Kill the Python process - pkill -f ".hack/chains" - rm -r .hack/chains - rm -f local.*.tar.gz -} -validate_arguments() { - # The script expects 2 arguments: - # 1. The path to the directory containing the Namada binaries - # 2. The BASE_DIR of the chain - - if [ "$#" -gt 3 ] || [ "$#" -lt 1 ]; then - echo "Error: Invalid number of arguments. Expected 1 or 2 or 3 arguments." - echo "See the help page by running --help for more information." - exit 1 - fi - - # Get absolute path of where the script is being run from - CURRENT_DIR="$(pwd)" - # Get the absolute directory of where the script is located - SCRIPT_DIR="$(dirname "$0")" - if [ -d "$SCRIPT_DIR" ]; then - SCRIPT_DIR="$(readlink -f "$SCRIPT_DIR")" - else - echo "Error: Script directory not found." - exit 1 - fi - NETWORK_CONFIG_PATH="$SCRIPT_DIR/../genesis/e2e-tests-single-node.toml" - NAMADA_BIN_DIR="$1" - - if [ "$#" -eq 1 ]; then - BASE_DIR="$($NAMADA_BIN_DIR/namadac utils default-base-dir)" - echo "Using default BASE_DIR: $BASE_DIR" - elif [ "$#" -eq 2 ]; then - # Check if the second argument is a directory or a file - if [ -f "$2" ]; then - NETWORK_CONFIG_PATH="$(readlink -f "$CURRENT_DIR/$2")" - else - BASE_DIR="$2" - fi - else - if [ -f "$2" ]; then - NETWORK_CONFIG_PATH="$(readlink -f "$CURRENT_DIR/$2")" - BASE_DIR="$3" - else - BASE_DIR="$2" - NETWORK_CONFIG_PATH="$(readlink -f "$CURRENT_DIR/$3")" - fi - fi - - if [ ! -d "$CURRENT_DIR/wasm" ]; then - echo "Error: Directory 'wasm' does not exist in the current directory." - exit 1 - fi - - # The first argument should be a path to a network config toml - if [ ! -f "$NETWORK_CONFIG_PATH" ]; then - echo "Error: Invalid network config path. Expected a path to a network config toml, yet found no file in the location." - echo "Check so that there exists a file in the location: $NETWORK_CONFIG_PATH" - exit 1 - fi - file="$NETWORK_CONFIG_PATH" # Get the file path from the first argument - extension="${file##*.}" - if [ "$extension" != "toml" ]; then - echo "Error: The first argument provided is not a .toml file." - exit 1 - fi - - cp "$NETWORK_CONFIG_PATH" "$SCRIPT_DIR/utils/network-config.toml" - NETWORK_CONFIG_PATH="$SCRIPT_DIR/utils/network-config.toml" - - # Delete the section [validator.validator] from the network config toml - python3 $SCRIPT_DIR/utils/clean_config.py "$NETWORK_CONFIG_PATH" - - check_toml_file "$NETWORK_CONFIG_PATH" - - local directory="$NAMADA_BIN_DIR" - - if [ ! -d "$directory" ]; then - echo "Error: Invalid directory. The specified directory does not exist." - exit 1 - fi - - local namadac_path="$directory/namadac" - - if [ ! -x "$namadac_path" ]; then - echo "Error: Missing executable 'namadac' in the specified directory." - exit 1 - fi - - check_wasm_files -} - -package() { - - # Clean up any existing chain data - rm -rf "$BASE_DIR" || true - git checkout --ours -- wasm/checksums.json - rm -f nohup.out || true - - mkdir -p "$BASE_DIR" - - CHAIN_DIR='.hack/chains' - mkdir -p $CHAIN_DIR - - ALIAS='validator-local-dev' - - $NAMADA_BIN_DIR/namadac --base-dir "$BASE_DIR" utils init-genesis-validator \ - --alias $ALIAS \ - --net-address 127.0.0.1:26656 \ - --commission-rate 0.01 \ - --max-commission-rate-change 0.05 \ - --unsafe-dont-encrypt - - # get the directory of this script - SCRIPT_DIR="$(dirname $0)" - NAMADA_NETWORK_CONFIG_PATH="${CHAIN_DIR}/network-config-processed.toml" - python3 $SCRIPT_DIR/utils/add_validator_shard.py "$BASE_DIR"/pre-genesis/$ALIAS/validator.toml $NETWORK_CONFIG_PATH >$NAMADA_NETWORK_CONFIG_PATH - - python3 wasm/checksums.py - - NAMADA_CHAIN_PREFIX='local' - - $NAMADA_BIN_DIR/namadac --base-dir "$BASE_DIR" utils init-network \ - --chain-prefix "$NAMADA_CHAIN_PREFIX" \ - --genesis-path "$NAMADA_NETWORK_CONFIG_PATH" \ - --wasm-checksums-path wasm/checksums.json \ - --unsafe-dont-encrypt - - basename *.tar.gz .tar.gz >${CHAIN_DIR}/chain-id - NAMADA_CHAIN_ID="$(cat ${CHAIN_DIR}/chain-id)" - rm -rf "$SCRIPT_DIR/utils/network-config.toml" - cp "$BASE_DIR/${NAMADA_CHAIN_ID}/setup/other/wallet.toml" "$BASE_DIR/wallet-genesis.toml" - rm -rf "$BASE_DIR/${NAMADA_CHAIN_ID}" - mv "${NAMADA_CHAIN_ID}.tar.gz" $CHAIN_DIR - - - # clean up the http server when the script exits - trap cleanup EXIT - - export NAMADA_NETWORK_CONFIGS_SERVER='http://localhost:8123' - nohup bash -c "python3 -m http.server --directory ${CHAIN_DIR} 8123 &" && - sleep 2 && - $NAMADA_BIN_DIR/namadac --base-dir "$BASE_DIR" utils join-network \ - --genesis-validator "$ALIAS" \ - --chain-id "${NAMADA_CHAIN_ID}" \ - --dont-prefetch-wasm - - cp wasm/*.wasm "$BASE_DIR/${NAMADA_CHAIN_ID}/wasm/" - cp wasm/checksums.json "$BASE_DIR/${NAMADA_CHAIN_ID}/wasm/" - - tar -cvzf "${NAMADA_CHAIN_ID}.prebuilt.tar.gz" "$BASE_DIR" - mv "${NAMADA_CHAIN_ID}.prebuilt.tar.gz" $CHAIN_DIR - - git checkout --ours -- wasm/checksums.json - rm -rf nohup.out - - # don't delete namada - so we're ready to go with the chain - echo "Run the ledger! (and when done follow the instructions to clean up)" -} - -main() { - if expr "x$*x" : "x*--help*x" >/dev/null; then - show_help - return 0 - fi - - validate_arguments "$@" - package "$@" -} - -main "$@" diff --git a/scripts/gen_localnet.py b/scripts/gen_localnet.py new file mode 100644 index 0000000000..b5e99f1479 --- /dev/null +++ b/scripts/gen_localnet.py @@ -0,0 +1,171 @@ +import argparse +import sys +import os +import subprocess +import shutil +import toml + + +def move_genesis_wallet(genesis_wallet_toml : str, wallet_toml : str): + genesis_wallet = toml.load(genesis_wallet_toml) + wallet = toml.load(wallet_toml) + + for key in genesis_wallet.keys(): + value_dict = genesis_wallet[key] + if key in wallet.keys(): + wallet[key].update(value_dict) + toml.dump(wallet, open(wallet_toml, 'w')) + +def edit_parameters(params_toml, **kwargs): + # Make sure the kwargs are valid + params = toml.load(params_toml) + for k in kwargs.keys(): + if k not in params.keys(): + print(f"Invalid parameter {k}") + del kwargs[k] + else: + for key in kwargs[k].keys(): + if key not in params[k].keys(): + print(f"Invalid parameter {key} for {k}") + del kwargs[k][key] + else: + params[k][key] = kwargs[k][key] + + toml.dump(params, open(params_toml, 'w')) + + + +# Get the absolute path to the directory of the script +script_dir = sys.path[0] +# Get absolute path of parent directory +namada_dir = script_dir[:script_dir.rfind('/')] + +# Create the parser +parser = argparse.ArgumentParser(description='Builds a localnet for testing purposes') + +# Add the arguments +parser.add_argument('--localnet-dir', type=str, help='The localnet directory containing the genesis templates.') +parser.add_argument('-m', '--mode', type=str, help='The mode to run the localnet in. Can be release or debug, defaults to debug.') +parser.add_argument('--epoch-length', type=int, help='The epoch length in seconds, defaults to parameters.toml value.') +parser.add_argument('--max-validator-slots', type=int, help='The maximum number of validators, defaults to parameters.toml value.') +# Change any parameters in the parameters.toml file +parser.add_argument('--params', type=str, help='A string representation of a dictionary of parameters to update in the parameters.toml. Must be of the same format.') + + +# Parse the arguments +args = parser.parse_args() + +# Access the arguments +if args.localnet_dir: + localnet_dir = namada_dir + '/' + args.localnet_dir + + if os.path.isdir(localnet_dir) and os.listdir(localnet_dir): + print('Using localnet directory: ' + localnet_dir) + else: + print('Cannot find localnet directory that is not empty') + sys.exit(1) +else: + localnet_dir = namada_dir + '/genesis/localnet' + +if args.mode: + mode = args.mode +else: + mode = 'debug' + +if mode.lower() != 'release': + mode = 'debug' + +params = {} +if args.params: + params = eval(args.params) +if args.max_validator_slots: + params['pos_params'] = {'max_validator_slots': args.max_validator_slots} +if args.epoch_length: + epochs_per_year = 365 * 24 * 60 * 60 / args.epoch_length + params['parameters'] = {'epochs_per_year': epochs_per_year } +if len(params.keys())>0: + edit_parameters(localnet_dir + '/parameters.toml', **params) + +namada_bin_dir = namada_dir + '/target/' + mode + '/' + +# Check that namada_bin_dir exists and is not empty +if not os.path.isdir(namada_bin_dir) or not os.listdir(namada_bin_dir): + print('Cannot find namada binary directory that is not empty') + sys.exit(1) + +namada_bin = namada_bin_dir + 'namada' +namadac_bin = namada_bin_dir + 'namadac' +namadan_bin = namada_bin_dir + 'namadan' +namadaw_bin = namada_bin_dir + 'namadaw' + +bins = [namada_bin, namadac_bin, namadan_bin, namadaw_bin] +# Check that each binary exists and is executable +for bin in bins: + if not os.path.isfile(bin) or not os.access(bin, os.X_OK): + print(f"Cannot find the {bin.split('/')[-1]} binary or it is not executable") + sys.exit(1) + + + +print(f"Using {bins[0].split('/')[-1]} version: {os.popen(bin + ' --version').read()}") + +# Run namadac utils init_network with the correct arguments +print('Running namadac utils init_network') +CHAIN_PREFIX='local' +GENESIS_TIME='2021-12-31T00:00:00Z' +TEMPLATES_PATH=localnet_dir +WASM_CHECKSUMS_PATH=namada_dir + '/wasm/checksums.json' +WASM_PATH=namada_dir + '/wasm/' +BASE_DIR = subprocess.check_output([namadac_bin, "utils", "default-base-dir"]).decode().strip() + +# Delete the base dir +if os.path.isdir(BASE_DIR): + shutil.rmtree(BASE_DIR) +os.mkdir(BASE_DIR) + +# Check that wasm checksums file exists +if not os.path.isfile(WASM_CHECKSUMS_PATH): + print(f"Cannot find the wasm checksums file at {WASM_CHECKSUMS_PATH}") + sys.exit(1) + +# Check that wasm directory exists and is not empty +if not os.path.isdir(WASM_PATH) or not os.listdir(WASM_PATH): + print(f"Cannot find wasm directory that is not empty at {WASM_PATH}") + sys.exit(1) + +os.system(f"{namadac_bin} utils init-network --chain-prefix {CHAIN_PREFIX} --genesis-time {GENESIS_TIME} --templates-path {TEMPLATES_PATH} --wasm-checksums-path {WASM_CHECKSUMS_PATH}") + +base_dir_files = os.listdir(BASE_DIR) +CHAIN_ID="" +for file in base_dir_files: + if file.startswith(CHAIN_PREFIX): + CHAIN_ID = file + break + +# create a new directory within the base_dir +temp_dir = BASE_DIR + '/tmp/' +os.mkdir(temp_dir) +shutil.move(BASE_DIR + '/' + CHAIN_ID, BASE_DIR + '/tmp/' + CHAIN_ID) +shutil.move(namada_dir + '/' + CHAIN_ID + '.tar.gz', temp_dir + CHAIN_ID + '.tar.gz') + +GENESIS_VALIDATOR='validator-0' +PRE_GENESIS_PATH=localnet_dir + '/src/pre-genesis/' + GENESIS_VALIDATOR +if not os.path.isdir(PRE_GENESIS_PATH) or not os.listdir(PRE_GENESIS_PATH): + print(f"Cannot find pre-genesis directory that is not empty at {PRE_GENESIS_PATH}") + sys.exit(1) + +os.system(f"NAMADA_NETWORK_CONFIGS_DIR='{temp_dir}' {namadac_bin} utils join-network --chain-id {CHAIN_ID} --genesis-validator {GENESIS_VALIDATOR} --pre-genesis-path {PRE_GENESIS_PATH} --dont-prefetch-wasm") + +shutil.rmtree(BASE_DIR + '/' + CHAIN_ID + '/wasm/') +shutil.move(temp_dir + CHAIN_ID + '/wasm/', BASE_DIR + '/' + CHAIN_ID + '/wasm/') + +# Move the genesis wallet to the base dir +genesis_wallet_toml = localnet_dir + '/src/pre-genesis' + '/wallet.toml' +wallet = BASE_DIR + '/' + CHAIN_ID + '/wallet.toml' +move_genesis_wallet(genesis_wallet_toml, wallet) +# Delete the temp dir +shutil.rmtree(temp_dir) + +print("Run the ledger using the following command:") +print(f"{namada_bin} ledger run") + diff --git a/scripts/utils/add_validator_shard.py b/scripts/utils/add_validator_shard.py deleted file mode 100755 index beb108fece..0000000000 --- a/scripts/utils/add_validator_shard.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -import sys -import toml - -validator_config = toml.load(sys.argv[1]) - -# Delete any section containing "validator.validator" -for key in list(validator_config.keys()): - if 'validator.validator' in key: - del validator_config[key] - - -alias = next(iter(validator_config['validator'].items()))[0] - -validator_config['validator'][alias]['tokens'] = 6714000000 -validator_config['validator'][alias]['non_staked_balance'] = 1000000000000 -validator_config['validator'][alias]['validator_vp'] = 'vp_validator' -validator_config['validator'][alias]['staking_reward_vp'] = 'vp_validator' -validator_config['validator'][alias]['commission_rate'] = "0.05" -validator_config['validator'][alias]['max_commission_rate_change'] = "0.01" - -network_config = toml.load(sys.argv[2]) - -if not network_config.get("validator"): - network_config['validator'] = {} -network_config["validator"] |= validator_config["validator"] - -print(toml.dumps(network_config)) diff --git a/scripts/utils/clean_config.py b/scripts/utils/clean_config.py deleted file mode 100644 index 98b71fb1e1..0000000000 --- a/scripts/utils/clean_config.py +++ /dev/null @@ -1,25 +0,0 @@ - -import sys -import toml - -# Load the TOML file -with open(sys.argv[1], 'r') as file: - data = toml.load(file) - -# Function to remove sections and keys with "validator.validator" in their names -def remove_validator_sections_and_keys(data): - new_data = {} - for key, value in data.items(): - if isinstance(value, dict): - if not key.startswith('validator'): - new_data[key] = remove_validator_sections_and_keys(value) - elif not (key.startswith('validator.validator') or key.endswith('.public_key')): - new_data[key] = value - return new_data - -# Remove validator sections and keys -data = remove_validator_sections_and_keys(data) - -# Save the modified data back to the TOML file -with open(sys.argv[1], 'w') as file: - toml.dump(data, file)