From da03c1b834ba51f9baba3aff9efb464e19ec1a17 Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Sat, 8 Apr 2023 21:04:11 +0000 Subject: [PATCH 01/10] Add support for chain spec modifier commands --- docs/src/network-definition-spec.md | 2 + .../packages/orchestrator/src/chainSpec.ts | 73 +++++++++++++++++++ .../orchestrator/src/configGenerator.ts | 54 ++++++++++++++ .../packages/orchestrator/src/constants.ts | 7 ++ .../packages/orchestrator/src/network.ts | 2 + .../packages/orchestrator/src/orchestrator.ts | 6 ++ javascript/packages/orchestrator/src/paras.ts | 12 ++- javascript/packages/orchestrator/src/types.ts | 6 ++ javascript/packages/utils/src/types.ts | 2 + 9 files changed, 163 insertions(+), 1 deletion(-) diff --git a/docs/src/network-definition-spec.md b/docs/src/network-definition-spec.md index dcdcf9b60..1f2f3a3e3 100644 --- a/docs/src/network-definition-spec.md +++ b/docs/src/network-definition-spec.md @@ -32,6 +32,7 @@ The network config can be provided both in `json` or `toml` format and each sect - `local_path`: string; - `remote_name`: string; - `default_resources`: (Object) **Only** available in `kubernetes`, represent the resources `limits`/`reservations` needed by the nodes by default. +- `chain_spec_modifier_commands`: (Array of arrays of strings, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. In the arguments, `${chainSpec}`, `${plainSpec}`, and `${plainChainSpec}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `${rawChainSpec}` and `${rawSpec}` will be substituted for the raw chain spec path. - `random_nominators_count`: (number, optional), if is set _and the stacking pallet is enabled_ zombienet will generate `x` nominators and will be injected in the genesis. - `max_nominations`: (number, default 24), the max allowed number of nominations by a nominator. This should match the value set in the runtime (e.g Kusama is 24 and Polkadot 16). - `nodes`: @@ -72,6 +73,7 @@ The network config can be provided both in `json` or `toml` format and each sect - `*id`: (Number) The id to assign to this parachain. Must be unique. - `add_to_genesis`: (Boolean, default true) flag to add parachain to genesis or register in runtime. - `cumulus_based`: (Boolean, default true) flag to use `cumulus` command generation. + - `chain_spec_modifier_commands`: (Array of arrays of strings, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. In the arguments, `${chainSpec}`, `${plainSpec}`, and `${plainChainSpec}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `${rawChainSpec}` and `${rawSpec}` will be substituted for the raw chain spec path. - `genesis_wasm_path`: (String) Path to the wasm file to use. - `genesis_wasm_generator`: (String) Command to generate the wasm file. - `genesis_state_path`: (String) Path to the state file to use. diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index a6c64e0fb..70ada9d8e 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -6,8 +6,10 @@ import { getRandom, readDataFile, } from "@zombienet/utils"; +import { ChildProcessWithoutNullStreams, spawn } from "child_process"; import crypto from "crypto"; import fs from "fs"; +import { PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, RAW_CHAIN_SPEC_IN_CMD_PATTERN } from "./constants"; import { generateKeyFromSeed } from "./keys"; import { ChainSpec, ComputedNetwork, HrmpChannelsConfig, Node } from "./types"; const JSONbig = require("json-bigint")({ useNativeBigInt: true }); @@ -18,6 +20,15 @@ const JSONStream = require("JSONStream"); // track 1st staking as default; let stakingBond: number | undefined; +const processes: { [key: string]: ChildProcessWithoutNullStreams } = {}; + +// kill any runnning processes related to non-node chain spec processing +export async function destroyChainSpecProcesses() { + for (const key of Object.keys(processes)) { + processes[key].kill(); + } +} + export type KeyType = "session" | "aura" | "grandpa"; export type GenesisNodeKey = [string, string, { [key: string]: string }]; @@ -663,6 +674,63 @@ export async function getChainIdFromSpec(specPath: string): Promise { }); } +export async function runCommandWithChainSpec( + chainSpecFullPath: string, + commandArgs: string[], +) { + const chainSpecSubstitutePattern = new RegExp( + RAW_CHAIN_SPEC_IN_CMD_PATTERN.source + "|" + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.source, + "gi", + ); + + const substitutedCommandArgs = commandArgs.map(arg => + `${arg.replaceAll(chainSpecSubstitutePattern, chainSpecFullPath)}`); + const chainSpecModifiedPath = chainSpecFullPath.replace('.json', '-modified.json'); + + new CreateLogTable({ colWidths: [30, 90] }).pushToPrint([[ + decorators.green("๐Ÿงช Mutating chain spec"), + decorators.white(substitutedCommandArgs.join(" ")), + ]]); + + try { + await new Promise(function (resolve, reject) { + if (processes["mutator"]) { + processes["mutator"].kill(); + } + + // spawn the chain spec mutator thread with the command and arguments + processes["mutator"] = spawn(substitutedCommandArgs[0], substitutedCommandArgs.slice(1)); + // flush the modified spec to a different file and then copy it back into the original path + let spec = fs.createWriteStream(chainSpecModifiedPath); + + // `pipe` since it deals with flushing and we need to guarantee that the data is flushed + // before we resolve the promise. + processes["mutator"].stdout.pipe(spec); + + processes["mutator"].stderr.pipe(process.stderr); + + processes["mutator"].on("close", () => { + resolve(); + }); + + processes["mutator"].on("error", (err) => { + reject(err); + }); + }); + + // copy the modified file back into the original path after the mutation has completed + fs.copyFileSync(chainSpecModifiedPath, chainSpecFullPath); + } catch (e: any) { + if (e.code !== "ERR_FS_FILE_TOO_LARGE") throw e; + + console.log( + `\n\t\t ๐Ÿšง ${decorators.yellow( + `Chain Spec file ${chainSpecFullPath} is TOO LARGE to customize (more than 2G).`, + )} ๐Ÿšง`, + ); + } +} + export async function customizePlainRelayChain( specPath: string, networkSpec: ComputedNetwork, @@ -714,6 +782,11 @@ export async function customizePlainRelayChain( if (networkSpec.hrmp_channels) { await addHrmpChannelsToGenesis(specPath, networkSpec.hrmp_channels); } + + // modify the plain chain spec with any custom commands + for (const cmd of networkSpec.relaychain.chainSpecModifierCommands) { + await runCommandWithChainSpec(specPath, cmd); + } } catch (err) { console.log( `\n ${decorators.red("Unexpected error: ")} \t ${decorators.bright( diff --git a/javascript/packages/orchestrator/src/configGenerator.ts b/javascript/packages/orchestrator/src/configGenerator.ts index 9d497a37e..a849f1420 100644 --- a/javascript/packages/orchestrator/src/configGenerator.ts +++ b/javascript/packages/orchestrator/src/configGenerator.ts @@ -24,6 +24,8 @@ import { DEFAULT_WASM_GENERATE_SUBCOMMAND, GENESIS_STATE_FILENAME, GENESIS_WASM_FILENAME, + RAW_CHAIN_SPEC_IN_CMD_PATTERN, + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, UNDYING_COLLATOR_BIN, ZOMBIE_WRAPPER, } from "./constants"; @@ -118,6 +120,8 @@ export async function generateNetworkSpec( defaultImage: config.relaychain.default_image || DEFAULT_IMAGE, defaultCommand: config.relaychain.default_command || DEFAULT_COMMAND, defaultArgs: config.relaychain.default_args || [], + chainSpecModifierCommands: [], + rawChainSpecModifierCommands: [], randomNominatorsCount: config.relaychain?.random_nominators_count || 0, maxNominations: config.relaychain?.max_nominations || DEFAULT_MAX_NOMINATIONS, @@ -178,6 +182,30 @@ export async function generateNetworkSpec( ).replace("{{DEFAULT_COMMAND}}", networkSpec.relaychain.defaultCommand); } + for (const cmd of config.relaychain.chain_spec_modifier_commands || []) { + const cmdHasRawSpec = cmd.some(arg => RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); + const cmdHasPlainSpec = cmd.some(arg => PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); + + if (cmdHasRawSpec && cmdHasPlainSpec) { + console.error( + decorators.red( + `Chain spec modifier command references both raw and plain chain specs!\n\t${cmd}`, + ), + ); + process.exit(); + } + else if (cmdHasRawSpec) networkSpec.relaychain.rawChainSpecModifierCommands.push(cmd); + else if (cmdHasPlainSpec) networkSpec.relaychain.chainSpecModifierCommands.push(cmd); + else { + console.error( + decorators.red( + `Chain spec modifier command does not attempt to reference a chain spec path!\n\t${cmd}`, + ), + ); + process.exit(); + } + } + const relayChainBootnodes: string[] = []; for (const node of config.relaychain.nodes || []) { const nodeSetup = await getNodeFromConfig( @@ -357,6 +385,8 @@ export async function generateNetworkSpec( name: getUniqueName(parachain.id.toString()), para, cumulusBased: isCumulusBased, + chainSpecModifierCommands: [], + rawChainSpecModifierCommands: [], addToGenesis: parachain.add_to_genesis === undefined ? true @@ -389,6 +419,30 @@ export async function generateNetworkSpec( } } + for (const cmd of parachain.chain_spec_modifier_commands || []) { + const cmdHasRawSpec = cmd.some(arg => RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); + const cmdHasPlainSpec = cmd.some(arg => PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); + + if (cmdHasRawSpec && cmdHasPlainSpec) { + console.error( + decorators.red( + `Chain spec modifier command references both raw and plain chain specs!\n\t${cmd}`, + ), + ); + process.exit(); + } + else if (cmdHasRawSpec) parachainSetup.rawChainSpecModifierCommands.push(cmd); + else if (cmdHasPlainSpec) parachainSetup.chainSpecModifierCommands.push(cmd); + else { + console.error( + decorators.red( + `Chain spec modifier command does not attempt to reference a chain spec path!\n\t${cmd}`, + ), + ); + process.exit(); + } + } + parachainSetup = { ...parachainSetup, ...(parachain.balance ? { balance: parachain.balance } : {}), diff --git a/javascript/packages/orchestrator/src/constants.ts b/javascript/packages/orchestrator/src/constants.ts index 3a44c7d8d..e633c393c 100644 --- a/javascript/packages/orchestrator/src/constants.ts +++ b/javascript/packages/orchestrator/src/constants.ts @@ -44,6 +44,11 @@ const FINISH_MAGIC_FILE = "/tmp/finished.txt"; const GENESIS_STATE_FILENAME = "genesis-state"; const GENESIS_WASM_FILENAME = "genesis-wasm"; +// TODO: replace ${} with {{ZOMBIE:RAWCHAINSPEC}} etc. for consistent style +const mapNonEnvCmdInsertionPattern = (keywords: string[]) => new RegExp(keywords.map(x => `\\\${\\s*${x}\\s*}`).join("|"), "gi"); +const RAW_CHAIN_SPEC_IN_CMD_PATTERN = mapNonEnvCmdInsertionPattern(["rawChainSpec", "rawSpec"]); +const PLAIN_CHAIN_SPEC_IN_CMD_PATTERN = mapNonEnvCmdInsertionPattern(["chainSpec", "plainChainSpec", "plainSpec"]); + const TMP_DONE = "echo done > /tmp/zombie-tmp-done"; const TRANSFER_CONTAINER_WAIT_LOG = "waiting for tar to finish"; const NODE_CONTAINER_WAIT_LOG = "waiting for copy files to finish"; @@ -132,6 +137,8 @@ export { FINISH_MAGIC_FILE, GENESIS_STATE_FILENAME, GENESIS_WASM_FILENAME, + RAW_CHAIN_SPEC_IN_CMD_PATTERN, + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, TMP_DONE, TRANSFER_CONTAINER_WAIT_LOG, NODE_CONTAINER_WAIT_LOG, diff --git a/javascript/packages/orchestrator/src/network.ts b/javascript/packages/orchestrator/src/network.ts index dde2588d2..32fbfbc44 100644 --- a/javascript/packages/orchestrator/src/network.ts +++ b/javascript/packages/orchestrator/src/network.ts @@ -10,6 +10,7 @@ import { import { Metrics } from "./metrics"; import { NetworkNode } from "./networkNode"; import { Client } from "./providers/client"; +import { destroyChainSpecProcesses } from "./chainSpec"; const debug = require("debug")("zombie::network"); export interface NodeMapping { @@ -152,6 +153,7 @@ export class Network { // Cleanup all api instances for (const node of Object.values(this.nodesByName)) node.apiInstance?.disconnect(); + await destroyChainSpecProcesses(); await this.client.destroyNamespace(); } diff --git a/javascript/packages/orchestrator/src/orchestrator.ts b/javascript/packages/orchestrator/src/orchestrator.ts index 8d48aecde..b304e93c6 100644 --- a/javascript/packages/orchestrator/src/orchestrator.ts +++ b/javascript/packages/orchestrator/src/orchestrator.ts @@ -21,6 +21,7 @@ import { addParachainToGenesis, customizePlainRelayChain, readAndParseChainSpec, + runCommandWithChainSpec, } from "./chainSpec"; import { generateBootnodeSpec, @@ -331,6 +332,11 @@ export async function start( networkSpec.relaychain.nodes.unshift(bootnodeSpec); } + // modify the raw chain spec with any custom commands + for (const cmd of networkSpec.relaychain.rawChainSpecModifierCommands) { + await runCommandWithChainSpec(chainSpecFullPath, cmd); + } + const monitorIsAvailable = await client.isPodMonitorAvailable(); let jaegerUrl: string | undefined = undefined; if (networkSpec.settings.enable_tracing) { diff --git a/javascript/packages/orchestrator/src/paras.ts b/javascript/packages/orchestrator/src/paras.ts index 13c338d79..5d1cc724d 100644 --- a/javascript/packages/orchestrator/src/paras.ts +++ b/javascript/packages/orchestrator/src/paras.ts @@ -1,6 +1,6 @@ import { decorators, getRandomPort } from "@zombienet/utils"; import fs from "fs"; -import chainSpecFns, { isRawSpec } from "./chainSpec"; +import chainSpecFns, { isRawSpec, runCommandWithChainSpec } from "./chainSpec"; import { getUniqueName } from "./configGenerator"; import { DEFAULT_COLLATOR_IMAGE, @@ -139,6 +139,11 @@ export async function generateParachainFiles( if (parachain.genesis) await changeGenesisConfig(chainSpecFullPathPlain, parachain.genesis); + // modify the plain chain spec with any custom commands + for (const cmd of parachain.chainSpecModifierCommands) { + await runCommandWithChainSpec(chainSpecFullPathPlain, cmd); + } + debug("creating chain spec raw"); // ensure needed file if (parachain.chain) @@ -184,6 +189,11 @@ export async function generateParachainFiles( // add spec file to copy to all collators. parachain.specPath = chainSpecFullPath; + + // modify the raw chain spec with any custom commands + for (const cmd of parachain.rawChainSpecModifierCommands) { + await runCommandWithChainSpec(chainSpecFullPath, cmd); + } } // state and wasm files are only needed: diff --git a/javascript/packages/orchestrator/src/types.ts b/javascript/packages/orchestrator/src/types.ts index c772c963f..f652cfef7 100644 --- a/javascript/packages/orchestrator/src/types.ts +++ b/javascript/packages/orchestrator/src/types.ts @@ -49,6 +49,7 @@ export interface RelayChainConfig { chain_spec_command?: string; default_args?: string[]; default_overrides?: Override[]; + chain_spec_modifier_commands?: string[][]; random_nominators_count?: number; max_nominations?: number; nodes?: NodeConfig[]; @@ -105,6 +106,7 @@ export interface ParachainConfig { chain_spec_path?: string; cumulus_based?: boolean; bootnodes?: string[]; + chain_spec_modifier_commands?: string[][]; // backward compatibility collator?: NodeConfig; collators?: NodeConfig[]; @@ -130,6 +132,8 @@ export interface ComputedNetwork { chain: string; chainSpecPath?: string; chainSpecCommand?: string; + chainSpecModifierCommands: string[][]; + rawChainSpecModifierCommands: string[][]; randomNominatorsCount: number; maxNominations: number; nodes: Node[]; @@ -217,6 +221,8 @@ export interface Parachain { balance?: number; collators: Node[]; genesis?: JSON | ObjectJSON; + chainSpecModifierCommands: string[][]; + rawChainSpecModifierCommands: string[][]; } export interface envVars { diff --git a/javascript/packages/utils/src/types.ts b/javascript/packages/utils/src/types.ts index bc25093e8..b322dec9a 100644 --- a/javascript/packages/utils/src/types.ts +++ b/javascript/packages/utils/src/types.ts @@ -42,6 +42,7 @@ export interface RelayChainConfig { chain_spec_command?: string; default_args?: string[]; default_overrides?: Override[]; + chain_spec_modifier_commands?: string[][]; random_nominators_count?: number; max_nominations?: number; nodes?: NodeConfig[]; @@ -94,6 +95,7 @@ export interface ParachainConfig { chain_spec_path?: string; cumulus_based?: boolean; bootnodes?: string[]; + chain_spec_modifier_commands?: string[][]; // backward compatibility collator?: NodeConfig; collators?: NodeConfig[]; From 9a67e5ef35409c7befd52f2afc53a4ad347335d6 Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 12 Apr 2023 07:36:57 +0000 Subject: [PATCH 02/10] Address comments on chain spec modifiers --- docs/src/network-definition-spec.md | 4 +- .../packages/orchestrator/src/chainSpec.ts | 38 +++++++---- .../orchestrator/src/configGenerator.ts | 66 +++++++++++-------- .../packages/orchestrator/src/constants.ts | 6 +- .../packages/orchestrator/src/network.ts | 5 +- javascript/packages/orchestrator/src/types.ts | 14 ++-- javascript/packages/utils/src/fs.ts | 4 ++ javascript/packages/utils/src/types.ts | 6 +- 8 files changed, 87 insertions(+), 56 deletions(-) diff --git a/docs/src/network-definition-spec.md b/docs/src/network-definition-spec.md index 1f2f3a3e3..d116729a7 100644 --- a/docs/src/network-definition-spec.md +++ b/docs/src/network-definition-spec.md @@ -32,7 +32,7 @@ The network config can be provided both in `json` or `toml` format and each sect - `local_path`: string; - `remote_name`: string; - `default_resources`: (Object) **Only** available in `kubernetes`, represent the resources `limits`/`reservations` needed by the nodes by default. -- `chain_spec_modifier_commands`: (Array of arrays of strings, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. In the arguments, `${chainSpec}`, `${plainSpec}`, and `${plainChainSpec}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `${rawChainSpec}` and `${rawSpec}` will be substituted for the raw chain spec path. +- `chain_spec_modifier_commands`: (Array of `commands`, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. `Commands` are themselves arrays of strings (each argument is a string). In the arguments, `{{'plain'|chainSpec}}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `{{'raw'|chainSpec}}` will be substituted for the raw chain spec path. - `random_nominators_count`: (number, optional), if is set _and the stacking pallet is enabled_ zombienet will generate `x` nominators and will be injected in the genesis. - `max_nominations`: (number, default 24), the max allowed number of nominations by a nominator. This should match the value set in the runtime (e.g Kusama is 24 and Polkadot 16). - `nodes`: @@ -73,7 +73,7 @@ The network config can be provided both in `json` or `toml` format and each sect - `*id`: (Number) The id to assign to this parachain. Must be unique. - `add_to_genesis`: (Boolean, default true) flag to add parachain to genesis or register in runtime. - `cumulus_based`: (Boolean, default true) flag to use `cumulus` command generation. - - `chain_spec_modifier_commands`: (Array of arrays of strings, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. In the arguments, `${chainSpec}`, `${plainSpec}`, and `${plainChainSpec}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `${rawChainSpec}` and `${rawSpec}` will be substituted for the raw chain spec path. + - `chain_spec_modifier_commands`: (Array of `commands`, optional) Set of commands and their arguments to modify the resulting chain spec before the chain is launched with it. `Commands` are themselves arrays of strings (each argument is a string). In the arguments, `{{'plain'|chainSpec}}` will be substituted for the plain spec path and run before the raw chain spec is generated, and `{{'raw'|chainSpec}}` will be substituted for the raw chain spec path. - `genesis_wasm_path`: (String) Path to the wasm file to use. - `genesis_wasm_generator`: (String) Command to generate the wasm file. - `genesis_state_path`: (String) Path to the state file to use. diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index 70ada9d8e..3b2286251 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -9,7 +9,10 @@ import { import { ChildProcessWithoutNullStreams, spawn } from "child_process"; import crypto from "crypto"; import fs from "fs"; -import { PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, RAW_CHAIN_SPEC_IN_CMD_PATTERN } from "./constants"; +import { + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, + RAW_CHAIN_SPEC_IN_CMD_PATTERN, +} from "./constants"; import { generateKeyFromSeed } from "./keys"; import { ChainSpec, ComputedNetwork, HrmpChannelsConfig, Node } from "./types"; const JSONbig = require("json-bigint")({ useNativeBigInt: true }); @@ -25,8 +28,8 @@ const processes: { [key: string]: ChildProcessWithoutNullStreams } = {}; // kill any runnning processes related to non-node chain spec processing export async function destroyChainSpecProcesses() { for (const key of Object.keys(processes)) { - processes[key].kill(); - } + processes[key].kill(); + } } export type KeyType = "session" | "aura" | "grandpa"; @@ -679,18 +682,26 @@ export async function runCommandWithChainSpec( commandArgs: string[], ) { const chainSpecSubstitutePattern = new RegExp( - RAW_CHAIN_SPEC_IN_CMD_PATTERN.source + "|" + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.source, + RAW_CHAIN_SPEC_IN_CMD_PATTERN?.source + + "|" + + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN?.source, "gi", ); - const substitutedCommandArgs = commandArgs.map(arg => - `${arg.replaceAll(chainSpecSubstitutePattern, chainSpecFullPath)}`); - const chainSpecModifiedPath = chainSpecFullPath.replace('.json', '-modified.json'); + const substitutedCommandArgs = commandArgs.map( + (arg) => `${arg.replaceAll(chainSpecSubstitutePattern, chainSpecFullPath)}`, + ); + const chainSpecModifiedPath = chainSpecFullPath.replace( + ".json", + "-modified.json", + ); - new CreateLogTable({ colWidths: [30, 90] }).pushToPrint([[ - decorators.green("๐Ÿงช Mutating chain spec"), - decorators.white(substitutedCommandArgs.join(" ")), - ]]); + new CreateLogTable({ colWidths: [30, 90] }).pushToPrint([ + [ + decorators.green("๐Ÿงช Mutating chain spec"), + decorators.white(substitutedCommandArgs.join(" ")), + ], + ]); try { await new Promise(function (resolve, reject) { @@ -699,7 +710,10 @@ export async function runCommandWithChainSpec( } // spawn the chain spec mutator thread with the command and arguments - processes["mutator"] = spawn(substitutedCommandArgs[0], substitutedCommandArgs.slice(1)); + processes["mutator"] = spawn( + substitutedCommandArgs[0], + substitutedCommandArgs.slice(1), + ); // flush the modified spec to a different file and then copy it back into the original path let spec = fs.createWriteStream(chainSpecModifiedPath); diff --git a/javascript/packages/orchestrator/src/configGenerator.ts b/javascript/packages/orchestrator/src/configGenerator.ts index a849f1420..b7d22e7c2 100644 --- a/javascript/packages/orchestrator/src/configGenerator.ts +++ b/javascript/packages/orchestrator/src/configGenerator.ts @@ -24,8 +24,8 @@ import { DEFAULT_WASM_GENERATE_SUBCOMMAND, GENESIS_STATE_FILENAME, GENESIS_WASM_FILENAME, - RAW_CHAIN_SPEC_IN_CMD_PATTERN, PLAIN_CHAIN_SPEC_IN_CMD_PATTERN, + RAW_CHAIN_SPEC_IN_CMD_PATTERN, UNDYING_COLLATOR_BIN, ZOMBIE_WRAPPER, } from "./constants"; @@ -183,26 +183,31 @@ export async function generateNetworkSpec( } for (const cmd of config.relaychain.chain_spec_modifier_commands || []) { - const cmdHasRawSpec = cmd.some(arg => RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); - const cmdHasPlainSpec = cmd.some(arg => PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); + const cmdHasRawSpec = cmd.some((arg) => + RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg), + ); + const cmdHasPlainSpec = cmd.some((arg) => + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg), + ); if (cmdHasRawSpec && cmdHasPlainSpec) { - console.error( - decorators.red( - `Chain spec modifier command references both raw and plain chain specs!\n\t${cmd}`, + console.log( + decorators.yellow( + `Chain spec modifier command references both raw and plain chain specs! Only the raw chain spec will be modified.\n\t${cmd}`, ), ); - process.exit(); } - else if (cmdHasRawSpec) networkSpec.relaychain.rawChainSpecModifierCommands.push(cmd); - else if (cmdHasPlainSpec) networkSpec.relaychain.chainSpecModifierCommands.push(cmd); - else { - console.error( - decorators.red( - `Chain spec modifier command does not attempt to reference a chain spec path!\n\t${cmd}`, + + if (cmdHasRawSpec) { + networkSpec.relaychain.rawChainSpecModifierCommands.push(cmd); + } else if (cmdHasPlainSpec) { + networkSpec.relaychain.chainSpecModifierCommands.push(cmd); + } else { + console.log( + decorators.yellow( + `Chain spec modifier command does not attempt to reference a chain spec path! It will not be executed.\n\t${cmd}`, ), ); - process.exit(); } } @@ -420,26 +425,31 @@ export async function generateNetworkSpec( } for (const cmd of parachain.chain_spec_modifier_commands || []) { - const cmdHasRawSpec = cmd.some(arg => RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); - const cmdHasPlainSpec = cmd.some(arg => PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg)); - + const cmdHasRawSpec = cmd.some((arg) => + RAW_CHAIN_SPEC_IN_CMD_PATTERN.test(arg), + ); + const cmdHasPlainSpec = cmd.some((arg) => + PLAIN_CHAIN_SPEC_IN_CMD_PATTERN.test(arg), + ); + if (cmdHasRawSpec && cmdHasPlainSpec) { - console.error( - decorators.red( - `Chain spec modifier command references both raw and plain chain specs!\n\t${cmd}`, + console.log( + decorators.yellow( + `Chain spec modifier command references both raw and plain chain specs! Only the raw chain spec will be modified.\n\t${cmd}`, ), ); - process.exit(); } - else if (cmdHasRawSpec) parachainSetup.rawChainSpecModifierCommands.push(cmd); - else if (cmdHasPlainSpec) parachainSetup.chainSpecModifierCommands.push(cmd); - else { - console.error( - decorators.red( - `Chain spec modifier command does not attempt to reference a chain spec path!\n\t${cmd}`, + + if (cmdHasRawSpec) { + parachainSetup.rawChainSpecModifierCommands.push(cmd); + } else if (cmdHasPlainSpec) { + parachainSetup.chainSpecModifierCommands.push(cmd); + } else { + console.log( + decorators.yellow( + `Chain spec modifier command does not attempt to reference a chain spec path! It will not be executed.\n\t${cmd}`, ), ); - process.exit(); } } diff --git a/javascript/packages/orchestrator/src/constants.ts b/javascript/packages/orchestrator/src/constants.ts index e633c393c..e3c760dda 100644 --- a/javascript/packages/orchestrator/src/constants.ts +++ b/javascript/packages/orchestrator/src/constants.ts @@ -44,10 +44,8 @@ const FINISH_MAGIC_FILE = "/tmp/finished.txt"; const GENESIS_STATE_FILENAME = "genesis-state"; const GENESIS_WASM_FILENAME = "genesis-wasm"; -// TODO: replace ${} with {{ZOMBIE:RAWCHAINSPEC}} etc. for consistent style -const mapNonEnvCmdInsertionPattern = (keywords: string[]) => new RegExp(keywords.map(x => `\\\${\\s*${x}\\s*}`).join("|"), "gi"); -const RAW_CHAIN_SPEC_IN_CMD_PATTERN = mapNonEnvCmdInsertionPattern(["rawChainSpec", "rawSpec"]); -const PLAIN_CHAIN_SPEC_IN_CMD_PATTERN = mapNonEnvCmdInsertionPattern(["chainSpec", "plainChainSpec", "plainSpec"]); +const RAW_CHAIN_SPEC_IN_CMD_PATTERN = new RegExp(/{{CHAIN_SPEC:RAW}}/gi); +const PLAIN_CHAIN_SPEC_IN_CMD_PATTERN = new RegExp(/{{CHAIN_SPEC:PLAIN}}/gi); const TMP_DONE = "echo done > /tmp/zombie-tmp-done"; const TRANSFER_CONTAINER_WAIT_LOG = "waiting for tar to finish"; diff --git a/javascript/packages/orchestrator/src/network.ts b/javascript/packages/orchestrator/src/network.ts index 32fbfbc44..9f86160fd 100644 --- a/javascript/packages/orchestrator/src/network.ts +++ b/javascript/packages/orchestrator/src/network.ts @@ -1,6 +1,7 @@ import { CreateLogTable, decorators } from "@zombienet/utils"; import axios from "axios"; import fs from "fs"; +import { destroyChainSpecProcesses } from "./chainSpec"; import { BAKCCHANNEL_POD_NAME, BAKCCHANNEL_PORT, @@ -10,7 +11,6 @@ import { import { Metrics } from "./metrics"; import { NetworkNode } from "./networkNode"; import { Client } from "./providers/client"; -import { destroyChainSpecProcesses } from "./chainSpec"; const debug = require("debug")("zombie::network"); export interface NodeMapping { @@ -151,8 +151,9 @@ export class Network { async stop() { // Cleanup all api instances - for (const node of Object.values(this.nodesByName)) + for (const node of Object.values(this.nodesByName)) { node.apiInstance?.disconnect(); + } await destroyChainSpecProcesses(); await this.client.destroyNamespace(); } diff --git a/javascript/packages/orchestrator/src/types.ts b/javascript/packages/orchestrator/src/types.ts index f652cfef7..b04bc71f9 100644 --- a/javascript/packages/orchestrator/src/types.ts +++ b/javascript/packages/orchestrator/src/types.ts @@ -49,7 +49,7 @@ export interface RelayChainConfig { chain_spec_command?: string; default_args?: string[]; default_overrides?: Override[]; - chain_spec_modifier_commands?: string[][]; + chain_spec_modifier_commands?: Command[]; random_nominators_count?: number; max_nominations?: number; nodes?: NodeConfig[]; @@ -106,7 +106,7 @@ export interface ParachainConfig { chain_spec_path?: string; cumulus_based?: boolean; bootnodes?: string[]; - chain_spec_modifier_commands?: string[][]; + chain_spec_modifier_commands?: Command[]; // backward compatibility collator?: NodeConfig; collators?: NodeConfig[]; @@ -132,8 +132,8 @@ export interface ComputedNetwork { chain: string; chainSpecPath?: string; chainSpecCommand?: string; - chainSpecModifierCommands: string[][]; - rawChainSpecModifierCommands: string[][]; + chainSpecModifierCommands: Command[]; + rawChainSpecModifierCommands: Command[]; randomNominatorsCount: number; maxNominations: number; nodes: Node[]; @@ -221,8 +221,8 @@ export interface Parachain { balance?: number; collators: Node[]; genesis?: JSON | ObjectJSON; - chainSpecModifierCommands: string[][]; - rawChainSpecModifierCommands: string[][]; + chainSpecModifierCommands: Command[]; + rawChainSpecModifierCommands: Command[]; } export interface envVars { @@ -230,6 +230,8 @@ export interface envVars { value: string; } +export type Command = string[]; + export interface ChainSpec { name: string; id: string; diff --git a/javascript/packages/utils/src/fs.ts b/javascript/packages/utils/src/fs.ts index e0128f550..3a6d14276 100644 --- a/javascript/packages/utils/src/fs.ts +++ b/javascript/packages/utils/src/fs.ts @@ -163,6 +163,10 @@ export function readNetworkConfig(filepath: string): LaunchConfig { return `{{ZOMBIE:${nodeName}:${key}}}`; }); + env.addFilter("chainSpec", function (chainSpecType: string) { + return `{{CHAIN_SPEC:${chainSpecType.toUpperCase()}}}`; + }); + const temmplateContent = fs.readFileSync(filepath).toString(); const content = env.renderString(temmplateContent, process.env); diff --git a/javascript/packages/utils/src/types.ts b/javascript/packages/utils/src/types.ts index b322dec9a..791061d8f 100644 --- a/javascript/packages/utils/src/types.ts +++ b/javascript/packages/utils/src/types.ts @@ -42,7 +42,7 @@ export interface RelayChainConfig { chain_spec_command?: string; default_args?: string[]; default_overrides?: Override[]; - chain_spec_modifier_commands?: string[][]; + chain_spec_modifier_commands?: Command[]; random_nominators_count?: number; max_nominations?: number; nodes?: NodeConfig[]; @@ -95,7 +95,7 @@ export interface ParachainConfig { chain_spec_path?: string; cumulus_based?: boolean; bootnodes?: string[]; - chain_spec_modifier_commands?: string[][]; + chain_spec_modifier_commands?: Command[]; // backward compatibility collator?: NodeConfig; collators?: NodeConfig[]; @@ -115,6 +115,8 @@ export interface envVars { value: string; } +export type Command = string[]; + // Utils export interface GlobalVolume { name: string; From a4addae6a2e8916ff4afa3e940a36c562ddd54c7 Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 19 Apr 2023 07:53:07 +0000 Subject: [PATCH 03/10] fix: chain spec mutator working file directory + fatal error handling --- .../packages/orchestrator/src/chainSpec.ts | 24 ++++++++++++++----- .../packages/orchestrator/src/orchestrator.ts | 7 +++++- javascript/packages/orchestrator/src/paras.ts | 9 +++++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index 3b2286251..c5d8d7200 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -14,7 +14,13 @@ import { RAW_CHAIN_SPEC_IN_CMD_PATTERN, } from "./constants"; import { generateKeyFromSeed } from "./keys"; -import { ChainSpec, ComputedNetwork, HrmpChannelsConfig, Node } from "./types"; +import { + ChainSpec, + Command, + ComputedNetwork, + HrmpChannelsConfig, + Node, +} from "./types"; const JSONbig = require("json-bigint")({ useNativeBigInt: true }); const debug = require("debug")("zombie::chain-spec"); @@ -679,7 +685,8 @@ export async function getChainIdFromSpec(specPath: string): Promise { export async function runCommandWithChainSpec( chainSpecFullPath: string, - commandArgs: string[], + commandWithArgs: Command, + workingDirectory: string | URL, ) { const chainSpecSubstitutePattern = new RegExp( RAW_CHAIN_SPEC_IN_CMD_PATTERN?.source + @@ -688,7 +695,7 @@ export async function runCommandWithChainSpec( "gi", ); - const substitutedCommandArgs = commandArgs.map( + const substitutedCommandArgs = commandWithArgs.map( (arg) => `${arg.replaceAll(chainSpecSubstitutePattern, chainSpecFullPath)}`, ); const chainSpecModifiedPath = chainSpecFullPath.replace( @@ -713,6 +720,7 @@ export async function runCommandWithChainSpec( processes["mutator"] = spawn( substitutedCommandArgs[0], substitutedCommandArgs.slice(1), + { cwd: workingDirectory }, ); // flush the modified spec to a different file and then copy it back into the original path let spec = fs.createWriteStream(chainSpecModifiedPath); @@ -723,8 +731,12 @@ export async function runCommandWithChainSpec( processes["mutator"].stderr.pipe(process.stderr); - processes["mutator"].on("close", () => { - resolve(); + processes["mutator"].on("close", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Process returned error code ${code}!`)); + } }); processes["mutator"].on("error", (err) => { @@ -799,7 +811,7 @@ export async function customizePlainRelayChain( // modify the plain chain spec with any custom commands for (const cmd of networkSpec.relaychain.chainSpecModifierCommands) { - await runCommandWithChainSpec(specPath, cmd); + await runCommandWithChainSpec(specPath, cmd, networkSpec.configBasePath); } } catch (err) { console.log( diff --git a/javascript/packages/orchestrator/src/orchestrator.ts b/javascript/packages/orchestrator/src/orchestrator.ts index b304e93c6..849451b08 100644 --- a/javascript/packages/orchestrator/src/orchestrator.ts +++ b/javascript/packages/orchestrator/src/orchestrator.ts @@ -239,6 +239,7 @@ export async function start( namespace, tmpDir.path, parachainFilesPath, + networkSpec.configBasePath, chainName, parachain, relayChainSpecIsRaw, @@ -334,7 +335,11 @@ export async function start( // modify the raw chain spec with any custom commands for (const cmd of networkSpec.relaychain.rawChainSpecModifierCommands) { - await runCommandWithChainSpec(chainSpecFullPath, cmd); + await runCommandWithChainSpec( + chainSpecFullPath, + cmd, + networkSpec.configBasePath, + ); } const monitorIsAvailable = await client.isPodMonitorAvailable(); diff --git a/javascript/packages/orchestrator/src/paras.ts b/javascript/packages/orchestrator/src/paras.ts index 5d1cc724d..975bb0efd 100644 --- a/javascript/packages/orchestrator/src/paras.ts +++ b/javascript/packages/orchestrator/src/paras.ts @@ -20,6 +20,7 @@ export async function generateParachainFiles( namespace: string, tmpDir: string, parachainFilesPath: string, + configBasePath: string | URL, relayChainName: string, parachain: Parachain, relayChainSpecIsRaw: boolean, @@ -141,7 +142,11 @@ export async function generateParachainFiles( // modify the plain chain spec with any custom commands for (const cmd of parachain.chainSpecModifierCommands) { - await runCommandWithChainSpec(chainSpecFullPathPlain, cmd); + await runCommandWithChainSpec( + chainSpecFullPathPlain, + cmd, + configBasePath, + ); } debug("creating chain spec raw"); @@ -192,7 +197,7 @@ export async function generateParachainFiles( // modify the raw chain spec with any custom commands for (const cmd of parachain.rawChainSpecModifierCommands) { - await runCommandWithChainSpec(chainSpecFullPath, cmd); + await runCommandWithChainSpec(chainSpecFullPath, cmd, configBasePath); } } From f504bf11f26f390a9efc2f1091bce1e1ce5a651a Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 19 Apr 2023 07:54:11 +0000 Subject: [PATCH 04/10] feat(chain spec mutator): README + examplle --- README.md | 21 ++++++++ ...0005-chain-spec-mutation-with-chainql.toml | 31 +++++++++++ examples/chainqlCopyBalances.jsonnet | 53 +++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 examples/0005-chain-spec-mutation-with-chainql.toml create mode 100644 examples/chainqlCopyBalances.jsonnet diff --git a/README.md b/README.md index e370fbef7..a383da926 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,27 @@ export COL_IMAGE=docker.io/paritypr/colander:master ./zombienet-macos spawn examples/0001-small-network.toml ``` +#### Custom commands to modify the resulting chain-specs + +If you want to customize the chain specification, plain or raw, that one of your chains will be launched with, beyond +what Zombienet provides by default, you can do so with `chain_spec_modifier_commands`. + +The `chain_spec_modifier_commands` option allows you to specify a list of CLI commands that will use, modify and return +the modified chain-spec before it is used to launch the network. These commands can modify the chain specification in any way you need. + +```toml +... +[[parachains]] +id = 100 +chain_spec_modifier_commands = [[ + "/path/to/your_custom_script.sh", + "{{'plain'|chainSpec}}" +]] +... +``` + +For a more in-depth example with a chain-querying tool `chainql`, you can visit the [examples](examples) directory. + ##### Teardown You can teardown the network (and cleanup the used resources) by terminating the process (`ctrl+c`). diff --git a/examples/0005-chain-spec-mutation-with-chainql.toml b/examples/0005-chain-spec-mutation-with-chainql.toml new file mode 100644 index 000000000..51fe06c82 --- /dev/null +++ b/examples/0005-chain-spec-mutation-with-chainql.toml @@ -0,0 +1,31 @@ +# examples/0005-chain-spec-mutation-with-chainql.toml +[relaychain] +default_image = "docker.io/parity/polkadot:latest" +default_command = "polkadot" +default_args = [ "-lparachain=debug" ] + +chain = "rococo-local" + + [[relaychain.nodes]] + name = "alice" + validator = true + + [[relaychain.nodes]] + name = "bob" + validator = true + +[[parachains]] +id = 100 +# make sure to have chainql installed (`cargo install chainql`) +# https://github.com/UniqueNetwork/chainql +chain_spec_modifier_commands = [[ + "chainql", + "--tla-code=rawSpec=import '{{'raw'|chainSpec}}'", + "--tla-str=pullFrom=wss://rococo-rockmine-rpc.polkadot.io:443", + "--trace-format=explaining", + "chainqlCopyBalances.jsonnet", +]] + + [[parachains.collator_groups]] + count = 2 + name = "collator" diff --git a/examples/chainqlCopyBalances.jsonnet b/examples/chainqlCopyBalances.jsonnet new file mode 100644 index 000000000..55b005735 --- /dev/null +++ b/examples/chainqlCopyBalances.jsonnet @@ -0,0 +1,53 @@ +// Get a live chain's system balances on the URL provided with `pullFrom`, and +// insert them into a raw spec to launch a new chain with. +// +// ### Arguments +// - `rawSpec`: Path to the raw chain spec to modify +// - `pullFrom`: URL of the chain's WS port to get the data from +// +// ### Usage +// `chainql --tla-code=rawSpec="import '/path/to/parachain-spec-raw.json'" --tla-str=pullFrom="wss://some-parachain.node:443" chainqlCopyBalances.jsonnet` +// +// Make sure to to have `chainql` installed: `cargo install chainql` + +function(rawSpec, pullFrom) +// get the latest state of the blockchain +local sourceChainState = cql.chain(pullFrom).latest; + +local + // store all keys under the `Account` storage of the `System` pallet + accounts = sourceChainState.System.Account._preloadKeys, + // get the encoded naming of `pallet_balances::TotalIssuance` for future use + totalIssuanceKey = sourceChainState.Balances._encodeKey.TotalIssuance([]), +; + +// output the raw spec with the following changes +rawSpec { + genesis+: { + raw+: { + // add the following entries to the `top` section + top+: + { + // encode key and value of every account under `system.account` and add them to the chain spec + [sourceChainState.System._encodeKey.Account([key])]: + sourceChainState.System._encodeValue.Account(accounts[key]) + for key in std.objectFields(accounts) + } + { + // add to the local, already-existing total issuance the issuance of all incoming accounts. + // NOTE: we do not take into consideration for total issuance's funds potential overlap with the testnet's present accounts. + [totalIssuanceKey]: sourceChainState.Balances._encodeValue.TotalIssuance( + // decode the chain-spec's already existing totalIssuance + sourceChainState.Balances._decodeValue.TotalIssuance(super[totalIssuanceKey]) + // iterate over and sum up the total issuance of the incoming accounts + + std.foldl( + function(issuance, acc) + issuance + acc.data.free + acc.data.reserved + , + std.objectValues(accounts), + std.bigint('0'), + ) + ) + }, + }, + }, +} \ No newline at end of file From 9f51226de8de0bc6014e9ccc95fbf8687de6aedd Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 19 Apr 2023 07:55:17 +0000 Subject: [PATCH 05/10] test(chain spec mutator): test components + add to CI --- .gitlab-ci.yml | 31 ++++++++++++++++++++++++ tests/0014-chain-spec-modification.toml | 18 ++++++++++++++ tests/0014-chain-spec-modification.zndsl | 10 ++++++++ tests/change-name-in-chain-spec.sh | 11 +++++++++ tests/check-testnet-name.js | 12 +++++++++ 5 files changed, 82 insertions(+) create mode 100644 tests/0014-chain-spec-modification.toml create mode 100644 tests/0014-chain-spec-modification.zndsl create mode 100755 tests/change-name-in-chain-spec.sh create mode 100644 tests/check-testnet-name.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03320931b..67a076efc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -459,3 +459,34 @@ db-snapshot: retry: 2 tags: - zombienet-polkadot-integration-test + +zombienet-chain-spec-custom-modification: + stage: deploy + <<: *kubernetes-env + image: "paritypr/zombienet:${CI_COMMIT_SHORT_SHA}" + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + # needs: + # - job: publish-docker-pr + + variables: + GH_DIR: "https://github.com/paritytech/zombienet/tree/${CI_COMMIT_SHORT_SHA}/tests" + + before_script: + - echo "Zombienet Tests Custom Chain Spec Modification" + - echo "paritypr/zombienet:${CI_COMMIT_SHORT_SHA}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/paritypr/polkadot-debug:master" + - export COL_IMAGE="docker.io/paritypr/colander:master" + + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --test="0014-chain-spec-modification.zndsl" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test diff --git a/tests/0014-chain-spec-modification.toml b/tests/0014-chain-spec-modification.toml new file mode 100644 index 000000000..1052dc61c --- /dev/null +++ b/tests/0014-chain-spec-modification.toml @@ -0,0 +1,18 @@ +[relaychain] +default_image = "docker.io/parity/polkadot:latest" +default_command = "polkadot" +default_args = [ "-lparachain=debug" ] +chain = "rococo-local" + +chain_spec_modifier_commands = [[ + "./change-name-in-chain-spec.sh", + "{{'plain'|chainSpec}}", +]] + + [[relaychain.nodes]] + name = "alice" + validator = true + + [[relaychain.nodes]] + name = "bob" + validator = true diff --git a/tests/0014-chain-spec-modification.zndsl b/tests/0014-chain-spec-modification.zndsl new file mode 100644 index 000000000..a2362f0f1 --- /dev/null +++ b/tests/0014-chain-spec-modification.zndsl @@ -0,0 +1,10 @@ +Description: Chain Spec Modification +Network: ./0014-chain-spec-modification.toml +Creds: config + + +alice: is up +bob: is up +alice: log line matches "Imported #[0-9]+" within 10 seconds + +alice: js-script ./check-testnet-name.js with "Tentset Lacol Ococor" return is true diff --git a/tests/change-name-in-chain-spec.sh b/tests/change-name-in-chain-spec.sh new file mode 100755 index 000000000..eb1364605 --- /dev/null +++ b/tests/change-name-in-chain-spec.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# The first argument sent to the script must be the path of the chain spec +chain_spec_path = $1 +# Assume that the name +old_name="Rococo Local Testnet" +# Set the new name for the network +new_name="Tentset Lacol Ococor" + +# Replace the name field in the JSON file +sed -i "s/\"name\": \"$old_name\"/\"name\": \"$new_name\"/" $chain_spec_path diff --git a/tests/check-testnet-name.js b/tests/check-testnet-name.js new file mode 100644 index 000000000..6a845cd25 --- /dev/null +++ b/tests/check-testnet-name.js @@ -0,0 +1,12 @@ +// Check if the testnet's name is the same as the expected name from the args +async function run(nodeName, networkInfo, args) { + const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName]; + const api = await zombie.connect(wsUri, userDefinedTypes); + + const expectedName = args[0] ?? "Rococo Local Testnet"; + const testnetName = await api.rpc.system.chain(); + + return testnetName == expectedName; +} + +module.exports = { run } \ No newline at end of file From 52cfef43abb669bc8be8dbc76abd7cc9159186ec Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 19 Apr 2023 08:20:11 +0000 Subject: [PATCH 06/10] refactor(chain spec mutator): chain spec parsing error --- javascript/packages/orchestrator/src/chainSpec.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index c5d8d7200..14d219c70 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -747,13 +747,10 @@ export async function runCommandWithChainSpec( // copy the modified file back into the original path after the mutation has completed fs.copyFileSync(chainSpecModifiedPath, chainSpecFullPath); } catch (e: any) { - if (e.code !== "ERR_FS_FILE_TOO_LARGE") throw e; - - console.log( - `\n\t\t ๐Ÿšง ${decorators.yellow( - `Chain Spec file ${chainSpecFullPath} is TOO LARGE to customize (more than 2G).`, - )} ๐Ÿšง`, + console.error( + `\n${decorators.red('Failed to mutate chain spec!')}`, ); + throw e; } } From 6af5c09b30aa1e97991ce02a8f55c20b4d0e42fa Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Wed, 19 Apr 2023 14:28:42 +0000 Subject: [PATCH 07/10] style(chain spec mutator): fix lint problems --- javascript/packages/orchestrator/src/chainSpec.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index 14d219c70..0c0ef4dac 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -723,7 +723,7 @@ export async function runCommandWithChainSpec( { cwd: workingDirectory }, ); // flush the modified spec to a different file and then copy it back into the original path - let spec = fs.createWriteStream(chainSpecModifiedPath); + const spec = fs.createWriteStream(chainSpecModifiedPath); // `pipe` since it deals with flushing and we need to guarantee that the data is flushed // before we resolve the promise. @@ -747,9 +747,7 @@ export async function runCommandWithChainSpec( // copy the modified file back into the original path after the mutation has completed fs.copyFileSync(chainSpecModifiedPath, chainSpecFullPath); } catch (e: any) { - console.error( - `\n${decorators.red('Failed to mutate chain spec!')}`, - ); + console.error(`\n${decorators.red("Failed to mutate chain spec!")}`); throw e; } } From 3bb1add2e6f01fd8e434357d7c47e568d250b0e1 Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Thu, 20 Apr 2023 07:56:03 +0000 Subject: [PATCH 08/10] test(chain spec mutator): fix improper return --- tests/0014-chain-spec-modification.zndsl | 2 +- tests/check-testnet-name.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/0014-chain-spec-modification.zndsl b/tests/0014-chain-spec-modification.zndsl index a2362f0f1..5ecdf4646 100644 --- a/tests/0014-chain-spec-modification.zndsl +++ b/tests/0014-chain-spec-modification.zndsl @@ -7,4 +7,4 @@ alice: is up bob: is up alice: log line matches "Imported #[0-9]+" within 10 seconds -alice: js-script ./check-testnet-name.js with "Tentset Lacol Ococor" return is true +alice: js-script ./check-testnet-name.js with "Tentset Lacol Ococor" return is 1 diff --git a/tests/check-testnet-name.js b/tests/check-testnet-name.js index 6a845cd25..73a641ac3 100644 --- a/tests/check-testnet-name.js +++ b/tests/check-testnet-name.js @@ -6,7 +6,7 @@ async function run(nodeName, networkInfo, args) { const expectedName = args[0] ?? "Rococo Local Testnet"; const testnetName = await api.rpc.system.chain(); - return testnetName == expectedName; + return testnetName == expectedName ? 1 : 0; } module.exports = { run } \ No newline at end of file From f830ed349493697cb8f6900514de9c57e48f31ed Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Thu, 20 Apr 2023 10:44:51 +0000 Subject: [PATCH 09/10] test(chain spec mutator): fix faulty mutator script --- tests/0014-chain-spec-modification.zndsl | 4 ++-- tests/change-name-in-chain-spec.sh | 6 ++++-- tests/check-testnet-name.js | 12 ------------ 3 files changed, 6 insertions(+), 16 deletions(-) delete mode 100644 tests/check-testnet-name.js diff --git a/tests/0014-chain-spec-modification.zndsl b/tests/0014-chain-spec-modification.zndsl index 5ecdf4646..5d68f35c5 100644 --- a/tests/0014-chain-spec-modification.zndsl +++ b/tests/0014-chain-spec-modification.zndsl @@ -5,6 +5,6 @@ Creds: config alice: is up bob: is up -alice: log line matches "Imported #[0-9]+" within 10 seconds -alice: js-script ./check-testnet-name.js with "Tentset Lacol Ococor" return is 1 +alice: log line matches "Tentset Lacol Ococor" +alice: log line matches "Imported #[0-9]+" within 10 seconds diff --git a/tests/change-name-in-chain-spec.sh b/tests/change-name-in-chain-spec.sh index eb1364605..1b79e6426 100755 --- a/tests/change-name-in-chain-spec.sh +++ b/tests/change-name-in-chain-spec.sh @@ -1,11 +1,13 @@ #!/bin/bash # The first argument sent to the script must be the path of the chain spec -chain_spec_path = $1 +chain_spec_path=$1 # Assume that the name old_name="Rococo Local Testnet" # Set the new name for the network new_name="Tentset Lacol Ococor" +input=$(cat $chain_spec_path) + # Replace the name field in the JSON file -sed -i "s/\"name\": \"$old_name\"/\"name\": \"$new_name\"/" $chain_spec_path +echo "$input" | sed "s/\"name\": \"$old_name\"/\"name\": \"$new_name\"/" diff --git a/tests/check-testnet-name.js b/tests/check-testnet-name.js deleted file mode 100644 index 73a641ac3..000000000 --- a/tests/check-testnet-name.js +++ /dev/null @@ -1,12 +0,0 @@ -// Check if the testnet's name is the same as the expected name from the args -async function run(nodeName, networkInfo, args) { - const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName]; - const api = await zombie.connect(wsUri, userDefinedTypes); - - const expectedName = args[0] ?? "Rococo Local Testnet"; - const testnetName = await api.rpc.system.chain(); - - return testnetName == expectedName ? 1 : 0; -} - -module.exports = { run } \ No newline at end of file From 9eb2766a71389e62a768a3f68ddadff3529fe2ac Mon Sep 17 00:00:00 2001 From: Fahrrader Date: Thu, 20 Apr 2023 10:46:01 +0000 Subject: [PATCH 10/10] refactor(chainSpec): remove error swallowing --- .../packages/orchestrator/src/chainSpec.ts | 96 +++++++++---------- 1 file changed, 44 insertions(+), 52 deletions(-) diff --git a/javascript/packages/orchestrator/src/chainSpec.ts b/javascript/packages/orchestrator/src/chainSpec.ts index 0c0ef4dac..3403d0158 100644 --- a/javascript/packages/orchestrator/src/chainSpec.ts +++ b/javascript/packages/orchestrator/src/chainSpec.ts @@ -756,64 +756,56 @@ export async function customizePlainRelayChain( specPath: string, networkSpec: ComputedNetwork, ): Promise { - try { - // Relay-chain spec customization logic - const plainRelayChainSpec = readAndParseChainSpec(specPath); - const keyType = specHaveSessionsKeys(plainRelayChainSpec) - ? "session" - : "aura"; - - // Clear all defaults - clearAuthorities(specPath); - - // add balances for nodes - await addBalances(specPath, networkSpec.relaychain.nodes); - - // add authorities for nodes - const validatorKeys = []; - for (const node of networkSpec.relaychain.nodes) { - if (node.validator) { - validatorKeys.push(node.accounts.sr_stash.address); - - if (keyType === "session") { - const key = getNodeKey(node); - await addAuthority(specPath, node, key); - } else { - await addAuraAuthority(specPath, node.name, node.accounts!); - await addGrandpaAuthority(specPath, node.name, node.accounts!); - } - - await addStaking(specPath, node); + // Relay-chain spec customization logic + const plainRelayChainSpec = readAndParseChainSpec(specPath); + const keyType = specHaveSessionsKeys(plainRelayChainSpec) + ? "session" + : "aura"; + + // Clear all defaults + clearAuthorities(specPath); + + // add balances for nodes + await addBalances(specPath, networkSpec.relaychain.nodes); + + // add authorities for nodes + const validatorKeys = []; + for (const node of networkSpec.relaychain.nodes) { + if (node.validator) { + validatorKeys.push(node.accounts.sr_stash.address); + + if (keyType === "session") { + const key = getNodeKey(node); + await addAuthority(specPath, node, key); + } else { + await addAuraAuthority(specPath, node.name, node.accounts!); + await addGrandpaAuthority(specPath, node.name, node.accounts!); } - } - if (networkSpec.relaychain.randomNominatorsCount) { - await generateNominators( - specPath, - networkSpec.relaychain.randomNominatorsCount, - networkSpec.relaychain.maxNominations, - validatorKeys, - ); + await addStaking(specPath, node); } + } - if (networkSpec.relaychain.genesis) { - await changeGenesisConfig(specPath, networkSpec.relaychain.genesis); - } + if (networkSpec.relaychain.randomNominatorsCount) { + await generateNominators( + specPath, + networkSpec.relaychain.randomNominatorsCount, + networkSpec.relaychain.maxNominations, + validatorKeys, + ); + } - if (networkSpec.hrmp_channels) { - await addHrmpChannelsToGenesis(specPath, networkSpec.hrmp_channels); - } + if (networkSpec.relaychain.genesis) { + await changeGenesisConfig(specPath, networkSpec.relaychain.genesis); + } - // modify the plain chain spec with any custom commands - for (const cmd of networkSpec.relaychain.chainSpecModifierCommands) { - await runCommandWithChainSpec(specPath, cmd, networkSpec.configBasePath); - } - } catch (err) { - console.log( - `\n ${decorators.red("Unexpected error: ")} \t ${decorators.bright( - err, - )}\n`, - ); + if (networkSpec.hrmp_channels) { + await addHrmpChannelsToGenesis(specPath, networkSpec.hrmp_channels); + } + + // modify the plain chain spec with any custom commands + for (const cmd of networkSpec.relaychain.chainSpecModifierCommands) { + await runCommandWithChainSpec(specPath, cmd, networkSpec.configBasePath); } } export default {