diff --git a/setup.py b/setup.py index a0b859c14..4cd448939 100644 --- a/setup.py +++ b/setup.py @@ -164,7 +164,7 @@ def run(self): 'natsort>=7.1.1', 'networkx>=2.5', 'numpy>=1.19.2', - 'omegaconf==2.0.6', + 'omegaconf==2.3.0', 'openpyxl>=3.0.7', 'packaging>=21.0', 'pandas>=1.1.5', @@ -198,7 +198,7 @@ def run(self): 'pydocstyle>=5.1.1', 'pytest-cov>=2.11.1', 'pytest-subtests>=0.4.0', - 'pytest>=6.2.2', + 'pytest>=6.2.2, <=7.4.4', 'types-markdown', 'types-pkg_resources', 'types-pyyaml', diff --git a/superbench/executor/executor.py b/superbench/executor/executor.py index bfff5cb7c..c4a812a9c 100644 --- a/superbench/executor/executor.py +++ b/superbench/executor/executor.py @@ -71,13 +71,13 @@ def __get_enabled_benchmarks(self): Return: list: List of benchmarks which will be executed. """ - if self._sb_config.superbench.enable: + if 'enable' in self._sb_config.superbench and self._sb_config.superbench.enable: if isinstance(self._sb_config.superbench.enable, str): return [self._sb_config.superbench.enable] elif isinstance(self._sb_config.superbench.enable, (list, ListConfig)): return list(self._sb_config.superbench.enable) # TODO: may exist order issue - return [k for k, v in self._sb_benchmarks.items() if v.enable] + return [k for k, v in self._sb_benchmarks.items() if 'enable' in v and v.enable] def __get_platform(self): """Detect runninng platform by environment.""" @@ -228,32 +228,37 @@ def exec(self): logger.warning('Monitor can not support CPU platform.') benchmark_real_name = benchmark_name.split(':')[0] - for framework in benchmark_config.frameworks or [Framework.NONE.value]: - if benchmark_real_name == 'model-benchmarks' or ( - ':' not in benchmark_name and benchmark_name.endswith('_models') - ): - for model in benchmark_config.models: - full_name = f'{benchmark_name}/{framework}-{model}' + if 'frameworks' in benchmark_config: + for framework in benchmark_config.frameworks or [Framework.NONE.value]: + if benchmark_real_name == 'model-benchmarks' or ( + ':' not in benchmark_name and benchmark_name.endswith('_models') + ): + for model in benchmark_config.models: + full_name = f'{benchmark_name}/{framework}-{model}' + logger.info('Executor is going to execute %s.', full_name) + context = BenchmarkRegistry.create_benchmark_context( + model, + platform=self.__get_platform(), + framework=Framework(framework.lower()), + parameters=self.__get_arguments( + {} if 'parameters' not in benchmark_config else benchmark_config.parameters + ) + ) + result = self.__exec_benchmark(full_name, context) + benchmark_results.append(result) + else: + full_name = benchmark_name logger.info('Executor is going to execute %s.', full_name) context = BenchmarkRegistry.create_benchmark_context( - model, + benchmark_real_name, platform=self.__get_platform(), framework=Framework(framework.lower()), - parameters=self.__get_arguments(benchmark_config.parameters) + parameters=self.__get_arguments( + {} if 'parameters' not in benchmark_config else benchmark_config.parameters + ) ) result = self.__exec_benchmark(full_name, context) benchmark_results.append(result) - else: - full_name = benchmark_name - logger.info('Executor is going to execute %s.', full_name) - context = BenchmarkRegistry.create_benchmark_context( - benchmark_real_name, - platform=self.__get_platform(), - framework=Framework(framework.lower()), - parameters=self.__get_arguments(benchmark_config.parameters) - ) - result = self.__exec_benchmark(full_name, context) - benchmark_results.append(result) if monitor: monitor.stop() diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py index 7e29f4dfe..cd0c8c4dc 100644 --- a/superbench/runner/runner.py +++ b/superbench/runner/runner.py @@ -67,24 +67,24 @@ def __validate_sb_config(self): # noqa: C901 InvalidConfigError: If input config is invalid. """ # TODO: add validation and defaulting - if not self._sb_config.superbench.env: + if 'env' not in self._sb_config.superbench: self._sb_config.superbench.env = {} for name in self._sb_benchmarks: - if not self._sb_benchmarks[name].modes: + if 'modes' not in self._sb_benchmarks[name]: self._sb_benchmarks[name].modes = [] for idx, mode in enumerate(self._sb_benchmarks[name].modes): - if not mode.env: + if 'env' not in mode: self._sb_benchmarks[name].modes[idx].env = {} if mode.name == 'local': - if not mode.proc_num: + if 'proc_num' not in mode: self._sb_benchmarks[name].modes[idx].proc_num = 1 - if not mode.prefix: + if 'prefix' not in mode: self._sb_benchmarks[name].modes[idx].prefix = '' elif mode.name == 'torch.distributed': - if not mode.proc_num: + if 'proc_num' not in mode: self._sb_benchmarks[name].modes[idx].proc_num = 8 elif mode.name == 'mpi': - if not mode.mca: + if 'machinefile' not in mode: self._sb_benchmarks[name].modes[idx].mca = { 'pml': 'ob1', 'btl': '^openib', @@ -93,8 +93,8 @@ def __validate_sb_config(self): # noqa: C901 } for key in ['PATH', 'LD_LIBRARY_PATH', 'SB_MICRO_PATH', 'SB_WORKSPACE']: self._sb_benchmarks[name].modes[idx].env.setdefault(key, None) - if mode.pattern: - if mode.pattern.type == 'topo-aware' and not mode.pattern.ibstat: + if 'pattern' in mode: + if mode.pattern.type == 'topo-aware' and 'ibstat' not in mode.pattern: self._sb_benchmarks[name].modes[idx].pattern.ibstat = gen_ibstat( self._ansible_config, str(self._output_path / 'ibstate_file.txt') ) @@ -105,12 +105,12 @@ def __get_enabled_benchmarks(self): Return: list: List of benchmarks which will be executed. """ - if self._sb_config.superbench.enable: + if 'enable' in self._sb_config.superbench and self._sb_config.superbench.enable: if isinstance(self._sb_config.superbench.enable, str): return [self._sb_config.superbench.enable] elif isinstance(self._sb_config.superbench.enable, (list, ListConfig)): return list(self._sb_config.superbench.enable) - return [k for k, v in self._sb_benchmarks.items() if v.enable] + return [k for k, v in self._sb_benchmarks.items() if 'enable' in v and v.enable] def __get_mode_command(self, benchmark_name, mode, timeout=None): """Get runner command for given mode. @@ -141,7 +141,7 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None): elif mode.name == 'torch.distributed': # TODO: replace with torch.distributed.run in v1.9 # TODO: only supports node_num=1 and node_num=all currently - torch_dist_params = '' if mode.node_num == 1 else \ + torch_dist_params = '' if 'node_num' in mode and mode.node_num == 1 else \ '--nnodes=$NNODES --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT ' mode_command = ( f'torchrun' @@ -158,8 +158,8 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None): '-bind-to numa ' # bind processes to numa '{mca_list} {env_list} {command}' ).format( - host_list=f'-host localhost:{mode.proc_num}' if mode.node_num == 1 else - f'-hostfile hostfile -map-by ppr:{mode.proc_num}:node' if mode.host_list is None else '-host ' + + host_list=f'-host localhost:{mode.proc_num}' if 'node_num' in mode and mode.node_num == 1 else + f'-hostfile hostfile -map-by ppr:{mode.proc_num}:node' if 'host_list' not in mode else '-host ' + ','.join(f'{host}:{mode.proc_num}' for host in mode.host_list), mca_list=' '.join(f'-mca {k} {v}' for k, v in mode.mca.items()), env_list=' '.join( @@ -206,6 +206,9 @@ def run_sys_info(self): logger.info('Runner is going to get node system info.') fcmd = "docker exec sb-workspace bash -c '{command}'" + + if 'skip' not in self._docker_config: + self._docker_config.skip = False if self._docker_config.skip: fcmd = "bash -c 'cd $SB_WORKSPACE && {command}'" ansible_runner_config = self._ansible_client.get_shell_config( @@ -225,7 +228,7 @@ def check_env(self): # pragma: no cover self._ansible_client.get_playbook_config( 'check_env.yaml', extravars={ - 'no_docker': bool(self._docker_config.skip), + 'no_docker': False if 'skip' not in self._docker_config else self._docker_config.skip, 'output_dir': str(self._output_path), 'env': '\n'.join(f'{k}={v}' for k, v in self._sb_config.superbench.env.items()), } @@ -441,15 +444,17 @@ def _run_proc(self, benchmark_name, mode, vars): int: Process return code. """ mode.update(vars) - if mode.name == 'mpi' and mode.pattern: + if mode.name == 'mpi' and 'pattern' in mode: mode.env.update({'SB_MODE_SERIAL_INDEX': mode.serial_index, 'SB_MODE_PARALLEL_INDEX': mode.parallel_index}) logger.info('Runner is going to run %s in %s mode, proc rank %d.', benchmark_name, mode.name, mode.proc_rank) - timeout = self._sb_benchmarks[benchmark_name].timeout + timeout = self._sb_benchmarks[benchmark_name].get('timeout', 60) if isinstance(timeout, int): timeout = max(timeout, 60) env_list = '--env-file /tmp/sb.env' + if 'skip' not in self._docker_config: + self._docker_config.skip = False if self._docker_config.skip: env_list = 'set -o allexport && source /tmp/sb.env && set +o allexport' for k, v in mode.env.items(): @@ -463,7 +468,7 @@ def _run_proc(self, benchmark_name, mode, vars): ansible_runner_config = self._ansible_client.get_shell_config( fcmd.format(env_list=env_list, command=self.__get_mode_command(benchmark_name, mode, timeout)) ) - if mode.name == 'mpi' and mode.node_num != 1: + if mode.name == 'mpi' and 'node_num' in mode and mode.node_num != 1: ansible_runner_config = self._ansible_client.update_mpi_config(ansible_runner_config) if isinstance(timeout, int): @@ -495,7 +500,7 @@ def run(self): ) ansible_rc = sum(rc_list) elif mode.name == 'torch.distributed' or mode.name == 'mpi': - if not mode.pattern: + if 'pattern' not in mode: ansible_rc = self._run_proc(benchmark_name, mode, {'proc_rank': 0}) else: if not os.path.exists(self._output_path / 'hostfile'): diff --git a/tests/executor/test_executor.py b/tests/executor/test_executor.py index a9365e6c1..984f0437e 100644 --- a/tests/executor/test_executor.py +++ b/tests/executor/test_executor.py @@ -44,7 +44,7 @@ def test_set_logger(self): def test_get_enabled_benchmarks_enable_none(self): """Test enabled benchmarks when superbench.enable is none.""" benchmarks = self.default_config.superbench.benchmarks - expected_enabled_benchmarks = [x for x in benchmarks if benchmarks[x]['enable']] + expected_enabled_benchmarks = [x for x in benchmarks if 'enable' in benchmarks[x] and benchmarks[x]['enable']] self.assertListEqual(self.executor._sb_enabled, expected_enabled_benchmarks) def test_get_enabled_benchmarks_enable_str(self):