diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f44dd2a4..7d4c67be99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Copy and pasting the git commit messages is __NOT__ enough. ### Added - Added a zoo agent, named Control-and-Supervised-Learning, from NeurIPS 2022 submission. This zoo agent runs in benchmark `driving_smarts==0.0`. - Added a zoo agent, named Discrete Soft Actor Critic, from NeurIPS 2022 submission. This zoo agent runs in benchmark `driving_smarts==0.0`. +- Added `"driving_smarts_full==0.0"` benchmark to the set of available benchmarks. +- Added `"driving-smarts-competition-bubble-env-v0"` to the set of available environments. This requires installing `bubble_env` in order to run. ### Changed ### Deprecated ### Fixed @@ -31,6 +33,7 @@ Copy and pasting the git commit messages is __NOT__ enough. - Individualised the agent instances in the `benchmark_runner_v0.py`. - Made driving_smarts_competition_v0 env configurable through supply of `AgentInterface`. - Observation of driving_smarts_competition_v0 env was fixed to be of type `ObservationOptions.unformatted`. +- The V0 benchmark runner now uses `load_config` from the benchmarks module. ### Deprecated ### Fixed - Fixed an exit error that occurs when envision attempts to close down. diff --git a/smarts/benchmark/__init__.py b/smarts/benchmark/__init__.py index c9c64c168d..533c53820b 100644 --- a/smarts/benchmark/__init__.py +++ b/smarts/benchmark/__init__.py @@ -25,6 +25,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional +from smarts.core.utils.resources import load_yaml_config as _load_config + BENCHMARK_LISTING_FILE = Path(__file__).parent.absolute() / "benchmark_listing.yaml" @@ -118,3 +120,10 @@ def list_benchmarks(benchmark_listing): from smarts.core.utils.resources import load_yaml_config_with_substitution return load_yaml_config_with_substitution(Path(benchmark_listing)) + + +def load_config(path: Path) -> Optional[Dict[str, Any]]: + """Load in a benchmark config.""" + if isinstance(path, (str,)): + path = Path(path) + return _load_config(path) diff --git a/smarts/benchmark/benchmark_listing.yaml b/smarts/benchmark/benchmark_listing.yaml index ab40af7e6f..575302abd1 100644 --- a/smarts/benchmark/benchmark_listing.yaml +++ b/smarts/benchmark/benchmark_listing.yaml @@ -10,4 +10,13 @@ benchmarks: requirements: ["ray<=2.2.0,>2.0"] params: benchmark_config: ${{smarts.benchmark.driving_smarts.v0}}/config.yaml + driving_smarts_full: + name: "Driving SMARTS Full" + versions: + - + version: 0.0 + entrypoint: "smarts.benchmark.entrypoints.benchmark_runner_v0.benchmark_from_configs" + requirements: ["ray<=2.2.0,>2.0"] + params: + benchmark_config: ${{smarts.benchmark.driving_smarts_full.v0}}/config.yaml \ No newline at end of file diff --git a/smarts/benchmark/driving_smarts/__init__.py b/smarts/benchmark/driving_smarts/__init__.py index 0df1109f62..2c65923417 100644 --- a/smarts/benchmark/driving_smarts/__init__.py +++ b/smarts/benchmark/driving_smarts/__init__.py @@ -19,14 +19,3 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from pathlib import Path -from typing import Any, Dict, Optional - -from smarts.core.utils.resources import load_yaml_config as _load_config - - -def load_config(path: Path) -> Optional[Dict[str, Any]]: - """Load in a benchmark config.""" - if isinstance(path, (str,)): - path = Path(path) - return _load_config(path) diff --git a/smarts/benchmark/driving_smarts_full/__init__.py b/smarts/benchmark/driving_smarts_full/__init__.py new file mode 100644 index 0000000000..2c65923417 --- /dev/null +++ b/smarts/benchmark/driving_smarts_full/__init__.py @@ -0,0 +1,21 @@ +# MIT License +# +# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. diff --git a/smarts/benchmark/driving_smarts_full/v0/__init__.py b/smarts/benchmark/driving_smarts_full/v0/__init__.py new file mode 100644 index 0000000000..84a9e0a91c --- /dev/null +++ b/smarts/benchmark/driving_smarts_full/v0/__init__.py @@ -0,0 +1,24 @@ +# MIT License +# +# Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from pathlib import Path + +DEFAULT_CONFIG = str((Path(__file__).parent / "config.yaml").absolute()) diff --git a/smarts/benchmark/driving_smarts_full/v0/config.yaml b/smarts/benchmark/driving_smarts_full/v0/config.yaml new file mode 100644 index 0000000000..3050b346c4 --- /dev/null +++ b/smarts/benchmark/driving_smarts_full/v0/config.yaml @@ -0,0 +1,53 @@ +# smarts/benchmark/driving_smarts_v0/config.yaml +--- +benchmark: + name: "Driving SMARTS V0" + message: | # A useful message given at the start of the benchmark. + This is a cleaned up version of the Driving SMARTS benchmark. + + Using `TargetPose` agent action has an applied 28m/s cap for agent motion. + Using `RelativeTargetPose` agent action, the constraint is inbuilt into the action space. + + If bubble_env is not installed, please install the bubble_env repository: + + ```bash + # set $REPOS to wherever you wish to store the repository. + git lfs clone https://bitbucket.org/malban/bubble_env.git $REPOS/bubble_env + # read $REPOS/README.md and follow those instructions + pip install $REPOS/bubble_env + ``` + + For history see: + - https://codalab.lisn.upsaclay.fr/competitions/6618 + - https://smarts-project.github.io/archive/2022_nips_driving_smarts/ + eval_episodes: 50 + shared_env_kwargs: + seed: 42 + headless: true + envs: + bubble: # reserved for bubble env + loc: "smarts.env:driving-smarts-competition-bubble-env-v0" + scenarios: + - 6 + - 4 + - 42 + standard: + loc: "smarts.env:driving-smarts-competition-v0" + scenarios: + - 1_to_2lane_left_turn_c + - 1_to_2lane_left_turn_t + - 3lane_merge_multi_agent + - 3lane_merge_single_agent + - 3lane_cruise_multi_agent + - 3lane_cruise_single_agent + - 3lane_cut_in + - 3lane_overtake + kwargs: + seed: 42 + # kwargs: + # naturalistic: # reserved for driving_smarts_competition_naturalistic-v0 + # loc: "smarts.env:driving_smarts_competition_naturalistic-v0" + # kwargs: + # scenario_dirs: + # - "./scenarios/naturalistic/waymo" + # - "./scenarios/naturalistic/ngsim" diff --git a/smarts/benchmark/entrypoints/benchmark_runner_v0.py b/smarts/benchmark/entrypoints/benchmark_runner_v0.py index d400f0b2ab..7048aa907d 100644 --- a/smarts/benchmark/entrypoints/benchmark_runner_v0.py +++ b/smarts/benchmark/entrypoints/benchmark_runner_v0.py @@ -27,7 +27,7 @@ import psutil import ray -from smarts.benchmark.driving_smarts import load_config +from smarts.benchmark import load_config from smarts.core.utils.logging import suppress_output from smarts.env.gymnasium.wrappers.metrics import Metrics, Score from smarts.zoo import registry as agent_registry diff --git a/smarts/env/gymnasium/__init__.py b/smarts/env/gymnasium/__init__.py index a766a429b4..77cf18c156 100644 --- a/smarts/env/gymnasium/__init__.py +++ b/smarts/env/gymnasium/__init__.py @@ -32,6 +32,12 @@ entry_point="smarts.env.gymnasium.driving_smarts_competition_env:driving_smarts_competition_v0_env", disable_env_checker=True, ) + + register( + id="driving-smarts-competition-bubble-env-v0", + entry_point="smarts.env.gymnasium.driving_smarts_bubble_env:checked_bubble_env_v0", + disable_env_checker=True, + ) except ModuleNotFoundError: import warnings diff --git a/smarts/env/gymnasium/driving_smarts_bubble_env.py b/smarts/env/gymnasium/driving_smarts_bubble_env.py new file mode 100644 index 0000000000..feec1d311a --- /dev/null +++ b/smarts/env/gymnasium/driving_smarts_bubble_env.py @@ -0,0 +1,80 @@ +# MIT License +# +# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +import gymnasium as gym + +from smarts.core.agent_interface import AgentInterface + + +def checked_bubble_env_v0( + agent_interface: AgentInterface, + headless: bool = True, + seed: int = 42, + **kwargs, +): + """Imports and generates the `bubble_env` environment which converts nearby replay + traffic (non-reactive) into model based traffic (reactive). This uses the NGSIM i80 + dataset as a basis. + + Args: + traffic_mode (Literal["traffic_A"]): The version of bubble traffic to use. + action_space (ActionSpaceType): The action space the agent should use. + img_meters (float): The square side dimensions of the surface the top-down rgb image + portrays. This affects resolution. + img_pixels (float): The total number of pixels in the top-down rgb image. This affects + resolution. + headless (bool): If the environment should display sumo-gui. + seed (int): The seed of the environment. + + + Returns: + (gymnasium.Env): The bubble_env environment. + + Raises: + ImportError: If `bubble_env` is not installed. + """ + try: + import bubble_env_contrib + except ImportError as err: + raise ImportError( + """ + Bubble env is not installed. + + If bubble_env is not installed, please install the bubble_env repository: + + ```bash + # set $REPOS to wherever you wish to store the repository. + git lfs clone https://bitbucket.org/malban/bubble_env.git $REPOS/bubble_env + # read $REPOS/README.md and follow those instructions + cd - + pip install $REPOS/bubble_env + ``` + """, + "Install bubble_env", + ) from err + + env = gym.make( + "bubble_env_contrib:bubble_env-v1", + agent_interface=agent_interface, + headless=headless, + seed=seed, + ) + return env