From aa7e6fbf0cf4a56acab8405b6c4783323e27c9d3 Mon Sep 17 00:00:00 2001 From: Long Tran Date: Sat, 8 Aug 2020 17:48:58 +0200 Subject: [PATCH 1/8] upgrade boxcars (#251) * Update requirements.txt * Update CARBALL_VERSION * wildcard boxcars-py dependency --- CARBALL_VERSION | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CARBALL_VERSION b/CARBALL_VERSION index 573541ac..d00491fd 100644 --- a/CARBALL_VERSION +++ b/CARBALL_VERSION @@ -1 +1 @@ -0 +1 diff --git a/requirements.txt b/requirements.txt index 6bd58a7f..7b95b1b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ numpy==1.18.2 protobuf==3.6.1 pandas==1.0.3 xlrd==1.1.0 -boxcars-py==0.1.3 +boxcars-py==0.1.* From fcd038e3e5b5e0cb72841142c16b4539e813a86a Mon Sep 17 00:00:00 2001 From: Long Tran Date: Thu, 13 Aug 2020 21:58:07 +0200 Subject: [PATCH 2/8] fix dataframe gzip export (#244) Co-authored-by: dtracers --- carball/analysis/analysis_manager.py | 17 ++++++++++------- carball/tests/export_test.py | 9 +++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/carball/analysis/analysis_manager.py b/carball/analysis/analysis_manager.py index c0d99de5..f4c98000 100644 --- a/carball/analysis/analysis_manager.py +++ b/carball/analysis/analysis_manager.py @@ -1,10 +1,11 @@ import logging import time -from typing import Dict, Callable +from typing import Dict, Callable, Union import pandas as pd import json import os +import gzip from google.protobuf.json_format import _Printer from typing.io import IO @@ -116,20 +117,22 @@ def write_proto_out_to_file(self, file: IO): raise IOError("Proto files must be binary use open(path,\"wb\")") ProtobufManager.write_proto_out_to_file(file, self.protobuf_game) - def write_pandas_out_to_file(self, file: IO): + def write_pandas_out_to_file(self, file: Union[IO, gzip.GzipFile]): """ - Writes the pandas data to the specified file, as bytes. + Writes the pandas data to the specified file, as bytes. File may be a GzipFile object to compress the data + frame. NOTES: The data is written as bytes (i.e. in binary), and the buffer mode must be 'wb'. - E.g. open(file_name, 'wb') + E.g. gzip.open(file_name, 'wb') The file will NOT be human-readable. :param file: The file object (or a buffer). """ - - if 'b' not in file.mode: - raise IOError("Proto files must be binary use open(path,\"wb\")") + if isinstance(file.mode, str) and 'b' not in file.mode: + raise IOError("Data frame files must be binary use open(path,\"wb\")") + if isinstance(file.mode, int) and file.mode != gzip.WRITE: + raise IOError("Gzip compressed data frame files must be opened in WRITE mode.") if self.df_bytes is not None: file.write(self.df_bytes) elif not self.should_store_frames: diff --git a/carball/tests/export_test.py b/carball/tests/export_test.py index 24078540..76d38ebb 100644 --- a/carball/tests/export_test.py +++ b/carball/tests/export_test.py @@ -1,5 +1,6 @@ from tempfile import NamedTemporaryFile +import gzip import pytest from carball.analysis.analysis_manager import AnalysisManager @@ -22,6 +23,14 @@ def test(analysis: AnalysisManager): run_analysis_test_on_replay(test, get_raw_replays()["DEFAULT_3_ON_3_AROUND_58_HITS"], cache=replay_cache) + def test_gzip_export(self, replay_cache): + def test(analysis: AnalysisManager): + with NamedTemporaryFile(mode='wb') as f: + gzip_file = gzip.GzipFile(mode='wb', fileobj=f) + analysis.write_pandas_out_to_file(gzip_file) + + run_analysis_test_on_replay(test, get_raw_replays()["DEFAULT_3_ON_3_AROUND_58_HITS"], cache=replay_cache) + def test_unicode_names(self, replay_cache): def test(analysis: AnalysisManager): with NamedTemporaryFile(mode='wb') as f: From becbc82440b204a0fd6d06ac455e45a37659580b Mon Sep 17 00:00:00 2001 From: Long Tran Date: Mon, 17 Aug 2020 03:57:33 +0200 Subject: [PATCH 3/8] Update setup.py (#252) Co-authored-by: Matthew Mage --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fb074fc9..6babf638 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ def run(self): version=version_string, packages=setuptools.find_packages(), include_package_data=True, - install_requires=['pandas==0.24.2', 'protobuf==3.6.1', 'xlrd==1.1.0', 'numpy==1.17.0', 'boxcars-py==0.1.3'], + install_requires=['pandas==1.0.3', 'protobuf==3.6.1', 'xlrd==1.1.0', 'numpy==1.18.2', 'boxcars-py==0.1.*'], url='https://github.com/SaltieRL/carball', keywords=['rocket-league'], license='Apache 2.0', From 8e2d562ea9c94befb8ab294703a6aa23a6fbf8e0 Mon Sep 17 00:00:00 2001 From: Matthew Mage Date: Mon, 17 Aug 2020 15:07:14 -0400 Subject: [PATCH 4/8] Update CARBALL_VERSION (#254) --- CARBALL_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CARBALL_VERSION b/CARBALL_VERSION index d00491fd..0cfbf088 100644 --- a/CARBALL_VERSION +++ b/CARBALL_VERSION @@ -1 +1 @@ -1 +2 From 0432b2b04c2d12e743b2b7731960a5a796244ec7 Mon Sep 17 00:00:00 2001 From: Matthew Mage Date: Tue, 15 Sep 2020 17:54:36 -0700 Subject: [PATCH 5/8] Demo counts (#255) * Add demos inflicted/taken stat * Fix error logging (why?) and online ID handling * Update CARBALL_VERSION --- CARBALL_VERSION | 2 +- api/stats/player_stats.proto | 6 +++ carball/analysis/analysis_manager.py | 2 +- .../kickoff_detection/kickoff_analysis.py | 4 +- carball/analysis/stats/demos/demos.py | 38 +++++++++++++++++++ carball/analysis/stats/stats_list.py | 4 +- carball/json_parser/player.py | 8 +++- 7 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 carball/analysis/stats/demos/demos.py diff --git a/CARBALL_VERSION b/CARBALL_VERSION index 0cfbf088..00750edc 100644 --- a/CARBALL_VERSION +++ b/CARBALL_VERSION @@ -1 +1 @@ -2 +3 diff --git a/api/stats/player_stats.proto b/api/stats/player_stats.proto index 03288fb3..d0213819 100644 --- a/api/stats/player_stats.proto +++ b/api/stats/player_stats.proto @@ -69,6 +69,7 @@ message PlayerStats { optional CarryDribbles ball_carries = 13; optional CumulativeKickoffStats kickoff_stats = 14; optional api.stats.DropshotStats dropshot_stats = 15; + optional DemoStats demo_stats = 16; } message Controller { @@ -100,3 +101,8 @@ message CumulativeKickoffStats { optional int32 num_time_first_touch = 7; optional float average_boost_used = 8; // The average amount of boost used over all kickoff events } + +message DemoStats { + optional int32 num_demos_inflicted = 1; + optional int32 num_demos_taken = 2; +} diff --git a/carball/analysis/analysis_manager.py b/carball/analysis/analysis_manager.py index f4c98000..2a67b454 100644 --- a/carball/analysis/analysis_manager.py +++ b/carball/analysis/analysis_manager.py @@ -232,7 +232,7 @@ def _get_game_metadata(self, game: Game, proto_game: game_pb2.Game) -> Dict[str, for player in game.players: player_proto = proto_game.players.add() ApiPlayer.create_from_player(player_proto, player, self.id_creator) - player_map[str(player.online_id)] = player_proto + player_map[player.online_id] = player_proto return player_map diff --git a/carball/analysis/events/kickoff_detection/kickoff_analysis.py b/carball/analysis/events/kickoff_detection/kickoff_analysis.py index a73d1171..cf5049ce 100644 --- a/carball/analysis/events/kickoff_detection/kickoff_analysis.py +++ b/carball/analysis/events/kickoff_detection/kickoff_analysis.py @@ -42,8 +42,8 @@ def get_kickoffs_from_game(game: Game, proto_game: game_pb2, id_creation:Callabl summed_time = smaller_data_frame['game']['delta'][frame:end_frame].sum() if summed_time > 0: cur_kickoff.touch_time = summed_time - logger.error("STRAIGHT TIME " + str(time)) - logger.error("SUM TIME" + str(summed_time)) + logger.info("STRAIGHT TIME " + str(time)) + logger.info("SUM TIME" + str(summed_time)) sum_vs_adding_diff = time - summed_time diff --git a/carball/analysis/stats/demos/demos.py b/carball/analysis/stats/demos/demos.py new file mode 100644 index 00000000..5abcd348 --- /dev/null +++ b/carball/analysis/stats/demos/demos.py @@ -0,0 +1,38 @@ +import logging +from typing import Dict + +import numpy as np +import pandas as pd +from carball.analysis.constants.field_constants import FieldConstants + +from carball.analysis.stats.utils.pandas_utils import sum_deltas_by_truthy_data +from ....analysis.stats.stats import BaseStat +from ....generated.api import game_pb2 +from ....generated.api.player_pb2 import Player +from ....generated.api.stats.player_stats_pb2 import PlayerStats +from ....json_parser.actor.boost import BOOST_PER_SECOND +from ....json_parser.game import Game + +logger = logging.getLogger(__name__) + + +class DemoStat(BaseStat): + def calculate_player_stat(self, player_stat_map: Dict[str, PlayerStats], game: Game, proto_game: game_pb2.Game, + player_map: Dict[str, Player], data_frame: pd.DataFrame): + player_demo_counts = {} + player_got_demoed_counts = {} + for demo in game.demos: + attacker = demo['attacker'].online_id + victim = demo['victim'].online_id + if attacker not in player_demo_counts: + player_demo_counts[attacker] = 1 + else: + player_demo_counts[attacker] += 1 + if victim not in player_got_demoed_counts: + player_got_demoed_counts[victim] = 1 + else: + player_got_demoed_counts[victim] += 1 + for player in player_demo_counts: + player_stat_map[player].demo_stats.num_demos_inflicted = player_demo_counts[player] + for player in player_got_demoed_counts: + player_stat_map[player].demo_stats.num_demos_taken = player_got_demoed_counts[player] diff --git a/carball/analysis/stats/stats_list.py b/carball/analysis/stats/stats_list.py index de755055..7cb5112b 100644 --- a/carball/analysis/stats/stats_list.py +++ b/carball/analysis/stats/stats_list.py @@ -1,5 +1,6 @@ from typing import List +from carball.analysis.stats.demos.demos import DemoStat from carball.analysis.stats.dribbles.ball_carry import CarryStat from carball.analysis.stats.kickoffs.kickoff_stat import KickoffStat from carball.analysis.stats.possession.per_possession import PerPossessionStat @@ -42,7 +43,8 @@ def get_player_stats() -> List[BaseStat]: SpeedTendencies(), RumbleItemStat(), KickoffStat(), - DropshotStats() + DropshotStats(), + DemoStat() ] @staticmethod diff --git a/carball/json_parser/player.py b/carball/json_parser/player.py index b5d69d68..5d8505b9 100644 --- a/carball/json_parser/player.py +++ b/carball/json_parser/player.py @@ -49,6 +49,11 @@ def __repr__(self): else: return '%s: %s' % (self.__class__.__name__, self.name) + def _get_player_id(self, online_id): + if type(online_id) == dict: + return online_id['online_id'] + return online_id + def create_from_actor_data(self, actor_data: dict, teams: List['Team'], objects: List[str]): self.name = actor_data['name'] if 'Engine.PlayerReplicationInfo:bBot' in actor_data and actor_data['Engine.PlayerReplicationInfo:bBot']: @@ -57,7 +62,8 @@ def create_from_actor_data(self, actor_data: dict, teams: List['Team'], objects: else: actor_type = list(actor_data["Engine.PlayerReplicationInfo:UniqueId"]['remote_id'].keys())[0] - self.online_id = actor_data["Engine.PlayerReplicationInfo:UniqueId"]['remote_id'][actor_type] + self.online_id = self._get_player_id(actor_data["Engine.PlayerReplicationInfo:UniqueId"] + ['remote_id'][actor_type]) try: self.score = actor_data["TAGame.PRI_TA:MatchScore"] except KeyError: From cd3dd59a2213fc1ca7ae4e4afdc1be0286312832 Mon Sep 17 00:00:00 2001 From: Matthew Mage Date: Wed, 23 Sep 2020 00:24:37 -0700 Subject: [PATCH 6/8] Demo count fix (#256) * Add demos inflicted/taken stat * Fix error logging (why?) and online ID handling * Update CARBALL_VERSION * Add __init__ * Update CARBALL_VERSION --- CARBALL_VERSION | 2 +- carball/analysis/stats/demos/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 carball/analysis/stats/demos/__init__.py diff --git a/CARBALL_VERSION b/CARBALL_VERSION index 00750edc..b8626c4c 100644 --- a/CARBALL_VERSION +++ b/CARBALL_VERSION @@ -1 +1 @@ -3 +4 diff --git a/carball/analysis/stats/demos/__init__.py b/carball/analysis/stats/demos/__init__.py new file mode 100644 index 00000000..e69de29b From 29a4c27c50ce11175d1a8bf5410636c24be943a5 Mon Sep 17 00:00:00 2001 From: mtdiedrich Date: Sat, 3 Oct 2020 01:04:02 -0500 Subject: [PATCH 7/8] Add instructions to README for resolving problems related to default shell permissions on MacOS (#258) * Add instructions to README for resolving problems related to default shell permissions on MacOS * Improve clarity --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index b7c1a3b6..7a75d0ed 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,17 @@ cd carball/ python init.py ``` +##### Mac +In MacOS Catalina, zsh replaced bash as the default shell, which may cause permission issues when trying to run `install-protoc.sh` in the above fashion. Simply invoking bash should resolve this issue, like so: +``` +git clone https://github.com/SaltieRL/carball +cd carball/ +bash ./_travis/install-protoc.sh +python init.py +``` +Apple's decision to replace bash as the default shell may foreshadow the removal of bash in a future version of MacOS. In such a case, Homebrew users can [install protoc](http://google.github.io/proto-lens/installing-protoc.html) by replacing `bash ./travis/install-protoc.sh` with `brew install protobuf`. + + ## Examples / Usage One of the main data structures used in carball is the pandas.DataFrame, to learn more, see [its wiki page](https://github.com/SaltieRL/carball/wiki/data_frame). From 3e66f175378f050b84bc9a3b15de0420505092a7 Mon Sep 17 00:00:00 2001 From: Matthew Mage Date: Tue, 13 Oct 2020 21:03:13 -0700 Subject: [PATCH 8/8] Add ballcam/handbrake times (#259) * Add ballcam/handbrake times * Remove comments --- CARBALL_VERSION | 2 +- api/stats/player_stats.proto | 2 ++ carball/analysis/stats/controls/controls.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CARBALL_VERSION b/CARBALL_VERSION index b8626c4c..7ed6ff82 100644 --- a/CARBALL_VERSION +++ b/CARBALL_VERSION @@ -1 +1 @@ -4 +5 diff --git a/api/stats/player_stats.proto b/api/stats/player_stats.proto index d0213819..5e9885a8 100644 --- a/api/stats/player_stats.proto +++ b/api/stats/player_stats.proto @@ -76,6 +76,8 @@ message Controller { optional bool is_keyboard = 1; optional float analogue_steering_input_percent = 2; optional float analogue_throttle_input_percent = 3; + optional float time_ballcam = 4; + optional float time_handbrake = 5; } // Stats for carrying diff --git a/carball/analysis/stats/controls/controls.py b/carball/analysis/stats/controls/controls.py index ea167a99..4f401922 100644 --- a/carball/analysis/stats/controls/controls.py +++ b/carball/analysis/stats/controls/controls.py @@ -1,9 +1,11 @@ from logging import getLogger from typing import Dict +import numpy as np import pandas as pd from ....analysis.stats.stats import BaseStat +from ....analysis.stats.utils.pandas_utils import sum_deltas_by_truthy_data from ....generated.api import game_pb2 from ....generated.api.player_pb2 import Player from ....generated.api.stats.player_stats_pb2 import PlayerStats @@ -20,6 +22,7 @@ def calculate_player_stat(self, player_stat_map: Dict[str, PlayerStats], game: G for player_key, stats in player_map.items(): try: player_name = player_map[player_key].name + player_data_frame = data_frame[player_name].copy() steering_percentage = self.get_analogue_percentage("steer", data_frame, player_name) throttle_percentage = self.get_analogue_percentage("throttle", data_frame, player_name) @@ -29,6 +32,12 @@ def calculate_player_stat(self, player_stat_map: Dict[str, PlayerStats], game: G controller_stats.is_keyboard = is_keyboard controller_stats.analogue_steering_input_percent = throttle_percentage controller_stats.analogue_throttle_input_percent = steering_percentage + if 'ball_cam' in player_data_frame: + time_ballcam = self.get_ballcam_duration(data_frame, player_data_frame) + controller_stats.time_ballcam = time_ballcam + if 'handbrake' in player_data_frame: + time_handbrake = self.get_handbrake_duration(data_frame, player_data_frame) + controller_stats.time_handbrake = time_handbrake except KeyError as e: logger.warning('Player never pressed control %s', e) @@ -36,3 +45,11 @@ def get_analogue_percentage(self, column: str, data_frame: pd.DataFrame, player_ total_frames = len(data_frame[player_name][column]) count = (data_frame[player_name][column] == 0).sum() + (data_frame[player_name][column] == 128).sum() + (data_frame[player_name][column] == 255).sum() + data_frame[player_name][column].isna().sum() return 100 - ((count * 100) / total_frames) + + @staticmethod + def get_ballcam_duration(data_frame: pd.DataFrame, player_dataframe: pd.DataFrame) -> np.float64: + return sum_deltas_by_truthy_data(data_frame, player_dataframe.ball_cam) + + @staticmethod + def get_handbrake_duration(data_frame: pd.DataFrame, player_dataframe: pd.DataFrame) -> np.float64: + return sum_deltas_by_truthy_data(data_frame, player_dataframe.handbrake)