From 00d924578964eca4bbd1dd39f33695ad80fa5a98 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 28 Mar 2024 08:02:59 -0600 Subject: [PATCH] Cobbling together a test to guage how starting ranks affect the rating pool --- Makefile | 4 +- .../analyze_glicko2_one_game_at_a_time.py | 112 +++-- analysis/util/InMemoryStorage.py | 3 + .../util/InMemoryWithStartingRankStorage.py | 256 +++++++++++ analysis/util/TallyGameAnalytics.py | 421 +++++++++++------- goratings/interfaces/Storage.py | 4 + goratings/math/glicko2.py | 123 +++-- 7 files changed, 709 insertions(+), 214 deletions(-) create mode 100644 analysis/util/InMemoryWithStartingRankStorage.py diff --git a/Makefile b/Makefile index 6ffb901..6c3f86e 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ help: virtualenv -ppython3 .venv .venv/bin/pip install -r requirements.txt -100k: - python -m goratings +1M: + analysis/analyze_glicko2_one_game_at_a_time.py --games 1000000 test: .venv .venv/bin/tox -e py3 diff --git a/analysis/analyze_glicko2_one_game_at_a_time.py b/analysis/analyze_glicko2_one_game_at_a_time.py index cbfebf6..e7f79a3 100755 --- a/analysis/analyze_glicko2_one_game_at_a_time.py +++ b/analysis/analyze_glicko2_one_game_at_a_time.py @@ -1,5 +1,7 @@ #!/usr/bin/env -S PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=..:. pypy3 +import sys +import logging from analysis.util import ( Glicko2Analytics, InMemoryStorage, @@ -12,9 +14,13 @@ rank_to_rating, should_skip_game, ) +from analysis.util.InMemoryWithStartingRankStorage import ( + InMemoryStorageWithStartingRankStorage, +) from goratings.interfaces import GameRecord, RatingSystem, Storage from goratings.math.glicko2 import Glicko2Entry, glicko2_update + class OneGameAtATime(RatingSystem): _storage: Storage @@ -23,10 +29,16 @@ def __init__(self, storage: Storage) -> None: def process_game(self, game: GameRecord) -> Glicko2Analytics: if game.black_manual_rank_update is not None: - self._storage.set(game.black_id, Glicko2Entry(rank_to_rating(game.black_manual_rank_update))) + self._storage.set( + game.black_id, + Glicko2Entry(rank_to_rating(game.black_manual_rank_update)), + ) if game.white_manual_rank_update is not None: - self._storage.set(game.white_id, Glicko2Entry(rank_to_rating(game.white_manual_rank_update))) + self._storage.set( + game.white_id, + Glicko2Entry(rank_to_rating(game.white_manual_rank_update)), + ) if should_skip_game(game, self._storage): return Glicko2Analytics(skipped=True, game=game) @@ -38,10 +50,16 @@ def process_game(self, game: GameRecord) -> Glicko2Analytics: black, [ ( - white.copy(get_handicap_adjustment("white", white.rating, game.handicap, - komi=game.komi, size=game.size, - rules=game.rules, - )), + white.copy( + get_handicap_adjustment( + "white", + white.rating, + game.handicap, + komi=game.komi, + size=game.size, + rules=game.rules, + ) + ), game.winner_id == game.black_id, ) ], @@ -52,10 +70,16 @@ def process_game(self, game: GameRecord) -> Glicko2Analytics: white, [ ( - black.copy(get_handicap_adjustment("black", black.rating, game.handicap, - komi=game.komi, size=game.size, - rules=game.rules, - )), + black.copy( + get_handicap_adjustment( + "black", + black.rating, + game.handicap, + komi=game.komi, + size=game.size, + rules=game.rules, + ) + ), game.winner_id == game.white_id, ) ], @@ -64,17 +88,23 @@ def process_game(self, game: GameRecord) -> Glicko2Analytics: self._storage.set(game.black_id, updated_black) self._storage.set(game.white_id, updated_white) - #self._storage.add_rating_history(game.black_id, game.ended, updated_black) - #self._storage.add_rating_history(game.white_id, game.ended, updated_white) + # self._storage.add_rating_history(game.black_id, game.ended, updated_black) + # self._storage.add_rating_history(game.white_id, game.ended, updated_white) return Glicko2Analytics( skipped=False, game=game, expected_win_rate=black.expected_win_probability( - white, get_handicap_adjustment("black", black.rating, game.handicap, - komi=game.komi, size=game.size, - rules=game.rules, - ), ignore_g=True + white, + get_handicap_adjustment( + "black", + black.rating, + game.handicap, + komi=game.komi, + size=game.size, + rules=game.rules, + ), + ignore_g=True, ), black_rating=black.rating, white_rating=white.rating, @@ -87,11 +117,24 @@ def process_game(self, game: GameRecord) -> Glicko2Analytics: ) - # Run +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + +cli.add_argument( + "--starting-ranks", + dest="starting_ranks", + action="store_true", + help="Use a two pass system to bin players into an appropriate starting rank.", +) + config(cli.parse_args(), "glicko2-one-game-at-a-time") game_data = GameData() -storage = InMemoryStorage(Glicko2Entry) + +if config.args.starting_ranks: + storage = InMemoryStorageWithStartingRankStorage(Glicko2Entry) +else: + storage = InMemoryStorage(Glicko2Entry) + engine = OneGameAtATime(storage) tally = TallyGameAnalytics(storage) @@ -101,17 +144,34 @@ def process_game(self, game: GameRecord) -> Glicko2Analytics: tally.print() +storage.finalize() + self_reported_ratings = tally.get_self_reported_rating() if self_reported_ratings: - aga_1d = (self_reported_ratings['aga'][30] if 'aga' in self_reported_ratings else [1950.123456]) + aga_1d = ( + self_reported_ratings["aga"][30] + if "aga" in self_reported_ratings + else [1950.123456] + ) avg_1d_aga = sum(aga_1d) / len(aga_1d) - egf_1d = (self_reported_ratings['egf'][30] if 'egf' in self_reported_ratings else [1950.123456]) + egf_1d = ( + self_reported_ratings["egf"][30] + if "egf" in self_reported_ratings + else [1950.123456] + ) avg_1d_egf = sum(egf_1d) / len(egf_1d) - ratings_1d = ((self_reported_ratings['egf'][30] if 'egf' in self_reported_ratings else [1950.123456]) + - (self_reported_ratings['aga'][30] if 'aga' in self_reported_ratings else [1950.123456])) + ratings_1d = ( + self_reported_ratings["egf"][30] + if "egf" in self_reported_ratings + else [1950.123456] + ) + ( + self_reported_ratings["aga"][30] + if "aga" in self_reported_ratings + else [1950.123456] + ) avg_1d_rating = sum(ratings_1d) / len(ratings_1d) - print("Avg 1d rating egf: %6.1f aga: %6.1f egf+aga: %6.1f" % (avg_1d_egf, avg_1d_aga, avg_1d_rating)) - - - + print( + "Avg 1d rating egf: %6.1f aga: %6.1f egf+aga: %6.1f" + % (avg_1d_egf, avg_1d_aga, avg_1d_rating) + ) diff --git a/analysis/util/InMemoryStorage.py b/analysis/util/InMemoryStorage.py index e070752..413b99e 100644 --- a/analysis/util/InMemoryStorage.py +++ b/analysis/util/InMemoryStorage.py @@ -87,3 +87,6 @@ def get_matches_newer_or_equal_to(self, player_id: int, timestamp: int) -> Any: else: break return [e[1] for e in self._match_history[player_id][-ct:]] + + def finalize(self) -> None: + pass diff --git a/analysis/util/InMemoryWithStartingRankStorage.py b/analysis/util/InMemoryWithStartingRankStorage.py new file mode 100644 index 0000000..b827ff3 --- /dev/null +++ b/analysis/util/InMemoryWithStartingRankStorage.py @@ -0,0 +1,256 @@ +import logging +import os +import json + +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List, Tuple + +from goratings.interfaces import Storage +from .CLI import cli +from analysis.util import config +from analysis.util import rank_to_rating + + +STORE_AFTER_N_GAMES = 10 +#STORE_AFTER_N_GAMES = 30 +INITIAL_DEVIATION = 250 +STORED_RATINGS_FILE = "starting_ratings.json" +#STORED_RATINGS_FILE = "starting_ratings-30.json" + +__all__ = ["InMemoryStorageWithStartingRankStorage"] + + +logger = logging.getLogger(__name__) + +# The starting rank system relies on a two pass system, the first pass we start +# everyone at Glicko 1500 and after a few games we store what their rating is. +# Subsequent runs will then use the stored rating to determine their starting +# rank band. + +cli.add_argument( + "--first-pass", + dest="first_pass", + action="store_true", + help="Compute starting ranks and save them for a second run. This implies *not* loading previous ranks.", +) + +_30k = 0 +_25k = 30 - 25 +_22k = 30 - 22 +_20k = 30 - 20 +_15k = 30 - 15 +_17k = 30 - 17 +_18k = 30 - 18 +_19k = 30 - 19 +_12k = 30 - 12 +_10k = 30 - 10 +_5k = 30 - 5 +_2k = 30 - 2 +_1d = 30 + +# Proposal: +# beginner = 25k +# basic = 22k +# intermediate = 12k +# advanced = 12k + +if False: + _new = _25k + _basic = _22k + _intermediate = _12k + _advanced = _2k +else: + _new = _20k + _basic = _17k + _intermediate = _12k + _advanced = _2k + + +def bin_(rating: float) -> float: + if rating < (rank_to_rating(_new) + rank_to_rating(_basic)) / 2: + return rank_to_rating(_new) + elif rating < (rank_to_rating(_basic) + rank_to_rating(_intermediate)) / 2: + return rank_to_rating(_basic) + elif rating < (rank_to_rating(_intermediate) + rank_to_rating(_advanced)) / 2: + return rank_to_rating(_intermediate) + else: + return rank_to_rating(_advanced) + + +cli.add_argument( + "--new", + dest="new", + type=float, + default=_new, + help="New player starting rank.", +) + +cli.add_argument( + "--basic", + dest="basic", + type=float, + default=_basic, + help="Basic player starting rank.", +) + +cli.add_argument( + "--intermediate", + dest="intermediate", + type=float, + default=_intermediate, + help="Intermediate player starting rank.", +) + +cli.add_argument( + "--advanced", + dest="advanced", + type=float, + default=_advanced, + help="Advanced player starting rank.", +) + + +def bin(rating: float) -> float: + if ( + rating + < (rank_to_rating(config.args.new) + rank_to_rating(config.args.basic)) / 2 + ): + return rank_to_rating(config.args.new) + elif ( + rating + < (rank_to_rating(config.args.basic) + rank_to_rating(config.args.intermediate)) + / 2 + ): + return rank_to_rating(config.args.basic) + elif ( + rating + < ( + rank_to_rating(config.args.intermediate) + + rank_to_rating(config.args.advanced) + ) + / 2 + ): + return rank_to_rating(config.args.intermediate) + else: + return rank_to_rating(config.args.advanced) + + +class InMemoryStorageWithStartingRankStorage(Storage): + _data: Dict[int, Any] + _timeout_flags: DefaultDict[int, bool] + _match_history: DefaultDict[int, List[Tuple[int, Any]]] + _rating_history: DefaultDict[int, List[Tuple[int, Any]]] + _set_count: DefaultDict[int, int] + entry_type: Any + first_pass: bool + _initial_ratings: Dict[int, Any] + + def __init__(self, entry_type: type) -> None: + self.first_pass = config.args.first_pass or not os.path.exists( + STORED_RATINGS_FILE + ) + + if not self.first_pass: + logger.info("Loading ranks from previous run.") + with open(STORED_RATINGS_FILE, "r") as f: + self._initial_ratings = { + int(k): v for k, v in json.loads(f.read()).items() + } + logger.info(f"{len(self._initial_ratings)} initial ratings loaded.") + + if self.first_pass: + logger.info("Starting rank storage is in first pass mode.") + self._initial_ratings = {} + + self._data = {} + self._timeout_flags = defaultdict(lambda: False) + self._match_history = defaultdict(lambda: []) + self._rating_history = defaultdict(lambda: []) + self._set_count = defaultdict(lambda: 0) + self.entry_type = entry_type + + def get(self, player_id: int) -> Any: + if player_id not in self._data: + if player_id in self._initial_ratings: + self._data[player_id] = self.entry_type( + rating=bin(self._initial_ratings[player_id]["rating"]), + deviation=INITIAL_DEVIATION, + ) + else: + self._data[player_id] = self.entry_type() + return self._data[player_id] + + def set(self, player_id: int, entry: Any) -> None: + self._data[player_id] = entry + self._set_count[player_id] += 1 + if self.first_pass and self._set_count[player_id] == STORE_AFTER_N_GAMES: + self._initial_ratings[player_id] = entry + + def clear_set_count(self, player_id: int) -> None: + self._set_count[player_id] = 0 + + def get_set_count(self, player_id: int) -> int: + return self._set_count[player_id] + + def all_players(self) -> Dict[int, Any]: + return self._data + + def get_timeout_flag(self, player_id: int) -> bool: + return self._timeout_flags[player_id] + + def set_timeout_flag(self, player_id: int, tf: bool) -> None: + self._timeout_flags[player_id] = tf + + # We assume we add these entries in ascending order (by timestamp). + def add_rating_history(self, player_id: int, timestamp: int, entry: Any) -> None: + self._rating_history[player_id].append((timestamp, entry)) + + def add_match_history(self, player_id: int, timestamp: int, entry: Any) -> None: + self._match_history[player_id].append((timestamp, entry)) + + def get_last_game_timestamp(self, player_id: int) -> int: + if player_id in self._rating_history: + return self._rating_history[player_id][-1][0] + return 0 + + def get_first_rating_older_than(self, player_id: int, timestamp: int) -> Any: + for e in reversed(self._rating_history[player_id]): + if e[0] < timestamp: + return e[1] + return self.entry_type() + + def get_ratings_newer_or_equal_to(self, player_id: int, timestamp: int) -> Any: + ct = 0 + for e in reversed(self._rating_history[player_id]): + if e[0] >= timestamp: + ct += 1 + else: + break + return [e[1] for e in self._rating_history[player_id][-ct:]] + + def get_first_timestamp_older_than(self, player_id: int, timestamp: int) -> Any: + for e in reversed(self._rating_history[player_id]): + if e[0] < timestamp: + return e[0] + return None + + def get_matches_newer_or_equal_to(self, player_id: int, timestamp: int) -> Any: + ct = 0 + for e in reversed(self._match_history[player_id]): + if e[0] >= timestamp: + ct += 1 + else: + break + return [e[1] for e in self._match_history[player_id][-ct:]] + + def finalize(self) -> None: + if self.first_pass: + logger.info("Saving starting ranks.") + with open(STORED_RATINGS_FILE, "w") as f: + f.write( + json.dumps( + {k: v.to_dict() for k, v in self._initial_ratings.items()}, + indent=2, + ) + ) + logger.info("Starting ranks saved.") diff --git a/analysis/util/TallyGameAnalytics.py b/analysis/util/TallyGameAnalytics.py index afa5fc0..60add98 100644 --- a/analysis/util/TallyGameAnalytics.py +++ b/analysis/util/TallyGameAnalytics.py @@ -17,7 +17,9 @@ from .GameData import datasets_used from .Glicko2Analytics import Glicko2Analytics from .GorAnalytics import GorAnalytics -from .InMemoryStorage import InMemoryStorage + +# from .InMemoryStorage import InMemoryStorage +from goratings.interfaces import Storage from .RatingMath import rating_config, rating_to_rank, get_handicap_rank_difference from .EGFGameData import EGFGameData from .AGAGameData import AGAGameData @@ -30,7 +32,7 @@ ALL: int = 999 EGF_OFFSET = 1000000000 AGA_OFFSET = 2000000000 -LAST_ORG_GAME_PLAYED_CUTOFF = 1559347200 # 2019-06-01 +LAST_ORG_GAME_PLAYED_CUTOFF = 1559347200 # 2019-06-01 MIN_ORG_GAMES_PLAYED_CUTOFF = 6 PROVISIONAL_DEVIATION_CUTOFF = 100 @@ -41,18 +43,26 @@ # rank, or rank+5 for 5 rank bands (the str "0+5", "5+5", "10+5", etc), `ALL` for all # Handicap, 0-9 or `ALL` for all ResultStorageType = DefaultDict[ - int, DefaultDict[int, DefaultDict[Union[int, str], DefaultDict[int, Union[int, float]]]], + int, + DefaultDict[int, DefaultDict[Union[int, str], DefaultDict[int, Union[int, float]]]], ] cli.add_argument( - "--mismatch-threshold-black-wins", dest="mismatch_threshold_black_wins", type=float, default=1.0, + "--mismatch-threshold-black-wins", + dest="mismatch_threshold_black_wins", + type=float, + default=1.0, help="Rank difference threshold for ignoring mismatched games in 'black wins' table", ) cli.add_argument( - "--mismatch-threshold-predictions", dest="mismatch_threshold_predictions", type=float, default=1.0, + "--mismatch-threshold-predictions", + dest="mismatch_threshold_predictions", + type=float, + default=1.0, help="Rank difference threshold for ignoring mismatched games in prediction tables", ) + class TallyGameAnalytics: games_ignored: int black_wins: ResultStorageType @@ -61,34 +71,57 @@ class TallyGameAnalytics: prediction_cost: ResultStorageType count: ResultStorageType count_black_wins: ResultStorageType - storage: InMemoryStorage + storage: Storage prefix: str - def __init__(self, storage: InMemoryStorage, prefix: str = '') -> None: + def __init__(self, storage: Storage, prefix: str = "") -> None: self.prefix = prefix self.games_ignored = 0 self.storage = storage - self.black_wins = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0)))) - self.predictions = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))) - self.predicted_outcome = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))) - self.prediction_cost = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))) - self.count = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0)))) - self.count_black_wins = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0)))) + self.black_wins = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0))) + ) + self.predictions = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))) + ) + self.predicted_outcome = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))) + ) + self.prediction_cost = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))) + ) + self.count = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0))) + ) + self.count_black_wins = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0))) + ) def add_glicko2_analytics(self, result: Glicko2Analytics) -> None: if result.skipped: return - if result.black_deviation > PROVISIONAL_DEVIATION_CUTOFF or result.white_deviation > PROVISIONAL_DEVIATION_CUTOFF: + if ( + result.black_deviation > PROVISIONAL_DEVIATION_CUTOFF + or result.white_deviation > PROVISIONAL_DEVIATION_CUTOFF + ): self.games_ignored += 1 return - handicap_rank_difference = get_handicap_rank_difference(handicap=result.game.handicap, - size=result.game.size, - komi=result.game.komi, - rules=result.game.rules) - effective_rank_difference = abs(result.black_rank + handicap_rank_difference - result.white_rank) - tally_black_wins = effective_rank_difference <= config.args.mismatch_threshold_black_wins - tally_predictions = effective_rank_difference <= config.args.mismatch_threshold_predictions + handicap_rank_difference = get_handicap_rank_difference( + handicap=result.game.handicap, + size=result.game.size, + komi=result.game.komi, + rules=result.game.rules, + ) + effective_rank_difference = abs( + result.black_rank + handicap_rank_difference - result.white_rank + ) + tally_black_wins = ( + effective_rank_difference <= config.args.mismatch_threshold_black_wins + ) + tally_predictions = ( + effective_rank_difference <= config.args.mismatch_threshold_predictions + ) if not tally_black_wins and not tally_predictions: self.games_ignored += 1 return @@ -104,24 +137,39 @@ def add_glicko2_analytics(self, result: Glicko2Analytics) -> None: int(result.black_rank), ]: for handicap in [ALL, result.game.handicap]: - if isinstance(rank, int) or isinstance(rank, str): # this is just to make mypy happy + if isinstance(rank, int) or isinstance( + rank, str + ): # this is just to make mypy happy if tally_black_wins: self.count_black_wins[size][speed][rank][handicap] += 1 if black_won: self.black_wins[size][speed][rank][handicap] += 1 if tally_predictions: - self.predictions[size][speed][rank][handicap] += result.expected_win_rate + self.predictions[size][speed][rank][handicap] += ( + result.expected_win_rate + ) self.predicted_outcome[size][speed][rank][handicap] += ( black_won if result.expected_win_rate > 0.5 - else (not black_won if result.expected_win_rate < 0.5 else 0.5) + else ( + not black_won + if result.expected_win_rate < 0.5 + else 0.5 + ) ) # Cap the expected win rate at 1 in 1M to avoid MathDomain errors. - capped_win_rate = max(min(result.expected_win_rate, 0.999999), 0.000001) - self.prediction_cost[size][speed][rank][handicap] += - math.log(capped_win_rate if black_won else 1 - capped_win_rate) + capped_win_rate = max( + min(result.expected_win_rate, 0.999999), 0.000001 + ) + self.prediction_cost[size][speed][rank][ + handicap + ] += -math.log( + capped_win_rate + if black_won + else 1 - capped_win_rate + ) self.count[size][speed][rank][handicap] += 1 - def add_gor_analytics(self, result: GorAnalytics) -> None: if result.skipped: return @@ -144,10 +192,14 @@ def add_gor_analytics(self, result: GorAnalytics) -> None: int(result.black_rank), ]: for handicap in [ALL, result.game.handicap]: - if isinstance(rank, int) or isinstance(rank, str): # this is just to make mypy happy + if isinstance(rank, int) or isinstance( + rank, str + ): # this is just to make mypy happy if black_won: self.black_wins[size][speed][rank][handicap] += 1 - self.predictions[size][speed][rank][handicap] += result.expected_win_rate + self.predictions[size][speed][rank][handicap] += ( + result.expected_win_rate + ) self.count[size][speed][rank][handicap] += 1 def print(self) -> None: @@ -155,34 +207,34 @@ def print(self) -> None: self.print_handicap_prediction() self.print_handicap_cost() self.print_inspected_players() - #self.print_median_games_per_timewindow() + # self.print_median_games_per_timewindow() self.print_compact_stats() self.print_self_reported_stats() self.update_visualizer_data() def print_compact_stats(self) -> None: - prediction = ( - self.prediction_cost[ALL][ALL][ALL][ALL] / max(1, self.count[ALL][ALL][ALL][ALL]) + prediction = self.prediction_cost[ALL][ALL][ALL][ALL] / max( + 1, self.count[ALL][ALL][ALL][ALL] ) - prediction_h0 = ( - self.prediction_cost[ALL][ALL][ALL][0] / max(1, self.count[ALL][ALL][ALL][0]) + prediction_h0 = self.prediction_cost[ALL][ALL][ALL][0] / max( + 1, self.count[ALL][ALL][ALL][0] ) - prediction_h1 = ( - self.prediction_cost[ALL][ALL][ALL][1] / max(1, self.count[ALL][ALL][ALL][1]) + prediction_h1 = self.prediction_cost[ALL][ALL][ALL][1] / max( + 1, self.count[ALL][ALL][ALL][1] ) - prediction_h2 = ( - self.prediction_cost[ALL][ALL][ALL][2] / max(1, self.count[ALL][ALL][ALL][2]) + prediction_h2 = self.prediction_cost[ALL][ALL][ALL][2] / max( + 1, self.count[ALL][ALL][ALL][2] ) - #unexp_change = ( + # unexp_change = ( # self.unexpected_rank_changes[ALL][ALL][ALL][ALL] / max(1, self.count[ALL][ALL][ALL][ALL]) / 2 - #) + # ) - #print("") - #print("") - #print("| Algorithm name | all | h0 | h1 | h2 | rating changed in the wrong direction |") - #print("|:---------------|--------:|--------:|--------:|--------:|---------------------------------------:") - #print( + # print("") + # print("") + # print("| Algorithm name | all | h0 | h1 | h2 | rating changed in the wrong direction |") + # print("|:---------------|--------:|--------:|--------:|--------:|---------------------------------------:") + # print( # "| {name:>s} | {prediction:>7.5f} | {prediction_h0:>7.5f} " # "| {prediction_h1:>7.5f} | {prediction_h2:>7.5f} | {unexp_change:>8.5%} |".format( print("") @@ -229,11 +281,15 @@ def print_inspected_players(self) -> None: for name in ini[section]: id = int(ini[section][name]) entry = self.storage.get(id) - last_game = self.storage.get_first_timestamp_older_than(id, 999999999999) + last_game = self.storage.get_first_timestamp_older_than( + id, 999999999999 + ) if last_game is None: rh = [] else: - rh = self.storage.get_ratings_newer_or_equal_to(id, last_game - 86400 * 28) + rh = self.storage.get_ratings_newer_or_equal_to( + id, last_game - 86400 * 28 + ) print( "%20s %3s %s %4.0f %4.0f %3.0f %3.0f" % ( @@ -258,9 +314,14 @@ def print_handicap_performance(self) -> None: for size in [9, 13, 19, ALL]: print("") if size == ALL: - print("Overall: %d games" % self.count_black_wins[size][ALL][ALL][ALL]) + print( + "Overall: %d games" % self.count_black_wins[size][ALL][ALL][ALL] + ) else: - print("%dx%d: %d games" % (size, size, self.count_black_wins[size][ALL][ALL][ALL])) + print( + "%dx%d: %d games" + % (size, size, self.count_black_wins[size][ALL][ALL][ALL]) + ) sys.stdout.write(" ") for handicap in range(10): @@ -273,7 +334,15 @@ def print_handicap_performance(self) -> None: for handicap in range(10): ct = self.count_black_wins[size][ALL][rankband][handicap] sys.stdout.write( - "%5.1f%% " % ((self.black_wins[size][ALL][rankband][handicap] / ct if ct else 0) * 100.0) + "%5.1f%% " + % ( + ( + self.black_wins[size][ALL][rankband][handicap] / ct + if ct + else 0 + ) + * 100.0 + ) ) sys.stdout.write("\n") @@ -286,7 +355,9 @@ def print_handicap_prediction(self) -> None: if size == ALL: print("Overall: %d games" % self.count[size][ALL][ALL][ALL]) else: - print("%dx%d: %d games" % (size, size, self.count[size][ALL][ALL][ALL])) + print( + "%dx%d: %d games" % (size, size, self.count[size][ALL][ALL][ALL]) + ) sys.stdout.write(" ") for handicap in range(10): @@ -300,7 +371,15 @@ def print_handicap_prediction(self) -> None: ct = self.count[size][ALL][rankband][handicap] sys.stdout.write( "%5.1f%% " - % ((self.predicted_outcome[size][ALL][rankband][handicap] / ct if ct else 0) * 100.0) + % ( + ( + self.predicted_outcome[size][ALL][rankband][handicap] + / ct + if ct + else 0 + ) + * 100.0 + ) ) sys.stdout.write("\n") @@ -308,88 +387,89 @@ def print_self_reported_stats(self) -> None: stats = self.get_self_reported_stats() if not stats: return - print('') - print('') + print("") + print("") BAND_WIDTH = 3 - header = ' ' - line = defaultdict(lambda: '') + header = " " + line = defaultdict(lambda: "") for band in range(0, 40, BAND_WIDTH): - header += '%-4s - %-4s\t' % (num2rank(band), num2rank(band + (BAND_WIDTH - 1))) + header += "%-4s - %-4s\t" % ( + num2rank(band), + num2rank(band + (BAND_WIDTH - 1)), + ) for key in stats.keys(): - line[key] = '%8s [%3d]:\t' % (key, len([item for sublist in stats[key] for item in sublist])) + line[key] = "%8s [%3d]:\t" % ( + key, + len([item for sublist in stats[key] for item in sublist]), + ) for band in range(0, 40, BAND_WIDTH): - flat = [item for sublist in stats[key][band:band + BAND_WIDTH] for item in sublist] + flat = [ + item + for sublist in stats[key][band : band + BAND_WIDTH] + for item in sublist + ] if len(flat): avg = mean(flat) size = len(flat) - line[key] += '%4.1f [%2d] \t' % (avg, size) + line[key] += "%4.1f [%2d] \t" % (avg, size) else: - line[key] += ' \t' + line[key] += " \t" print(header) for key in stats.keys(): print(line[key]) - - - - - - def get_self_reported_stats(self) -> Dict[str, Dict[int, List[float]]]: datasets = datasets_used() if not datasets["ogs"]: return - - if os.path.exists('./data'): - pathname = './data/' - elif os.path.exists('../data'): - pathname = '../data/' + if os.path.exists("./data"): + pathname = "./data/" + elif os.path.exists("../data"): + pathname = "../data/" else: - raise Exception('Failed to find data directory') + raise Exception("Failed to find data directory") - if os.path.exists(pathname + 'self_reported_account_links.full.json'): - pathname += 'self_reported_account_links.full.json' - elif os.path.exists(pathname + 'self_reported_account_links.json'): - pathname = 'self_reported_account_links.json' + if os.path.exists(pathname + "self_reported_account_links.full.json"): + pathname += "self_reported_account_links.full.json" + elif os.path.exists(pathname + "self_reported_account_links.json"): + pathname = "self_reported_account_links.json" else: - raise Exception('Failed to find self_reported_account_links json file') - + raise Exception("Failed to find self_reported_account_links json file") - with open(pathname, 'r') as f: + with open(pathname, "r") as f: stats = json.loads(f.read()) def get_org_rank(entry, org_country): - for org in ['org1', 'org2', 'org3']: + for org in ["org1", "org2", "org3"]: if org in entry and entry[org] == org_country: - if org + '_rank' in entry: - return entry[org + '_rank'] + if org + "_rank" in entry: + return entry[org + "_rank"] return None def get_org_id(entry, org_country): - for org in ['org1', 'org2', 'org3']: + for org in ["org1", "org2", "org3"]: if org in entry and entry[org] == org_country: - if org + '_id' in entry: + if org + "_id" in entry: try: - return int(entry[org + '_id']) + return int(entry[org + "_id"]) except: return None return None def date(timestamp) -> str: - return ctime(timestamp) if timestamp else '' - + return ctime(timestamp) if timestamp else "" egf_count = 0 aga_count = 0 - bins = defaultdict(lambda: defaultdict(lambda: list())) + bins = defaultdict(lambda: defaultdict(lambda: list())) for e in stats: id = e[0] @@ -398,33 +478,50 @@ def date(timestamp) -> str: player = self.storage.get(id) rank = rating_to_rank(player.rating) - aga = get_org_rank(entry, 'us') - egf = get_org_rank(entry, 'eu') - aga_id = get_org_id(entry, 'us') - egf_id = get_org_id(entry, 'eu') + aga = get_org_rank(entry, "us") + egf = get_org_rank(entry, "eu") + aga_id = get_org_id(entry, "us") + egf_id = get_org_id(entry, "eu") if (aga and aga > 100) or (egf and egf > 100): - pass # throwout pros for our purposes - + pass # throwout pros for our purposes - aga_num_games_played = agadb.num_games_played(aga_id + AGA_OFFSET) if aga_id else 0 - aga_last_game_played = agadb.last_game_played(aga_id + AGA_OFFSET) if aga_id else 0 - egf_num_games_played = egfdb.num_games_played(egf_id + EGF_OFFSET) if egf_id else 0 - egf_last_game_played = egfdb.last_game_played(egf_id + EGF_OFFSET) if egf_id else 0 + aga_num_games_played = ( + agadb.num_games_played(aga_id + AGA_OFFSET) if aga_id else 0 + ) + aga_last_game_played = ( + agadb.last_game_played(aga_id + AGA_OFFSET) if aga_id else 0 + ) + egf_num_games_played = ( + egfdb.num_games_played(egf_id + EGF_OFFSET) if egf_id else 0 + ) + egf_last_game_played = ( + egfdb.last_game_played(egf_id + EGF_OFFSET) if egf_id else 0 + ) jan_2019 = 1546300800 - #if aga and aga_last_game_played > 0: + # if aga and aga_last_game_played > 0: if aga and aga_last_game_played > jan_2019 and aga_num_games_played > 5: - bins['aga'][aga].append(rank - aga) + bins["aga"][aga].append(rank - aga) - #if egf and egf_last_game_played > 0: + # if egf and egf_last_game_played > 0: if egf and egf_last_game_played > jan_2019 and egf_num_games_played > 5: - bins['egf'][egf].append(rank - egf) - - - for server in ['dgs', 'fox', 'kgs', 'igs', 'fox', 'yike', 'golem', 'tygem', 'goquest', 'wbaduk']: - if ('%s_rank' % server) in entry: - server_rank = int(entry[('%s_rank' % server)]) + bins["egf"][egf].append(rank - egf) + + for server in [ + "dgs", + "fox", + "kgs", + "igs", + "fox", + "yike", + "golem", + "tygem", + "goquest", + "wbaduk", + ]: + if ("%s_rank" % server) in entry: + server_rank = int(entry[("%s_rank" % server)]) bins[server][server_rank].append(rank - server_rank) ret = {} @@ -439,57 +536,53 @@ def date(timestamp) -> str: return ret - def get_self_reported_rating(self) -> Dict[str, Dict[int, List[float]]]: datasets = datasets_used() if not datasets["ogs"]: return - - if os.path.exists('./data'): - pathname = './data/' - elif os.path.exists('../data'): - pathname = '../data/' + if os.path.exists("./data"): + pathname = "./data/" + elif os.path.exists("../data"): + pathname = "../data/" else: - raise Exception('Failed to find data directory') + raise Exception("Failed to find data directory") - if os.path.exists(pathname + 'self_reported_account_links.full.json'): - pathname += 'self_reported_account_links.full.json' - elif os.path.exists(pathname + 'self_reported_account_links.json'): - pathname = 'self_reported_account_links.json' + if os.path.exists(pathname + "self_reported_account_links.full.json"): + pathname += "self_reported_account_links.full.json" + elif os.path.exists(pathname + "self_reported_account_links.json"): + pathname = "self_reported_account_links.json" else: - raise Exception('Failed to find self_reported_account_links json file') - + raise Exception("Failed to find self_reported_account_links json file") - with open(pathname, 'r') as f: + with open(pathname, "r") as f: stats = json.loads(f.read()) def get_org_rank(entry, org_country): - for org in ['org1', 'org2', 'org3']: + for org in ["org1", "org2", "org3"]: if org in entry and entry[org] == org_country: - if org + '_rank' in entry: - return entry[org + '_rank'] + if org + "_rank" in entry: + return entry[org + "_rank"] return None def get_org_id(entry, org_country): - for org in ['org1', 'org2', 'org3']: + for org in ["org1", "org2", "org3"]: if org in entry and entry[org] == org_country: - if org + '_id' in entry: + if org + "_id" in entry: try: - return int(entry[org + '_id']) + return int(entry[org + "_id"]) except: return None return None def date(timestamp) -> str: - return ctime(timestamp) if timestamp else '' - + return ctime(timestamp) if timestamp else "" egf_count = 0 aga_count = 0 - bins = defaultdict(lambda: defaultdict(lambda: list())) + bins = defaultdict(lambda: defaultdict(lambda: list())) for e in stats: id = e[0] @@ -500,33 +593,50 @@ def date(timestamp) -> str: if player.rating == 1500: continue - aga = get_org_rank(entry, 'us') - egf = get_org_rank(entry, 'eu') - aga_id = get_org_id(entry, 'us') - egf_id = get_org_id(entry, 'eu') + aga = get_org_rank(entry, "us") + egf = get_org_rank(entry, "eu") + aga_id = get_org_id(entry, "us") + egf_id = get_org_id(entry, "eu") if (aga and aga > 100) or (egf and egf > 100): - continue # throwout pros for our purposes - + continue # throwout pros for our purposes - aga_num_games_played = agadb.num_games_played(aga_id + AGA_OFFSET) if aga_id else 0 - aga_last_game_played = agadb.last_game_played(aga_id + AGA_OFFSET) if aga_id else 0 - egf_num_games_played = egfdb.num_games_played(egf_id + EGF_OFFSET) if egf_id else 0 - egf_last_game_played = egfdb.last_game_played(egf_id + EGF_OFFSET) if egf_id else 0 + aga_num_games_played = ( + agadb.num_games_played(aga_id + AGA_OFFSET) if aga_id else 0 + ) + aga_last_game_played = ( + agadb.last_game_played(aga_id + AGA_OFFSET) if aga_id else 0 + ) + egf_num_games_played = ( + egfdb.num_games_played(egf_id + EGF_OFFSET) if egf_id else 0 + ) + egf_last_game_played = ( + egfdb.last_game_played(egf_id + EGF_OFFSET) if egf_id else 0 + ) jan_2019 = 1546300800 - #if aga and aga_last_game_played > 0: + # if aga and aga_last_game_played > 0: if aga and aga_last_game_played > jan_2019 and aga_num_games_played > 5: - bins['aga'][aga].append(player.rating) + bins["aga"][aga].append(player.rating) - #if egf and egf_last_game_played > 0: + # if egf and egf_last_game_played > 0: if egf and egf_last_game_played > jan_2019 and egf_num_games_played > 5: - bins['egf'][egf].append(player.rating) - - - for server in ['dgs', 'fox', 'kgs', 'igs', 'fox', 'yike', 'golem', 'tygem', 'goquest', 'wbaduk']: - if ('%s_rank' % server) in entry: - server_rank = int(entry[('%s_rank' % server)]) + bins["egf"][egf].append(player.rating) + + for server in [ + "dgs", + "fox", + "kgs", + "igs", + "fox", + "yike", + "golem", + "tygem", + "goquest", + "wbaduk", + ]: + if ("%s_rank" % server) in entry: + server_rank = int(entry[("%s_rank" % server)]) bins[server][server_rank].append(player.rating) ret = {} @@ -550,7 +660,9 @@ def print_handicap_cost(self) -> None: if size == ALL: print("Overall: %d games" % self.count[size][ALL][ALL][ALL]) else: - print("%dx%d: %d games" % (size, size, self.count[size][ALL][ALL][ALL])) + print( + "%dx%d: %d games" % (size, size, self.count[size][ALL][ALL][ALL]) + ) sys.stdout.write(" ") for handicap in range(10): @@ -564,7 +676,10 @@ def print_handicap_cost(self) -> None: ct = self.count[size][ALL][rankband][handicap] sys.stdout.write( "%5.3f " - % (self.prediction_cost[size][ALL][rankband][handicap] / max(1,ct)) + % ( + self.prediction_cost[size][ALL][rankband][handicap] + / max(1, ct) + ) ) sys.stdout.write("\n") @@ -606,7 +721,7 @@ def get_descriptive_name(self) -> str: if "p" in cfg["rating_config"]: lst.append(str(cfg["rating_config"]["p"])) - return self.prefix + '-' + (":".join(lst)) + return self.prefix + "-" + (":".join(lst)) def get_visualizer_data(self) -> Any: obj: Any = {} diff --git a/goratings/interfaces/Storage.py b/goratings/interfaces/Storage.py index 5692dc9..0122bc2 100644 --- a/goratings/interfaces/Storage.py +++ b/goratings/interfaces/Storage.py @@ -32,3 +32,7 @@ def get_timeout_flag(self, player_id: int) -> bool: @abc.abstractmethod def set_timeout_flag(self, player_id: int, tf: bool) -> None: raise NotImplementedError + + @abc.abstractmethod + def finalize(self) -> None: + raise NotImplementedError diff --git a/goratings/math/glicko2.py b/goratings/math/glicko2.py index 856b3fc..fd41e9c 100644 --- a/goratings/math/glicko2.py +++ b/goratings/math/glicko2.py @@ -27,9 +27,15 @@ class Glicko2Entry: volatility: float mu: float phi: float - timestamp: int - - def __init__(self, rating: float = 1500, deviation: float = 350, volatility: float = 0.06, timestamp: int | None = None) -> None: + timestamp: int | None + + def __init__( + self, + rating: float = 1500, + deviation: float = 350, + volatility: float = 0.06, + timestamp: int | None = None, + ) -> None: self.rating = rating self.deviation = deviation self.volatility = volatility @@ -46,20 +52,34 @@ def __str__(self) -> str: 0 if self.timestamp is None else self.timestamp, ) - def copy(self, rating_adjustment: float = 0.0, rd_adjustment: float = 0.0) -> "Glicko2Entry": - ret = Glicko2Entry(self.rating + rating_adjustment, self.deviation + rd_adjustment, self.volatility, self.timestamp,) + def copy( + self, rating_adjustment: float = 0.0, rd_adjustment: float = 0.0 + ) -> "Glicko2Entry": + ret = Glicko2Entry( + self.rating + rating_adjustment, + self.deviation + rd_adjustment, + self.volatility, + self.timestamp, + ) return ret - def expand_deviation_because_no_games_played(self, n_periods: int = 1) -> "Glicko2Entry": + def expand_deviation_because_no_games_played( + self, n_periods: int = 1 + ) -> "Glicko2Entry": return self.expand_deviation(age=n_periods, override_aging_period=1) - def after_aging_to_timestamp(self, timestamp: int | None, minus_one_period: bool = False) -> "Glicko2Entry": + def after_aging_to_timestamp( + self, timestamp: int | None, minus_one_period: bool = False + ) -> "Glicko2Entry": global AGING_PERIOD_SECONDS # Create copy with the new timestamp and expand the deviation if the # timestamp is moving forward. if timestamp and AGING_PERIOD_SECONDS and minus_one_period: - timestamp = max(self.timestamp if self.timestamp else 0, timestamp - AGING_PERIOD_SECONDS) + timestamp = max( + self.timestamp if self.timestamp else 0, + timestamp - AGING_PERIOD_SECONDS, + ) copy = self.copy() copy.timestamp = timestamp @@ -67,7 +87,9 @@ def after_aging_to_timestamp(self, timestamp: int | None, minus_one_period: bool copy.expand_deviation(age=copy.timestamp - self.timestamp) return copy - def expand_deviation(self, age: int, override_aging_period: int | None = None) -> "Glicko2Entry": + def expand_deviation( + self, age: int, override_aging_period: int | None = None + ) -> "Glicko2Entry": # Implementation as defined by [glicko2], but converted to closed form, # allowing deviation to expand continuously over fractional periods. # @@ -81,20 +103,43 @@ def expand_deviation(self, age: int, override_aging_period: int | None = None) - return self - def expected_win_probability(self, white: "Glicko2Entry", handicap_adjustment: float, ignore_g: bool = False) -> float: + def expected_win_probability( + self, white: "Glicko2Entry", handicap_adjustment: float, ignore_g: bool = False + ) -> float: # Implementation extracted from glicko2_update below. if not ignore_g: + def g() -> float: return 1 else: - def g() -> float: - return 1 / sqrt(1 + (3 * white.phi ** 2) / (pi ** 2)) - E = 1 / (1 + exp(-g() * (self.rating + handicap_adjustment - white.rating) / GLICKO2_SCALE)) + def g() -> float: + return 1 / sqrt(1 + (3 * white.phi**2) / (pi**2)) + + E = 1 / ( + 1 + + exp( + -g() + * (self.rating + handicap_adjustment - white.rating) + / GLICKO2_SCALE + ) + ) return E + def to_dict(self) -> dict[str, float]: + return { + "rating": round(self.rating, 2), + "deviation": round(self.deviation, 2), + "volatility": round(self.volatility, 6), + } + -def _age_phi(phi: float, volatility: float, age: int | None, override_aging_period: int | None = None) -> float: +def _age_phi( + phi: float, + volatility: float, + age: int | None, + override_aging_period: int | None = None, +) -> float: # Implementation as defined by [glicko2], but converted to closed form, # allowing deviation to expand continuously over fractional periods. # @@ -104,24 +149,30 @@ def _age_phi(phi: float, volatility: float, age: int | None, override_aging_peri aging_factor = 1 if age is not None: assert age >= 0 - aging_period = AGING_PERIOD_SECONDS if override_aging_period is None else override_aging_period + aging_period = ( + AGING_PERIOD_SECONDS + if override_aging_period is None + else override_aging_period + ) if aging_period: assert aging_period >= 0 aging_factor = float(age) / aging_period - return sqrt(phi ** 2 + aging_factor * volatility ** 2) + return sqrt(phi**2 + aging_factor * volatility**2) -def glicko2_update(player: Glicko2Entry, matches: List[Tuple[Glicko2Entry, int]], - timestamp: int | None = None) -> Glicko2Entry: +def glicko2_update( + player: Glicko2Entry, + matches: List[Tuple[Glicko2Entry, int]], + timestamp: int | None = None, +) -> Glicko2Entry: # Implementation as defined by: http://www.glicko.net/glicko/glicko2.pdf if len(matches) == 0: return player.after_aging_to_timestamp(timestamp) # Expand the deviation due to inactivity, in case the last game was more # than a period ago. - player = player.after_aging_to_timestamp(timestamp, minus_one_period=True); - + player = player.after_aging_to_timestamp(timestamp, minus_one_period=True) # step 1/2 implicitly done during Glicko2Entry construction # step 3 / 4, compute 'v' and delta @@ -130,26 +181,28 @@ def glicko2_update(player: Glicko2Entry, matches: List[Tuple[Glicko2Entry, int]] for m in matches: p = m[0].after_aging_to_timestamp(timestamp, minus_one_period=True) outcome = m[1] - g_phi_j = 1 / sqrt(1 + (3 * p.phi ** 2) / (pi ** 2)) + g_phi_j = 1 / sqrt(1 + (3 * p.phi**2) / (pi**2)) E = 1 / (1 + exp(-g_phi_j * (player.mu - p.mu))) - v_sum += g_phi_j ** 2 * E * (1 - E) + v_sum += g_phi_j**2 * E * (1 - E) delta_sum += g_phi_j * (outcome - E) v = 1.0 / v_sum if v_sum else 9999 delta = v * delta_sum # step 5 - a = log(player.volatility ** 2) + a = log(player.volatility**2) def f(x: float) -> float: ex = exp(x) - return (ex * (delta ** 2 - player.phi ** 2 - v - ex) / (2 * ((player.phi ** 2 + v + ex) ** 2))) - ( - (x - a) / (TAO ** 2) - ) + return ( + ex + * (delta**2 - player.phi**2 - v - ex) + / (2 * ((player.phi**2 + v + ex) ** 2)) + ) - ((x - a) / (TAO**2)) A = a - if delta ** 2 > player.phi ** 2 + v: - B = log(delta ** 2 - player.phi ** 2 - v) + if delta**2 > player.phi**2 + v: + B = log(delta**2 - player.phi**2 - v) else: k = 1 safety = 100 @@ -178,11 +231,11 @@ def f(x: float) -> float: new_volatility = exp(A / 2) # step 6 - phi_star = sqrt(player.phi ** 2 + new_volatility ** 2) + phi_star = sqrt(player.phi**2 + new_volatility**2) # step 7 - phi_prime = 1 / sqrt(1 / phi_star ** 2 + 1 / v) - mu_prime = player.mu + (phi_prime ** 2) * delta_sum + phi_prime = 1 / sqrt(1 / phi_star**2 + 1 / v) + mu_prime = player.mu + (phi_prime**2) * delta_sum # step 8 ret = Glicko2Entry( @@ -194,7 +247,9 @@ def f(x: float) -> float: return ret -def glicko2_configure(tao: float, min_rd: float, max_rd: float, aging_period_days: float) -> None: +def glicko2_configure( + tao: float, min_rd: float, max_rd: float, aging_period_days: float +) -> None: global TAO global MIN_RD global MAX_RD @@ -203,4 +258,6 @@ def glicko2_configure(tao: float, min_rd: float, max_rd: float, aging_period_day TAO = tao MIN_RD = min_rd MAX_RD = max_rd - AGING_PERIOD_SECONDS = int(aging_period_days * 24 * 60 * 60) if aging_period_days else None + AGING_PERIOD_SECONDS = ( + int(aging_period_days * 24 * 60 * 60) if aging_period_days else None + )