diff --git a/README.md b/README.md index 63f4fc0..ed22d77 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ but has since grown to include a wide range of features, including: | ![screenshot](img/screenshot_analyze.png) | ![screenshot](img/screenshot_play.png) | ## Quickstart + * You can right-click most button or checkbox labels to get a tooltip with help. * To analyze a game, load it using the button in the bottom right, or press `ctrl-L` * To play against AI, pick an AI from the drop down a color and either 'human' or 'teach' for yourself and start playing. @@ -28,8 +29,8 @@ but has since grown to include a wide range of features, including: ### Quick Installation for Windows users -* Currently not available for this pre-release, will be added soon. - + * See the [releases tab](https://github.com/sanderland/katrain/releases) for pre-built installers. + ### Installation from source for Windows users * Download the repository by clicking the green *Clone or download* on this page and *Download zip*. Extract the contents. @@ -53,7 +54,7 @@ Under the 'play' tab you can select who is playing black and white. * Human is simple play with potential feedback, but without auto-undo. * Teach will give you instant feedback, and auto-undo bad moves to give you a second chance. * Settings for this mode can be found under 'Configure Teacher' -* AI will activate the AI in the dropdown next to the buttons. +* AI will activate the AI in the dropdown menu next to the buttons. * Settings for all AIs can be found under 'Configure AIs' If you do not want to see 'Points lost' or other feedback for your moves, @@ -70,8 +71,6 @@ The dots indicate how many points were lost by that move. In short, if you are a weaker player you should mostly on large dots that are red or purple, while stronger players can pay more attention to smaller mistakes. - - #### AIs Available AIs, with strength indicating an estimate for the default settings, are: diff --git a/ai_performance.pickle b/ai_performance.pickle deleted file mode 100644 index 53352ef..0000000 Binary files a/ai_performance.pickle and /dev/null differ diff --git a/bots/BOTS.md b/bots/BOTS.md index e860de1..aa63724 100644 --- a/bots/BOTS.md +++ b/bots/BOTS.md @@ -1,2 +1,3 @@ # Bots -This directory contains the source code used to run the katrain-* bots on OGS. +This directory contains the source code used to run the katrain-* bots on OGS, + and some code to test them. It is liable to break without warning. diff --git a/bots/ai_performance.pickle b/bots/ai_performance.pickle new file mode 100644 index 0000000..32e6b2b Binary files /dev/null and b/bots/ai_performance.pickle differ diff --git a/bots/selfplay.py b/bots/selfplay.py new file mode 100644 index 0000000..3375587 --- /dev/null +++ b/bots/selfplay.py @@ -0,0 +1,190 @@ +# This is a script I use to test the performance of AIs +import pickle +import sys +import threading +import time +import traceback +from collections import defaultdict +from concurrent.futures.thread import ThreadPoolExecutor + +from ai import ai_move +from common import OUTPUT_DEBUG, OUTPUT_ERROR, OUTPUT_INFO +from elote import EloCompetitor +from engine import KataGoEngine +from game import Game +import json + +DB_FILENAME = "bots/ai_performance.pickle" + + +class Logger: + def log(self, msg, level): + if level <= OUTPUT_INFO: + print(msg) + if level <= OUTPUT_ERROR: + print(msg, file=sys.stderr) + + +logger = Logger() + +with open("config.json") as f: + settings = json.load(f) + DEFAULT_AI_SETTINGS = settings["ai"] + + +class AI: + DEFAULT_ENGINE_SETTINGS = { + "katago": "KataGo/katago-bs", + "model": "KataGo/models/b15-1.3.2.txt.gz", + "config": "KataGo/analysis_config.cfg", + "max_visits": 1, + "max_time": 300.0, + "_enable_ownership": False, + } + NUM_THREADS = 8 + IGNORE_SETTINGS_IN_TAG = {"threads", "_enable_ownership", "katago"} # katago for switching from/to bs version + ENGINES = [] + LOCK = threading.Lock() + + def __init__(self, strategy, ai_settings, engine_settings=None): + self.elo_comp = EloCompetitor(initial_rating=1000) + self.strategy = strategy + self.ai_settings = ai_settings + self.engine_settings = engine_settings or {} + fmt_settings = [f"{k}={v}" for k, v in {**self.ai_settings, **self.engine_settings}.items() if k not in AI.IGNORE_SETTINGS_IN_TAG] + self.name = f"{strategy}({ ','.join(fmt_settings) })" + self.fix_settings() + + def fix_settings(self): + self.ai_settings = {**DEFAULT_AI_SETTINGS[self.strategy], **self.ai_settings} + self.engine_settings = {**AI.DEFAULT_ENGINE_SETTINGS, **self.engine_settings, "threads": AI.NUM_THREADS} + + def get_engine(self): # factory + with AI.LOCK: + for existing_engine_settings, engine in AI.ENGINES: + if existing_engine_settings == self.engine_settings: + return engine + engine = KataGoEngine(logger, self.engine_settings) + AI.ENGINES.append((self.engine_settings, engine)) + print("Creating new engine for", self.engine_settings, "now have", len(AI.ENGINES), "engines up") + return engine + + def __eq__(self, other): + return self.name == other.name # should capture all relevant setting differences + + +try: + with open(DB_FILENAME, "rb") as f: + ai_database, all_results = pickle.load(f) + for ai in ai_database: + ai.fix_settings() # update as required +except FileNotFoundError: + ai_database = [] + all_results = [] + + +def add_ai(ai): + if ai not in ai_database: + ai_database.append(ai) + print(f"Adding {ai.name}") + else: + print(f"AI {ai.name} already in DB") + + +def retrieve_ais(selected_ais): + return [ai for ai in ai_database if ai in selected_ais] + + +test_ais = [ + # AI("Jigo", {}, {"max_visits": 100}), + AI("Policy", {}), + AI("P:Local", {}), + AI("P:Pick", {}), + AI("P:Noise", {}), + AI("P:Tenuki", {}), + AI("P:Local", {}), + AI("P:Influence", {}), + AI("P:Territory", {}), +] + +for ai in test_ais: + add_ai(ai) + +N_GAMES = 25 + +ais_to_test = retrieve_ais(test_ais) + +results = defaultdict(list) + + +def play_games(black: AI, white: AI, n: int = N_GAMES): + players = {"B": black, "W": white} + engines = {"B": black.get_engine(), "W": white.get_engine()} + tag = f"{black.name} vs {white.name}" + try: + for i in range(n): + game = Game(Logger(), engines, {}) + game.root.add_list_property("PW", [white.name]) + game.root.add_list_property("PB", [black.name]) + start_time = time.time() + while not game.ended: + p = game.current_node.next_player + move = ai_move(game, players[p].strategy, players[p].ai_settings) + while not game.current_node.analysis_ready: + time.sleep(0.001) + game.game_id += f"_{game.current_node.format_score()}" + print(f"{tag}\tGame {i+1} finished in {time.time()-start_time:.1f}s {game.current_node.format_score()} -> {game.write_sgf('sgf_selfplay/')}", file=sys.stderr) + score = game.current_node.score + if score > 0.3: + black.elo_comp.beat(white.elo_comp) + elif score > -0.3: + black.elo_comp.tied(white.elo_comp) + + results[tag].append(score) + all_results.append((black.name, white.name, score)) + + with open("bots/tmp.pickle", "wb") as f: + pickle.dump((ai_database, all_results), f) + except Exception as e: + print(f"Exception in playing {tag}: {e}") + print(f"Exception in playing {tag}: {e}", file=sys.stderr) + traceback.print_exc() + traceback.print_exc(file=sys.stderr) + + +def fmt_score(score): + return f"{'B' if score >= 0 else 'W'}+{abs(score):.1f}" + + +print(len(ais_to_test), "ais to test") +global_start = time.time() + +with ThreadPoolExecutor(max_workers=16) as threadpool: + for b in ais_to_test: + for w in ais_to_test: + if b is not w: + threadpool.submit(play_games, b, w) + +print("POOL EXIT") + +print("---- RESULTS ----") +for k, v in results.items(): + b_win = sum([s > 0.3 for s in v]) + w_win = sum([s < -0.3 for s in v]) + print(f"{b_win} {k} {w_win} : {list(map(fmt_score,v))}") + +print("---- ELO ----") +for ai in sorted(ai_database, key=lambda a: -a.elo_comp.rating): + wins = [(b, w, s) for (b, w, s) in all_results if s > 0.3 and b == ai.name or w == ai.name and s < -0.3] + losses = [(b, w, s) for (b, w, s) in all_results if s < -0.3 and b == ai.name or w == ai.name and s > -0.3] + draws = [(b, w, s) for (b, w, s) in all_results if -0.3 <= s <= 0.3 and (b == ai.name or w == ai.name)] + out = f"{'*' if ai in ais_to_test else ' '} {ai.name}: ELO {ai.elo_comp.rating:.1f} WINS {len(wins)} LOSSES {len(losses)} DRAWS {len(draws)}" + # print("Wins:",wins) + print(out) + print(out, file=sys.stderr) + +with open(DB_FILENAME, "wb") as f: + pickle.dump((ai_database, all_results), f) + +print(f"Done! saving {len(all_results)} to pickle", file=sys.stderr) +print(f"Time taken {time.time()-global_start:.1f}s", file=sys.stderr) diff --git a/selfplay.py b/selfplay.py deleted file mode 100644 index acd395c..0000000 --- a/selfplay.py +++ /dev/null @@ -1,279 +0,0 @@ -# This is a script I use to test the performance of AIs -import pickle -import sys -import threading -import time -import traceback -from collections import defaultdict -from concurrent.futures.thread import ThreadPoolExecutor - -from ai import ai_move -from common import OUTPUT_DEBUG, OUTPUT_ERROR, OUTPUT_INFO -from elote import EloCompetitor -from engine import KataGoEngine -from game import Game - -DB_FILENAME = "ai_performance.pickle" - - -class Logger: - def log(self, msg, level): - if level <= OUTPUT_INFO: - print(msg) - if level <= OUTPUT_ERROR: - print(msg, file=sys.stderr) - - -logger = Logger() - - -class AI: - DEFAULT_ENGINE_SETTINGS = { - "katago": "KataGo/katago-bs", - "model": " models/b15-1.3.2.txt.gz", - "config": "KataGo/analysis_config.cfg", - "max_visits": 1, - "max_time": 300.0, - "enable_ownership": False, - } - NUM_THREADS = 8 - DEFAULT_SETTINGS = { - "target_score": 2, - "random_loss": 1, - "max_loss": 5, - "min_visits": 20, - "noise_strength": 0.8, - "pick_n": 10, - "pick_frac": 0.2, - "local_stddev": 10, - "line_weight": 10, - "pick_override": 0.95, - } - IGNORE_SETTINGS_IN_TAG = {"threads", "enable_ownership", "katago"} # katago for switching from/to bs version - ENGINES = [] - LOCK = threading.Lock() - - def __init__(self, strategy, ai_settings, engine_settings={}): - self.elo_comp = EloCompetitor(initial_rating=1000) - self.strategy = strategy - self.ai_settings = ai_settings - self.engine_settings = engine_settings - fmt_settings = [f"{k}={v}" for k, v in {**ai_settings, **engine_settings}.items() if k not in AI.IGNORE_SETTINGS_IN_TAG] - self.name = f"{strategy}({ ','.join(fmt_settings) })" - self.fix_settings() - - def fix_settings(self): - self.ai_settings = {**AI.DEFAULT_SETTINGS, **self.ai_settings} - self.engine_settings = {**AI.DEFAULT_ENGINE_SETTINGS, **self.engine_settings} - self.engine_settings["threads"] = AI.NUM_THREADS - - def get_engine(self): # factory - with AI.LOCK: - for existing_engine_settings, engine in AI.ENGINES: - if existing_engine_settings == self.engine_settings: - return engine - engine = KataGoEngine(logger, self.engine_settings) - AI.ENGINES.append((self.engine_settings, engine)) - print("Creating new engine for", self.engine_settings, "now have", len(AI.ENGINES), "engines up") - return engine - - def __eq__(self, other): - return self.name == other.name # should capture all relevant setting differences - - -try: - with open(DB_FILENAME, "rb") as f: - ai_database, all_results = pickle.load(f) - for ai in ai_database: - ai.fix_settings() # update as required -except FileNotFoundError: - ai_database = [] - all_results = [] - - -def add_ai(ai): - if ai not in ai_database: - ai_database.append(ai) - print(f"Adding {ai.name}") - else: - print(f"AI {ai.name} already in DB") - - -def retrieve_ais(selected_ais): - return [ai for ai in ai_database if ai in selected_ais] - - -test_ais = [ - AI("P:Local", {"local_stddev": 1, "pick_frac": 0.1}), - AI("P:Local", {"local_stddev": 1, "pick_frac": 0.05}), - AI("P:Local", {"local_stddev": 5}), - AI("P:Pick", {"pick_frac": 0.4, "pick_n": 20}), - AI("P:Noise", {"noise_strength": 0.8}), - AI("P:Noise", {"noise_strength": 0.9}), - AI("P:Tenuki", {"local_stddev": 1}), - AI("P:Tenuki", {"local_stddev": 5}), - AI("P:Tenuki", {"local_stddev": 10}), - AI("P:Pick", {"pick_frac": 0.2, "pick_n": 10}), - AI("P:Pick", {"pick_frac": 0.3, "pick_n": 10}), - AI("P:Local", {"local_stddev": 10}), - AI("P:Local", {"local_stddev": 5}), - AI("P:Pick", {"pick_frac": 0.0, "pick_n": 1}), - AI("Jigo", {}, {"max_visits": 50}), - AI("Policy", {}), - # AI("Policy", {},{'model':'models/g170-b40c256x2-s2990766336-d830712531.bin.gz'}), -] - -test_ais = [ - AI("Policy", {}), - AI("P:Noise", {"noise_strength": 0.6}), - AI("P:Noise", {"noise_strength": 0.7}), - AI("P:Noise", {"noise_strength": 0.8}), - AI("P:Noise", {"noise_strength": 0.9}), - AI("P:Pick", {}), - AI("P:Pick", {"pick_frac": 0.3, "pick_n": 20}), - AI("P:Pick", {"pick_frac": 0.5, "pick_n": 0}), - AI("P:Influence", {"pick_frac": 0.2, "pick_n": 20}), - AI("P:Territory", {"pick_frac": 0.2, "pick_n": 20}), - AI("P:Influence", {"pick_frac": 0.33, "line_weight": 20}), - AI("P:Territory", {"pick_frac": 0.33, "line_weight": 20}), - AI("P:Pick", {"pick_frac": 0.0, "pick_n": 1}), - AI("P:Tenuki", {"local_stddev": 20}), - AI("P:Tenuki", {"local_stddev": 10}), - AI("P:Tenuki", {"local_stddev": 5}), - AI("P:Local", {"local_stddev": 10}), - AI("P:Local", {"local_stddev": 5}), - AI("P:Local", {"local_stddev": 1}), - AI("P:Local", {"local_stddev": 1, "pick_frac": 0.0, "pick_n": 20}), - AI("P:Weighted", {"pick_override": 1.0}), -] - -test_ais = [ - AI("Policy", {}), - AI("P:Noise", {"noise_strength": 0.8}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 0.95, "lower_bound": 0.0, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 0.9, "lower_bound": 0.0, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 0.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 1.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 2}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.01, "weaken_fac": 0.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.01, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.01, "weaken_fac": 1.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.01, "weaken_fac": 2}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.001, "weaken_fac": 0.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.001, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.001, "weaken_fac": 1.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.001, "weaken_fac": 2}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.005, "weaken_fac": 0.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.005, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.005, "weaken_fac": 1.5}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.005, "weaken_fac": 2}), - AI("P:Pick", {"pick_frac": 0.5, "pick_n": 0}), -] - - -test_ais = [ - AI("Policy", {}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.0, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.01, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.001, "weaken_fac": 1}), - AI("P:Weighted", {"pick_override": 1.0, "lower_bound": 0.005, "weaken_fac": 1}), -] - - -# test_ais = [ -# AI("Policy", {}), -# AI("P:Noise", {"noise_strength": 0.4}), -# AI("P:Noise", {"noise_strength": 0.5}), -# AI("P:Noise", {"noise_strength": 0.6}), -# AI("P:Noise", {"noise_strength": 0.7}), -# AI("P:Noise", {"noise_strength": 0.8}), -# ] - -# ai_database = [ai for ai in ai_database if "Territory" not in ai.name and "Influence" not in ai.name] -for ai in test_ais: - add_ai(ai) - -N_GAMES = 20 - -ais_to_test = retrieve_ais(test_ais) -# ais_to_test = ai_database -# ais_to_test = [ai for ai in ai_database if 'visits' not in ai.name] -# ais_to_test = [ai for ai in ai_database if 'visits' not in ai.name and (not 'model' in ai.name or 'b40' in ai.name)] - - -results = defaultdict(list) - - -def play_games(black: AI, white: AI, n: int = N_GAMES): - players = {"B": black, "W": white} - engines = {"B": black.get_engine(), "W": white.get_engine()} - tag = f"{black.name} vs {white.name}" - try: - for i in range(n): - game = Game(Logger(), engines, {}) - game.root.add_list_property("PW", [white.name]) - game.root.add_list_property("PB", [black.name]) - start_time = time.time() - while not game.ended: - p = game.current_node.next_player - move = ai_move(game, players[p].strategy, players[p].ai_settings) - while not game.current_node.analysis_ready: - time.sleep(0.001) - game.game_id += f"_{game.current_node.format_score()}" - print(f"{tag}\tGame {i+1} finished in {time.time()-start_time:.1f}s {game.current_node.format_score()} -> {game.write_sgf('sgf_selfplay/')}", file=sys.stderr) - score = game.current_node.score - if score > 0.3: - black.elo_comp.beat(white.elo_comp) - elif score > -0.3: - black.elo_comp.tied(white.elo_comp) - - results[tag].append(score) - all_results.append((black.name, white.name, score)) - - with open("tmp.pickle", "wb") as f: - pickle.dump((ai_database, all_results), f) - except Exception as e: - print(f"Exception in playing {tag}: {e}") - print(f"Exception in playing {tag}: {e}", file=sys.stderr) - traceback.print_exc() - traceback.print_exc(file=sys.stderr) - - -def fmt_score(score): - return f"{'B' if score >= 0 else 'W'}+{abs(score):.1f}" - - -print(len(ais_to_test), "ais to test") -global_start = time.time() - -with ThreadPoolExecutor(max_workers=16) as threadpool: - for b in ais_to_test: - for w in ais_to_test: - if b is not w: - threadpool.submit(play_games, b, w) - -print("POOL EXIT") - -print("---- RESULTS ----") -for k, v in results.items(): - b_win = sum([s > 0.3 for s in v]) - w_win = sum([s < -0.3 for s in v]) - print(f"{b_win} {k} {w_win} : {list(map(fmt_score,v))}") - -print("---- ELO ----") -for ai in sorted(ai_database, key=lambda a: -a.elo_comp.rating): - wins = [(b, w, s) for (b, w, s) in all_results if s > 0.3 and b == ai.name or w == ai.name and s < -0.3] - losses = [(b, w, s) for (b, w, s) in all_results if s < -0.3 and b == ai.name or w == ai.name and s > -0.3] - draws = [(b, w, s) for (b, w, s) in all_results if -0.3 <= s <= 0.3 and (b == ai.name or w == ai.name)] - out = f"{'*' if ai in ais_to_test else ' '} {ai.name}: ELO {ai.elo_comp.rating:.1f} WINS {len(wins)} LOSSES {len(losses)} DRAWS {len(draws)}" - # print("Wins:",wins) - print(out) - print(out, file=sys.stderr) - -with open(DB_FILENAME, "wb") as f: - pickle.dump((ai_database, all_results), f) - -print(f"Done! saving {len(all_results)} to pickle", file=sys.stderr) -print(f"Time taken {time.time()-global_start:.1f}s", file=sys.stderr)