diff --git a/bots/6b.bin.gz b/bots/6b.bin.gz deleted file mode 100644 index e533839..0000000 Binary files a/bots/6b.bin.gz and /dev/null differ diff --git a/bots/6b104-s22347264.txt.gz b/bots/6b104-s22347264.txt.gz deleted file mode 100644 index be57940..0000000 Binary files a/bots/6b104-s22347264.txt.gz and /dev/null differ diff --git a/bots/6b104-s42364928.txt.gz b/bots/6b104-s42364928.txt.gz deleted file mode 100644 index 6522217..0000000 Binary files a/bots/6b104-s42364928.txt.gz and /dev/null differ diff --git a/bots/BOTS.md b/bots/BOTS.md deleted file mode 100644 index aa63724..0000000 --- a/bots/BOTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# Bots -This directory contains the source code used to run the katrain-* bots on OGS, - and some code to test them. It is liable to break without warning. diff --git a/bots/__init__.py b/bots/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bots/ai2gtp.py b/bots/ai2gtp.py deleted file mode 100644 index c91d3be..0000000 --- a/bots/ai2gtp.py +++ /dev/null @@ -1,174 +0,0 @@ -# This is a script that turns a KaTrain AI into a sort-of GTP compatible bot -import json -import sys -import time -import random - -from katrain.core.ai import ai_move -from katrain.core.common import OUTPUT_ERROR, OUTPUT_INFO -from bots.settings import bot_strategy_names -from katrain.core.engine import EngineDiedException, KataGoEngine -from katrain.core.game import Game -from katrain.core.sgf_parser import Move - -if len(sys.argv) < 2: - bot = "dev" -else: - bot = sys.argv[1].strip() -port = int(sys.argv[2]) if len(sys.argv) > 2 else 8587 -REPORT_SCORE_THRESHOLD = 1.5 -MAX_WAIT_ANALYSIS = 10 -MAX_PASS = 3 # after opponent passes this many times, we always pass - - -class Logger: - def log(self, msg, level): - if level <= OUTPUT_INFO: - print(msg, file=sys.stderr) - - -logger = Logger() - - -ENGINE_SETTINGS = { - "katago": f"python bots/engine_connector.py {port}", # actual engine settings in engine_server.py - "model": "models/b15-1.3.2.txt.gz", - "config": "KataGo/analysis_config.cfg", - "max_visits": 5, - "max_time": 5.0, - "_enable_ownership": False, - "threads": 1, -} - -engine = KataGoEngine(logger, ENGINE_SETTINGS) - -with open("katrain/config.json") as f: - settings = json.load(f) - all_ai_settings = settings["ai"] - -if bot == "dev": - engine.override_settings["maxVisits"] = 500 -all_ai_settings["dev"] = all_ai_settings["ScoreLoss"] - -ai_strategy = bot_strategy_names[bot] -ai_settings = all_ai_settings[ai_strategy] - -print(f"starting bot {bot} using server port {port}", file=sys.stderr) -print(ENGINE_SETTINGS, file=sys.stderr) -print(ai_strategy, ai_settings, file=sys.stderr) - -logger.log(f"STARTED ENGINE", OUTPUT_ERROR) - -game = Game(Logger(), engine, {}) - - -def malkovich_analysis(cn): - start = time.time() - while not cn.analysis_ready: - time.sleep(0.001) - if engine.katago_process.poll() is not None: # TODO: clean up - raise EngineDiedException(f"Engine for {cn.next_player} ({engine.config}) died") - if time.time() - start > MAX_WAIT_ANALYSIS: - logger.log(f"Waiting for analysis timed out!", OUTPUT_ERROR) - return - if cn.analysis_ready and cn.parent and cn.parent.analysis_ready: - dscore = cn.analysis["root"]["scoreLead"] - cn.parent.analysis["root"]["scoreLead"] - logger.log(f"dscore {dscore} = {cn.analysis['root']['scoreLead']} {cn.parent.analysis['root']['scoreLead']} at {move}...", OUTPUT_ERROR) - if abs(dscore) > REPORT_SCORE_THRESHOLD and (cn.player == "B" and dscore < 0 or cn.player == "W" and dscore > 0): # relevant mistakes - favpl = "B" if dscore > 0 else "W" - msg = f"MALKOVICH:{cn.player} {cn.move.gtp()} caused a significant score change ({favpl} gained {abs(dscore):.1f} points)" - if cn.ai_thoughts: - msg += f" -> Win Rate {cn.format_win_rate()} Score {cn.format_score()} AI Thoughts: {cn.ai_thoughts}" - else: - comment = cn.comment(sgf=True).replace("\n", " ") - msg += f" -> Detailed move analysis: {comment}" - print(msg, file=sys.stderr) - sys.stderr.flush() - - -while True: - line = input() - logger.log(f"GOT INPUT {line}", OUTPUT_ERROR) - if "boardsize" in line: - _, *size = line.strip().split(" ") - if len(size) > 1: - size = f"{size[0]}:{size[1]}" - else: - size = int(size[0]) - game = Game(Logger(), engine, {"init_size": size}) - logger.log(f"Init game {game.root.properties}", OUTPUT_ERROR) - elif "komi" in line: - _, komi = line.split(" ") - game.root.set_property("KM", komi.strip()) - game.root.set_property("RU", "chinese") - logger.log(f"Setting komi {game.root.properties}", OUTPUT_ERROR) - elif "place_free_handicap" in line: - _, n = line.split(" ") - n = int(n) - game.place_handicap_stones(n) - handicaps = set(game.root.get_list_property("AB")) - bx, by = game.board_size - while len(handicaps) < min(n, bx * by): # really obscure cases - handicaps.add(Move((random.randint(0, bx - 1), random.randint(0, by - 1)), player="B").sgf(board_size=game.board_size)) - game.root.set_property("AB", list(handicaps)) - game._calculate_groups() - gtp = [Move.from_sgf(m, game.board_size, "B").gtp() for m in handicaps] - logger.log(f"Chose handicap placements as {gtp}", OUTPUT_ERROR) - print(f"= {' '.join(gtp)}\n") - sys.stdout.flush() - game.analyze_all_nodes() # re-evaluate root - while engine.queries: # and make sure this gets processed - time.sleep(0.001) - continue - elif "set_free_handicap" in line: - _, *stones = line.split(" ") - game.root.set_property("AB", [Move.from_gtp(move.upper()).sgf(game.board_size) for move in stones]) - game._calculate_groups() - game.analyze_all_nodes() # re-evaluate root - while engine.queries: # and make sure this gets processed - time.sleep(0.001) - logger.log(f"Set handicap placements to {game.root.get_list_property('AB')}", OUTPUT_ERROR) - elif "genmove" in line: - _, player = line.strip().split(" ") - if player[0].upper() != game.next_player: - logger.log(f"ERROR generating move: UNEXPECTED PLAYER {player} != {game.next_player}.", OUTPUT_ERROR) - print(f"= ??\n") - sys.stdout.flush() - continue - logger.log(f"{ai_strategy} generating move", OUTPUT_ERROR) - game.current_node.analyze(engine) - malkovich_analysis(game.current_node) - game.root.properties[f"P{game.current_node.next_player}"] = [f"KaTrain {ai_strategy}"] - num_passes = sum([int(n.is_pass or False) for n in game.current_node.nodes_from_root[::-1][0 : 2 * MAX_PASS : 2]]) - bx, by = game.board_size - if num_passes >= MAX_PASS and game.current_node.depth - 2 * MAX_PASS >= bx + by: - logger.log(f"Forced pass as opponent is passing {MAX_PASS} times", OUTPUT_ERROR) - pol = game.current_node.policy - if not pol: - pol = ["??"] - print(f"DISCUSSION:OK, since you passed {MAX_PASS} times after the {bx+by}th move, I will pass as well [policy {pol[-1]:.3%}].", file=sys.stderr) - move = game.play(Move(None, player=game.next_player)).move - else: - move, node = ai_move(game, ai_strategy, ai_settings) - logger.log(f"Generated move {move}", OUTPUT_ERROR) - print(f"= {move.gtp()}\n") - sys.stdout.flush() - malkovich_analysis(game.current_node) - continue - elif "play" in line: - _, player, move = line.split(" ") - node = game.play(Move.from_gtp(move.upper(), player=player[0].upper()), analyze=False) - logger.log(f"played {player} {move}", OUTPUT_ERROR) - elif "final_score" in line: - score = game.current_node.format_score() - game.game_id += f"_{score}" - sgf = game.write_sgf("sgf_ogs/") - logger.log(f"Game ended. Score was {score} -> saved sgf to {sgf}", OUTPUT_ERROR) - print(f"= {score}\n") - sys.stdout.flush() - continue - elif "quit" in line: - print(f"= \n") - break - print(f"= \n") - sys.stdout.flush() diff --git a/bots/ai_performance.pickle b/bots/ai_performance.pickle deleted file mode 100644 index dddb3cc..0000000 Binary files a/bots/ai_performance.pickle and /dev/null differ diff --git a/bots/engine_connector.py b/bots/engine_connector.py deleted file mode 100644 index 086d93e..0000000 --- a/bots/engine_connector.py +++ /dev/null @@ -1,26 +0,0 @@ -# used to connect many bots to one kata engine -import socket -import sys -import time - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8587 - -restart = False - -while True: - try: - sock = socket.create_connection(("localhost", PORT)).makefile(mode="rw") - while True: - line = input() - sock.write(line + "\n") - sock.flush() - response = sock.readline() - print(response.strip()) - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - - if not restart: - break - - print("Failed to connect or disconnected, waiting to reconnect", file=sys.stderr) - time.sleep(5) diff --git a/bots/engine_server.py b/bots/engine_server.py deleted file mode 100644 index 1f89cb3..0000000 --- a/bots/engine_server.py +++ /dev/null @@ -1,75 +0,0 @@ -# used to connect many bots to one kata engine -import json -import random -import socket -import sys -import threading -import traceback - -from katrain.core.common import OUTPUT_INFO -from katrain.core.engine import KataGoEngine - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8587 - -ENGINE_SETTINGS = { - "katago": "my/katago25", - # "katago": "KataGo/katago", - "model": "katrain/models/g170e-b15c192-s1672170752-d466197061.bin.gz", - "config": "katrain/KataGo/analysis_config.cfg", - "max_visits": 50, - "max_time": 1.0, - "_enable_ownership": False, - "threads": 32, -} - - -class Logger: - def log(self, msg, level): - if level <= OUTPUT_INFO: - print(f"[{level} {msg}") - - -engine = KataGoEngine(Logger(), ENGINE_SETTINGS) - - -def engine_thread(conn, addr): - sockfile = conn.makefile(mode="rw") - try: - while True: - print(f"Waiting for input from {addr}") - line = sockfile.readline() - if not line: - break - query = {"id": "???"} - try: - query = json.loads(line) - tag = f"{int(random.random()*1000000000):09d}__" - query["id"] = tag + str(query["id"]) - - def callback(analysis, *args): - print(f"Returning {analysis['id']} for {addr} -> {len(engine.queries)} outstanding queries") - analysis["id"] = analysis["id"][len(tag) :] - sockfile.write(json.dumps(analysis) + "\n") - sockfile.flush() - - engine.send_query(query, callback=callback, error_callback=callback) - except Exception as e: - print("Sent error to {addr}") - traceback.print_exc() - sockfile.write(json.dumps({"id": query["id"], "error": str(e)}) + "\n") - sockfile.flush() - except Exception as e: - traceback.print_exc() - print(f"Error: {e}") - print(f"Disconnected: {addr}") - conn.close() - - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.bind(("localhost", PORT)) -sock.listen(100) -print("Listening..") -while True: - conn, addr = sock.accept() - print(f"Accepted connection from {addr}") - threading.Thread(target=engine_thread, args=(conn, addr), daemon=True).start() diff --git a/bots/lowmem.cfg b/bots/lowmem.cfg deleted file mode 100644 index d2d2d18..0000000 --- a/bots/lowmem.cfg +++ /dev/null @@ -1,193 +0,0 @@ -# Example config for C++ (non-python) gtp bot - -# SEE NOTES ABOUT PERFORMANCE AND MEMORY USAGE IN gtp_example.cfg - -# Logs------------------------------------------------------------------------------------ - -# Where to output log? -logFile = gtp.log - -# Controls the number of moves after the first move in a variation. -# analysisPVLen = 15 - -# Report winrates for analysis as (BLACK|WHITE|SIDETOMOVE). -reportAnalysisWinratesAs = BLACK - -# Bot behavior--------------------------------------------------------------------------------------- - -# Handicap ------------- - -# Assume that if black makes many moves in a row right at the start of the game, then the game is a handicap game. -# This is necessary on some servers and for some GUIs and also when initializing from many SGF files, which may -# set up a handicap games using repeated GTP "play" commands for black rather than GTP "place_free_handicap" commands. -# However, it may also lead to incorrect undersanding of komi if whiteBonusPerHandicapStone = 1 and a server does NOT -# have such a practice. -# Defaults to true! Uncomment and set to false to disable this behavior. -# assumeMultipleStartingBlackMovesAreHandicap = true - -# Passing and cleanup ------------- - -# Make the bot never assume that its pass will end the game, even if passing would end and "win" under Tromp-Taylor rules. -# Usually this is a good idea when using it for analysis or playing on servers where scoring may be implemented non-tromp-taylorly. -# Defaults to true! Uncomment and set to false to disable this. -conservativePass = true - -# When using territory scoring, self-play games continue beyond two passes with special cleanup -# rules that may be confusing for human players. This option prevents the special cleanup phases from being -# reachable when using the bot for GTP play. -# Defaults to true! Uncomment and set to false if you want KataGo to be able to enter special cleanup. -# For example, if you are testing it against itself, or against another bot that has precisely implemented the rules -# documented at https://lightvector.github.io/KataGo/rules.html -# preventCleanupPhase = true - -# Search limits----------------------------------------------------------------------------------- - -# By default, if NOT specified in an individual request, limit maximum number of root visits per search to this much -maxVisits = 500 -# If provided, cap search time at this many seconds -# maxTime = 60 - -# Number of threads to use in each search in parallel for any SINGLE position. -# NOTE: Analysis engine can specify number of POSITIONS to be able to search in parallel via command line argument -# so this number does not necessarily need to be larger than 1, although you can still set it larger if you prefer -# to analyze fewer positions in parallel but spend more threads on each position. -# Generally, having more threads on a single position will worsen the quality of search slightly, holding fixed the -# number of visits, and thread contention will reduce efficiency, so cross-position parallelization is preferable -# to numSearchThreads, but numSearchThreads is preferable if you want to reduce latency, and have individual -# searches complete faster by doing fewer of them at a time. -numSearchThreads = 2 - -# GPU Settings------------------------------------------------------------------------------- - -# Maximum number of positions to send to GPU at once. -nnMaxBatchSize = 32 -# Cache up to 2 ** this many neural net evaluations in case of transpositions in the tree. -nnCacheSizePowerOfTwo = 14 -# Size of mutex pool for nnCache is 2 ** this -nnMutexPoolSizePowerOfTwo = 14 -# Randomize board orientation when running neural net evals? -nnRandomize = true - - -# TO USE MULTIPLE GPUS: -# Set this to the number of GPUs you have and/or would like to use... -# AND if it is more than 1, uncomment the appropriate CUDA or OpenCL section below. -# numNNServerThreadsPerModel = 1 - - -# CUDA GPU settings-------------------------------------- -# These only apply when using the CUDA version of KataGo. - -# IF USING ONE GPU: optionally uncomment and change this if the GPU you want to use turns out to be not device 0 -# cudaDeviceToUse = 0 - -# IF USING TWO GPUS: Uncomment these two lines (AND set numNNServerThreadsPerModel above): -# cudaDeviceToUseThread0 = 0 # change this if the first GPU you want to use turns out to be not device 0 -# cudaDeviceToUseThread1 = 1 # change this if the second GPU you want to use turns out to be not device 1 - -# IF USING THREE GPUS: Uncomment these three lines (AND set numNNServerThreadsPerModel above): -# cudaDeviceToUseThread0 = 0 # change this if the first GPU you want to use turns out to be not device 0 -# cudaDeviceToUseThread1 = 1 # change this if the second GPU you want to use turns out to be not device 1 -# cudaDeviceToUseThread2 = 2 # change this if the third GPU you want to use turns out to be not device 2 - -# You can probably guess the pattern if you have four, five, etc. GPUs. - -# KataGo will automatically use FP16 or not based on the compute capability of your NVIDIA GPU. If you -# want to try to force a particular behavior though you can uncomment these lines and change them -# to "true" or "false". E.g. it's using FP16 but on your card that's giving an error, or it's not using -# FP16 but you think it should. -# cudaUseFP16 = auto -# cudaUseNHWC = auto - - -# OpenCL GPU settings-------------------------------------- -# These only apply when using the OpenCL version of KataGo. - -# Uncomment to tune OpenCL for every board size separately, rather than only the largest possible size -# openclReTunePerBoardSize = true - -# IF USING ONE GPU: optionally uncomment and change this if the best device to use is guessed incorrectly. -# The default behavior tries to guess the 'best' GPU or device on your system to use, usually it will be a good guess. -# openclDeviceToUse = 0 - -# IF USING TWO GPUS: Uncomment these two lines and replace X and Y with the device ids of the devices you want to use. -# It might NOT be 0 and 1, some computers will have many OpenCL devices. You can see what the devices are when -# KataGo starts up - it should print or log all the devices it finds. -# (AND also set numNNServerThreadsPerModel above) -# openclDeviceToUseThread0 = X -# openclDeviceToUseThread1 = Y - -# IF USING THREE GPUS: Uncomment these three lines and replace X and Y and Z with the device ids of the devices you want to use. -# It might NOT be 0 and 1 and 2, some computers will have many OpenCL devices. You can see what the devices are when -# KataGo starts up - it should print or log all the devices it finds. -# (AND also set numNNServerThreadsPerModel above) -# openclDeviceToUseThread0 = X -# openclDeviceToUseThread1 = Y -# openclDeviceToUseThread2 = Z - -# You can probably guess the pattern if you have four, five, etc. GPUs. - - -# Root move selection and biases------------------------------------------------------------------------------ -# Uncomment and edit any of the below values to change them from their default. -# Not all of these parameters are applicable to analysis, some are only used for actual play - -# Temperature for the early game, randomize between chosen moves with this temperature -# chosenMoveTemperatureEarly = 0.5 -# Decay temperature for the early game by 0.5 every this many moves, scaled with board size. -# chosenMoveTemperatureHalflife = 19 -# At the end of search after the early game, randomize between chosen moves with this temperature -# chosenMoveTemperature = 0.10 -# Subtract this many visits from each move prior to applying chosenMoveTemperature -# (unless all moves have too few visits) to downweight unlikely moves -# chosenMoveSubtract = 0 -# The same as chosenMoveSubtract but only prunes moves that fall below the threshold, does not affect moves above -# chosenMovePrune = 1 - -# Number of symmetries to sample (WITH replacement) and average at the root -# rootNumSymmetriesToSample = 1 - -# Using LCB for move selection? -# useLcbForSelection = true -# How many stdevs a move needs to be better than another for LCB selection -# lcbStdevs = 5.0 -# Only use LCB override when a move has this proportion of visits as the top move -# minVisitPropForLCB = 0.15 - -# Internal params------------------------------------------------------------------------------ -# Uncomment and edit any of the below values to change them from their default. - -# Scales the utility of winning/losing -# winLossUtilityFactor = 1.0 -# Scales the utility for trying to maximize score -# staticScoreUtilityFactor = 0.10 -# dynamicScoreUtilityFactor = 0.30 -# Adjust dynamic score center this proportion of the way towards zero, capped at a reasonable amount. -# dynamicScoreCenterZeroWeight = 0.20 -# dynamicScoreCenterScale = 0.75 -# The utility of getting a "no result" due to triple ko or other long cycle in non-superko rulesets (-1 to 1) -# noResultUtilityForWhite = 0.0 -# The number of wins that a draw counts as, for white. (0 to 1) -# drawEquivalentWinsForWhite = 0.5 - -# Exploration constant for mcts -# cpuctExploration = 0.9 -# cpuctExplorationLog = 0.4 -# FPU reduction constant for mcts -# fpuReductionMax = 0.2 -# rootFpuReductionMax = 0.1 -# Use parent average value for fpu base point instead of point value net estimate -# fpuUseParentAverage = true -# Amount to apply a downweighting of children with very bad values relative to good ones -# valueWeightExponent = 0.5 -# Slight incentive for the bot to behave human-like with regard to passing at the end, filling the dame, -# not wasting time playing in its own territory, etc, and not play moves that are equivalent in terms of -# points but a bit more unfriendly to humans. -# rootEndingBonusPoints = 0.5 -# Make the bot prune useless moves that are just prolonging the game to avoid losing yet -# rootPruneUselessMoves = true - -# How big to make the mutex pool for search synchronization -# mutexPoolSize = 8192 -# How many virtual losses to add when a thread descends through a node -# numVirtualLossesPerThread = 1 diff --git a/bots/selfplay.py b/bots/selfplay.py deleted file mode 100644 index 0585402..0000000 --- a/bots/selfplay.py +++ /dev/null @@ -1,205 +0,0 @@ -# This is a script I use to test the performance of AIs -import pickle -import sys -import threading -import time -import traceback -from collections import defaultdict -from concurrent.futures.thread import ThreadPoolExecutor - -from katrain.core.ai import ai_move -from katrain.core.common import OUTPUT_ERROR, OUTPUT_INFO -from elote import EloCompetitor -from katrain.core.engine import KataGoEngine -from katrain.core.game import Game -import json - -DB_FILENAME = "bots/ai_performance.pickle" - - -class Logger: - def log(self, msg, level): - if level <= OUTPUT_INFO: - print(msg) - if level <= OUTPUT_ERROR: - print(msg, file=sys.stderr) - - -logger = Logger() - -with open("config.json") as f: - settings = json.load(f) - DEFAULT_AI_SETTINGS = settings["ai"] - - -class AI: - DEFAULT_ENGINE_SETTINGS = { - "katago": "KataGo/katago", - "model": "KataGo/models/b15-1.3.2.txt.gz", - "config": "bots/lowmem.cfg", - "max_visits": 1, - "max_time": 300.0, - "_enable_ownership": False, - } - NUM_THREADS = 32 - IGNORE_SETTINGS_IN_TAG = {"threads", "_enable_ownership", "katago"} # katago for switching from/to bs version - ENGINES = [] - LOCK = threading.Lock() - - def __init__(self, strategy, ai_settings, engine_settings=None): - self.elo_comp = EloCompetitor(initial_rating=1000) - self.strategy = strategy - self.ai_settings = ai_settings - self.engine_settings = engine_settings or {} - fmt_settings = [f"{k}={v}" for k, v in {**self.ai_settings, **self.engine_settings}.items() if k not in AI.IGNORE_SETTINGS_IN_TAG] - self.name = f"{strategy}({ ','.join(fmt_settings) })" - self.fix_settings() - - def fix_settings(self): - self.ai_settings = {**DEFAULT_AI_SETTINGS[self.strategy], **self.ai_settings} - self.engine_settings = {**AI.DEFAULT_ENGINE_SETTINGS, **self.engine_settings, "threads": AI.NUM_THREADS} - - def get_engine(self): # factory - with AI.LOCK: - for existing_engine_settings, engine in AI.ENGINES: - if existing_engine_settings == self.engine_settings: - return engine - engine = KataGoEngine(logger, self.engine_settings) - AI.ENGINES.append((self.engine_settings, engine)) - print("Creating new engine for", self.engine_settings, "now have", len(AI.ENGINES), "engines up") - return engine - - def __eq__(self, other): - return self.name == other.name # should capture all relevant setting differences - - -try: - with open(DB_FILENAME, "rb") as f: - ai_database_loaded, all_results = pickle.load(f) - ai_database = [] - for ai in ai_database_loaded: - try: - ai.fix_settings() # update as required - ai_database.append(ai) - except: - print("Error loading AI", ai.strategy) -except FileNotFoundError: - ai_database = [] - all_results = [] - - -def add_ai(ai): - if ai not in ai_database: - ai_database.append(ai) - print(f"Adding {ai.name}") - else: - print(f"AI {ai.name} already in DB") - - -def retrieve_ais(selected_ais): - return [ai for ai in ai_database if ai in selected_ais] - - -test_ais = [ - AI("Default", {}, {"model": "bots/6b.bin.gz", "max_visits": 500}), - AI("Default", {}, {"model": "bots/6b104-s22347264.txt.gz", "max_visits": 500}), - AI("Default", {}, {"model": "bots/6b104-s42364928.txt.gz", "max_visits": 500}), - # AI("Default", {}, {"model": "KataGo/models/b10-1.3.txt.gz", "max_visits": 500}), - AI("Policy", {}), - AI("P:Local", {}), - AI("P:Weighted", {}), - AI("P:Pick", {}), - AI("ScoreLoss", {"max_visits": 500}), - # AI("P:Tenuki", {}), - # AI("P:Local", {}), - AI("P:Influence", {}), - # AI("P:Territory", {}), -] - - -for ai in test_ais: - add_ai(ai) - -N_GAMES = 1 -BOARDSIZE = 19 - -ais_to_test = retrieve_ais(test_ais) - -results = defaultdict(list) - - -def play_games(black: AI, white: AI): - players = {"B": black, "W": white} - engines = {"B": black.get_engine(), "W": white.get_engine()} - tag = f"{black.name} vs {white.name}" - try: - game = Game(Logger(), engines, {"init_size": BOARDSIZE}) - game.root.add_list_property("PW", [white.name]) - game.root.add_list_property("PB", [black.name]) - start_time = time.time() - while not game.ended and game.current_node.depth < 300: - p = game.current_node.next_player - move, node = ai_move(game, players[p].strategy, players[p].ai_settings) - while not game.current_node.analysis_ready: - time.sleep(0.001) - game.game_id += f"_{game.current_node.format_score()}" - print( - f"{tag}\tGame finished in {time.time()-start_time:.1f}s @ move {game.current_node.depth} {game.current_node.format_score()} -> {game.write_sgf('sgf_selfplay/')}", - file=sys.stderr, - ) - score = game.current_node.score - if score > 0.3: - black.elo_comp.beat(white.elo_comp) - elif score > -0.3: - black.elo_comp.tied(white.elo_comp) - - results[tag].append(score) - all_results.append((black.name, white.name, score)) - - except Exception as e: - print(f"Exception in playing {tag}: {e}") - print(f"Exception in playing {tag}: {e}", file=sys.stderr) - traceback.print_exc() - traceback.print_exc(file=sys.stderr) - - -def fmt_score(score): - return f"{'B' if score >= 0 else 'W'}+{abs(score):.1f}" - - -print(len(ais_to_test), "ais to test") -global_start = time.time() - -for n in range(N_GAMES): - for _, e in AI.ENGINES: # no caching/replays - e.shutdown() - AI.ENGINES = [] - - with ThreadPoolExecutor(max_workers=16) as threadpool: - for b in ais_to_test: - for w in ais_to_test: - if b is not w: - threadpool.submit(play_games, b, w) - print("POOL EXIT") - - print(f"---- RESULTS ({n}) ----") - for k, v in results.items(): - b_win = sum([s > 0.3 for s in v]) - w_win = sum([s < -0.3 for s in v]) - print(f"{b_win} {k} {w_win} : {list(map(fmt_score,v))}") - - print("---- ELO ----") - for ai in sorted(ai_database, key=lambda a: -a.elo_comp.rating): - wins = [(b, w, s) for (b, w, s) in all_results if s > 0.3 and b == ai.name or w == ai.name and s < -0.3] - losses = [(b, w, s) for (b, w, s) in all_results if s < -0.3 and b == ai.name or w == ai.name and s > -0.3] - draws = [(b, w, s) for (b, w, s) in all_results if -0.3 <= s <= 0.3 and (b == ai.name or w == ai.name)] - out = f"{'*' if ai in ais_to_test else ' '} {ai.name}: ELO {ai.elo_comp.rating:.1f} WINS {len(wins)} LOSSES {len(losses)} DRAWS {len(draws)}" - # print("Wins:",wins) - print(out) - print(out, file=sys.stderr) - - with open(DB_FILENAME, "wb") as f: - pickle.dump((ai_database, all_results), f) - print(f"Saving {len(all_results)} to pickle", file=sys.stderr) - -print(f"Done!Time taken {time.time()-global_start:.1f}s", file=sys.stderr) diff --git a/bots/settings.py b/bots/settings.py deleted file mode 100644 index 1dd6282..0000000 --- a/bots/settings.py +++ /dev/null @@ -1,26 +0,0 @@ -bot_strategy_names = { - # "dev": "P:Noise", - "dev": "ScoreLoss", - "dev-beta": "P:Weighted", - "strong": "Policy", - "influence": "P:Influence", - "territory": "P:Territory", - "balanced": "P:Pick", - "weighted": "P:Weighted", - "local": "P:Local", - "tenuki": "P:Tenuki", -} - - -greetings = { - # "dev": "Policy+Dirichlet noise.", - "dev": "Point loss-weighted random move.", - "dev-beta": "Play a policy-weighted move.", - "strong": "Play top policy move.", - "influence": "Play an influential style.", - "territory": "Play a territorial style.", - "balanced": "Play the best move out of a random selection.", - "weighted": "Play a policy-weighted move.", - "local": "Prefer local responses.", - "tenuki": "Prefer to tenuki.", -} diff --git a/bots/start_bot.py b/bots/start_bot.py deleted file mode 100755 index 010680a..0000000 --- a/bots/start_bot.py +++ /dev/null @@ -1,48 +0,0 @@ -import json -import os -import sys -from bots.settings import bot_strategy_names, greetings - -if len(sys.argv) < 2: - exit(0) - -bot = sys.argv[1].strip() -port = int(sys.argv[2]) if len(sys.argv) > 2 else 8587 - -MAXGAMES = 10 -if True or bot in ["dev", "local"]: - GTP2OGS = "node ../gtp2ogs" -else: - GTP2OGS = "node ../stable-gtp2ogs" -BOT_SETTINGS = f" --maxconnectedgames {MAXGAMES} --maxhandicapunranked 25 --maxhandicapranked 1 --boardsizesranked 19 --boardsizesunranked all --komisranked automatic,5.5,6.5,7.5 --komisunranked all" -if "beta" in bot: - BOT_SETTINGS += " --beta" -else: - BOT_SETTINGS += "" # --rankedonly" - -username = f"katrain-{bot}" - -with open("katrain/config.json") as f: - settings = json.load(f) - all_ai_settings = settings["ai"] - -ai_settings = all_ai_settings[bot_strategy_names[bot]] - -with open("my/apikey.json") as f: - apikeys = json.load(f) - -if bot not in greetings or username not in apikeys: - print("BOT NOT FOUND") - exit(1) - -APIKEY = apikeys[username] -settings_dump = ", ".join(f"{k}={v}" for k, v in ai_settings.items() if not k.startswith("_")) -print(settings_dump) -GREETING = f"Hello, play with these bots at any time by downloading KaTrain at github.com/sanderland/katrain - Current mode is {bot_strategy_names[bot]} ({greetings[bot]})" -if settings: - GREETING += f" Settings: {settings_dump}." -BYEMSG = "Thank you for playing. If you have any feedback, please message my admin! " - -cmd = f'{GTP2OGS} --debug --apikey {APIKEY} --rejectnewfile ~/shutdown_bots --username {username} --greeting "{GREETING}" --farewell "{BYEMSG}" {BOT_SETTINGS} --farewellscore --aichat --noclock --nopause --speeds blitz,live --persist --minrank 25k -- python bots/ai2gtp.py {bot} {port}' -print(f"starting bot {username} using server port {port} --> {cmd}") -os.system(cmd)