1#!/usr/bin/env python3 2# 3# This file is part of Leela Zero. 4# Copyright (C) 2017 Andy Olsen 5# 6# Leela Zero is free software: you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation, either version 3 of the License, or 9# (at your option) any later version. 10# 11# Leela Zero is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with Leela Zero. If not, see <http://www.gnu.org/licenses/>. 18 19import argparse 20import math 21import os 22import sys 23 24class GameStats: 25 def __init__(self, filename): 26 self.filename = filename 27 self.total_moves = None 28 self.resign_movenum = None 29 self.resign_type = None # "Correct", "Wrong" 30 self.winner = None 31 32class TotalStats: 33 def __init__(self): 34 self.num_games = 0 35 self.no_resign_count = 0 36 self.correct_resign_count = 0 37 self.wrong_resign_count = 0 38 self.game_len_sum = 0 39 self.resigned_game_len_sum = 0 40 def calcOverall(self, b, w): 41 self.num_games = b.num_games + w.num_games 42 self.no_resign_count = b.no_resign_count + w.no_resign_count 43 self.correct_resign_count = ( 44 b.correct_resign_count + w.correct_resign_count) 45 self.wrong_resign_count = ( 46 b.wrong_resign_count + w.wrong_resign_count) 47 self.game_len_sum = b.game_len_sum + w.game_len_sum 48 self.resigned_game_len_sum = ( 49 b.resigned_game_len_sum + w.resigned_game_len_sum) 50 51def to_move_str(to_move): 52 if (to_move): return "W" 53 else: return "B" 54 55def parseGameBody(filename, fh, tfh, verbose, resignthr): 56 gs = GameStats(filename) 57 movenum = 0 58 while 1: 59 movenum += 1 60 for _ in range(16): 61 line = tfh.readline() # Board input planes 62 if not line: break 63 to_move = int(tfh.readline()) # 0 = black, 1 = white 64 policy_weights = tfh.readline() # 361 moves + 1 pass 65 side_to_move_won = int(tfh.readline()) # 1 for win, -1 for loss 66 if not gs.winner: 67 if side_to_move_won == 1: gs.winner = to_move 68 else : gs.winner = 1 - to_move 69 (netwinrate, root_uctwinrate, child_uctwinrate, bestmovevisits) = ( 70 fh.readline().split()) 71 netwinrate = float(netwinrate) 72 root_uctwinrate = float(root_uctwinrate) 73 child_uctwinrate = float(child_uctwinrate) 74 bestmovevisits = int(bestmovevisits) 75 if side_to_move_won == 1: 76 if verbose >= 3: 77 print("+", to_move, movenum, netwinrate, child_uctwinrate, 78 bestmovevisits) 79 if not gs.resign_type and child_uctwinrate < resignthr: 80 if verbose >= 1: 81 print(("Wrong resign -- %s rt=%0.3f wr=%0.3f " 82 "winner=%s movenum=%d") % 83 (filename, resignthr, child_uctwinrate, 84 to_move_str(to_move), movenum)) 85 if verbose >= 3: 86 print("policy_weights", policy_weights) 87 gs.resign_type = "Wrong" 88 gs.resign_movenum = movenum 89 else: 90 if verbose >= 2: 91 print("-", to_move, movenum, netwinrate, child_uctwinrate, 92 bestmovevisits) 93 if not gs.resign_type and child_uctwinrate < resignthr: 94 if verbose >= 2: 95 print("Correct resign -- %s" % (filename)) 96 gs.resign_type = "Correct" 97 gs.resign_movenum = movenum 98 gs.total_moves = movenum 99 return gs 100 101def parseGames(filenames, resignthr, verbose, prefixes): 102 gsd = {} 103 for filename in filenames: 104 training_filename = filename.replace(".debug", "") 105 with open(filename) as fh, open(training_filename) as tfh: 106 version = fh.readline().rstrip() 107 assert version == "2" 108 (cfg_resignpct, network) = fh.readline().split() 109 if prefixes: 110 net_name = os.path.basename(network) 111 matches = filter(lambda n: net_name.startswith(n), prefixes) 112 # Require at least one matching net prefix. 113 if not list(matches): 114 continue 115 cfg_resignpct = int(cfg_resignpct) 116 if cfg_resignpct == 0: 117 gsd[filename] = parseGameBody(filename, fh, tfh, verbose, resignthr) 118 elif verbose >= 2: 119 print("{} was played with -r {}, skipping".format( 120 filename, cfg_resignpct)) 121 return gsd 122 123def resignStats(gsd, resignthr): 124 # [ B wins, W wins, Overall ] 125 stats = [ TotalStats(), TotalStats(), TotalStats() ] 126 for gs in gsd.values(): 127 stats[gs.winner].num_games += 1 128 if not gs.resign_type: 129 stats[gs.winner].no_resign_count += 1 130 stats[gs.winner].resigned_game_len_sum += gs.total_moves 131 elif gs.resign_type == "Correct": 132 stats[gs.winner].correct_resign_count += 1 133 stats[gs.winner].resigned_game_len_sum += gs.resign_movenum 134 else: 135 assert gs.resign_type == "Wrong" 136 stats[gs.winner].wrong_resign_count += 1 137 stats[gs.winner].resigned_game_len_sum += gs.resign_movenum 138 stats[gs.winner].game_len_sum += gs.total_moves 139 stats[2].calcOverall(stats[0], stats[1]) 140 print("Resign thr: %0.2f - Black won %d/%d (%0.2f%%)" % ( 141 resignthr, 142 stats[0].num_games, 143 stats[0].num_games+stats[1].num_games, 144 100 * stats[0].num_games / (stats[0].num_games+stats[1].num_games))) 145 for winner in (0,1,2): 146 win_str = 'Overall ' 147 if winner==0: 148 win_str = 'Black wins' 149 elif winner==1: 150 win_str = 'White wins' 151 if stats[winner].num_games == 0: 152 print(" No games to report") 153 continue 154 avg_len = stats[winner].game_len_sum / stats[winner].num_games 155 resigned_avg_len = (stats[winner].resigned_game_len_sum / 156 stats[winner].num_games) 157 avg_reduction = (avg_len - resigned_avg_len) / avg_len 158 print(("%s - Wrong: %d/%d (%0.2f%%) Correct: %d/%d (%0.2f%%) " 159 "No Resign: %d/%d (%0.2f%%)") % ( 160 win_str, 161 stats[winner].wrong_resign_count, 162 stats[winner].num_games, 163 100 * stats[winner].wrong_resign_count / stats[winner].num_games, 164 stats[winner].correct_resign_count, 165 stats[winner].num_games, 166 100 * stats[winner].correct_resign_count / stats[winner].num_games, 167 stats[winner].no_resign_count, 168 stats[winner].num_games, 169 100 * stats[winner].no_resign_count / stats[winner].num_games)) 170 print("%s - Average game length: %d/%d (%0.2f%% reduction)" % ( 171 win_str, resigned_avg_len, avg_len, avg_reduction*100)) 172 print() 173 return stats 174 175if __name__ == "__main__": 176 usage_str = """ 177This script analyzes the debug output from leelaz 178to determine the impact of various resign thresholds. 179 180Process flow: 181 Run autogtp with debug on: 182 autogtp -k savedir -d savedir 183 184 Unzip training and debug files: 185 gunzip savedir/*.gz 186 187 Analyze results with this script: 188 ./resign_analysis.py savedir/*.debug.txt.0 189 190Note the script takes the debug files hash.debug.txt.0 191as the input arguments, but it also expects the training 192files hash.txt.0 to be in the same directory.""" 193 parser = argparse.ArgumentParser( 194 formatter_class=argparse.RawDescriptionHelpFormatter, 195 description=usage_str) 196 default_resignthrs="0.5,0.2,0.15,0.1,0.05,0.02,0.01" 197 parser.add_argument( 198 "-r", metavar="Resign_thresholds", dest="resignthrs", type=str, 199 default=default_resignthrs, 200 help="comma separated resign thresholds (default {})".format( 201 default_resignthrs)) 202 parser.add_argument( 203 "-R", metavar="Resign_rate", dest="resignrate", type=float, 204 help="If specified, a search is performed that finds the maximum \ 205 resign threshold that can be set without exceeding the given \ 206 resign rate") 207 parser.add_argument( 208 "-v", metavar="Verbose", dest="verbose", type=int, default=0, 209 help="Verbosity level (default 0)") 210 parser.add_argument( 211 "data", metavar="files", type=str, nargs="+", 212 help="Debug data files (*.debug.txt.0)") 213 parser.add_argument( 214 "-n", metavar="Prefix", dest="networks", nargs="+", 215 help="Prefixes of specific networks to analyze") 216 args = parser.parse_args() 217 resignthrs = [float(i) for i in args.resignthrs.split(",")] 218 if args.networks: 219 print("Analyzing networks starting with: {}".format( 220 ",".join(args.networks))) 221 222 for resignthr in (resignthrs): 223 gsd = parseGames(args.data, resignthr, args.verbose, args.networks) 224 if gsd: 225 resignStats(gsd, resignthr) 226 else: 227 print("No games to analyze (for more info try running with -v 2)") 228 229 if args.resignrate: 230 L = 0.0 231 R = 0.5 232 while L < R : 233 resignthr = math.floor((L + R) * 50) / 100 234 gsd = parseGames(args.data, resignthr, args.verbose, args.networks) 235 if not gsd: 236 print("No games to analyze (for more info try running with -v 2)") 237 break 238 stats = resignStats(gsd, resignthr) 239 wrong_rate = stats[2].wrong_resign_count / stats[2].num_games 240 if wrong_rate > args.resignrate: 241 if R == resignthr: 242 R = (math.floor(resignthr * 100) - 1) / 100 243 else: 244 R = resignthr 245 else: 246 L = (math.floor(resignthr * 100) + 1) / 100 247 if (L == R): 248 print(("The highest the resign threshold should be set to: %0.2f") 249 % (R - 0.01)) 250