1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
#!/usr/bin/env python3
#
# This file is part of Leela Zero.
# Copyright (C) 2017 Andy Olsen
#
# Leela Zero is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Leela Zero is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Leela Zero. If not, see <http://www.gnu.org/licenses/>.
import argparse
import math
import os
import sys
class GameStats:
def __init__(self, filename):
self.filename = filename
self.total_moves = None
self.resign_movenum = None
self.resign_type = None # "Correct", "Wrong"
self.winner = None
class TotalStats:
def __init__(self):
self.num_games = 0
self.no_resign_count = 0
self.correct_resign_count = 0
self.wrong_resign_count = 0
self.game_len_sum = 0
self.resigned_game_len_sum = 0
def calcOverall(self, b, w):
self.num_games = b.num_games + w.num_games
self.no_resign_count = b.no_resign_count + w.no_resign_count
self.correct_resign_count = (
b.correct_resign_count + w.correct_resign_count)
self.wrong_resign_count = (
b.wrong_resign_count + w.wrong_resign_count)
self.game_len_sum = b.game_len_sum + w.game_len_sum
self.resigned_game_len_sum = (
b.resigned_game_len_sum + w.resigned_game_len_sum)
def to_move_str(to_move):
if (to_move): return "W"
else: return "B"
def parseGameBody(filename, fh, tfh, verbose, resignthr):
gs = GameStats(filename)
movenum = 0
while 1:
movenum += 1
for _ in range(16):
line = tfh.readline() # Board input planes
if not line: break
to_move = int(tfh.readline()) # 0 = black, 1 = white
policy_weights = tfh.readline() # 361 moves + 1 pass
side_to_move_won = int(tfh.readline()) # 1 for win, -1 for loss
if not gs.winner:
if side_to_move_won == 1: gs.winner = to_move
else : gs.winner = 1 - to_move
(netwinrate, root_uctwinrate, child_uctwinrate, bestmovevisits) = (
fh.readline().split())
netwinrate = float(netwinrate)
root_uctwinrate = float(root_uctwinrate)
child_uctwinrate = float(child_uctwinrate)
bestmovevisits = int(bestmovevisits)
if side_to_move_won == 1:
if verbose >= 3:
print("+", to_move, movenum, netwinrate, child_uctwinrate,
bestmovevisits)
if not gs.resign_type and child_uctwinrate < resignthr:
if verbose >= 1:
print(("Wrong resign -- %s rt=%0.3f wr=%0.3f "
"winner=%s movenum=%d") %
(filename, resignthr, child_uctwinrate,
to_move_str(to_move), movenum))
if verbose >= 3:
print("policy_weights", policy_weights)
gs.resign_type = "Wrong"
gs.resign_movenum = movenum
else:
if verbose >= 2:
print("-", to_move, movenum, netwinrate, child_uctwinrate,
bestmovevisits)
if not gs.resign_type and child_uctwinrate < resignthr:
if verbose >= 2:
print("Correct resign -- %s" % (filename))
gs.resign_type = "Correct"
gs.resign_movenum = movenum
gs.total_moves = movenum
return gs
def parseGames(filenames, resignthr, verbose, prefixes):
gsd = {}
for filename in filenames:
training_filename = filename.replace(".debug", "")
with open(filename) as fh, open(training_filename) as tfh:
version = fh.readline().rstrip()
assert version == "2"
(cfg_resignpct, network) = fh.readline().split()
if prefixes:
net_name = os.path.basename(network)
matches = filter(lambda n: net_name.startswith(n), prefixes)
# Require at least one matching net prefix.
if not list(matches):
continue
cfg_resignpct = int(cfg_resignpct)
if cfg_resignpct == 0:
gsd[filename] = parseGameBody(filename, fh, tfh, verbose, resignthr)
elif verbose >= 2:
print("{} was played with -r {}, skipping".format(
filename, cfg_resignpct))
return gsd
def resignStats(gsd, resignthr):
# [ B wins, W wins, Overall ]
stats = [ TotalStats(), TotalStats(), TotalStats() ]
for gs in gsd.values():
stats[gs.winner].num_games += 1
if not gs.resign_type:
stats[gs.winner].no_resign_count += 1
stats[gs.winner].resigned_game_len_sum += gs.total_moves
elif gs.resign_type == "Correct":
stats[gs.winner].correct_resign_count += 1
stats[gs.winner].resigned_game_len_sum += gs.resign_movenum
else:
assert gs.resign_type == "Wrong"
stats[gs.winner].wrong_resign_count += 1
stats[gs.winner].resigned_game_len_sum += gs.resign_movenum
stats[gs.winner].game_len_sum += gs.total_moves
stats[2].calcOverall(stats[0], stats[1])
print("Resign thr: %0.2f - Black won %d/%d (%0.2f%%)" % (
resignthr,
stats[0].num_games,
stats[0].num_games+stats[1].num_games,
100 * stats[0].num_games / (stats[0].num_games+stats[1].num_games)))
for winner in (0,1,2):
win_str = 'Overall '
if winner==0:
win_str = 'Black wins'
elif winner==1:
win_str = 'White wins'
if stats[winner].num_games == 0:
print(" No games to report")
continue
avg_len = stats[winner].game_len_sum / stats[winner].num_games
resigned_avg_len = (stats[winner].resigned_game_len_sum /
stats[winner].num_games)
avg_reduction = (avg_len - resigned_avg_len) / avg_len
print(("%s - Wrong: %d/%d (%0.2f%%) Correct: %d/%d (%0.2f%%) "
"No Resign: %d/%d (%0.2f%%)") % (
win_str,
stats[winner].wrong_resign_count,
stats[winner].num_games,
100 * stats[winner].wrong_resign_count / stats[winner].num_games,
stats[winner].correct_resign_count,
stats[winner].num_games,
100 * stats[winner].correct_resign_count / stats[winner].num_games,
stats[winner].no_resign_count,
stats[winner].num_games,
100 * stats[winner].no_resign_count / stats[winner].num_games))
print("%s - Average game length: %d/%d (%0.2f%% reduction)" % (
win_str, resigned_avg_len, avg_len, avg_reduction*100))
print()
return stats
if __name__ == "__main__":
usage_str = """
This script analyzes the debug output from leelaz
to determine the impact of various resign thresholds.
Process flow:
Run autogtp with debug on:
autogtp -k savedir -d savedir
Unzip training and debug files:
gunzip savedir/*.gz
Analyze results with this script:
./resign_analysis.py savedir/*.debug.txt.0
Note the script takes the debug files hash.debug.txt.0
as the input arguments, but it also expects the training
files hash.txt.0 to be in the same directory."""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=usage_str)
default_resignthrs="0.5,0.2,0.15,0.1,0.05,0.02,0.01"
parser.add_argument(
"-r", metavar="Resign_thresholds", dest="resignthrs", type=str,
default=default_resignthrs,
help="comma separated resign thresholds (default {})".format(
default_resignthrs))
parser.add_argument(
"-R", metavar="Resign_rate", dest="resignrate", type=float,
help="If specified, a search is performed that finds the maximum \
resign threshold that can be set without exceeding the given \
resign rate")
parser.add_argument(
"-v", metavar="Verbose", dest="verbose", type=int, default=0,
help="Verbosity level (default 0)")
parser.add_argument(
"data", metavar="files", type=str, nargs="+",
help="Debug data files (*.debug.txt.0)")
parser.add_argument(
"-n", metavar="Prefix", dest="networks", nargs="+",
help="Prefixes of specific networks to analyze")
args = parser.parse_args()
resignthrs = [float(i) for i in args.resignthrs.split(",")]
if args.networks:
print("Analyzing networks starting with: {}".format(
",".join(args.networks)))
for resignthr in (resignthrs):
gsd = parseGames(args.data, resignthr, args.verbose, args.networks)
if gsd:
resignStats(gsd, resignthr)
else:
print("No games to analyze (for more info try running with -v 2)")
if args.resignrate:
L = 0.0
R = 0.5
while L < R :
resignthr = math.floor((L + R) * 50) / 100
gsd = parseGames(args.data, resignthr, args.verbose, args.networks)
if not gsd:
print("No games to analyze (for more info try running with -v 2)")
break
stats = resignStats(gsd, resignthr)
wrong_rate = stats[2].wrong_resign_count / stats[2].num_games
if wrong_rate > args.resignrate:
if R == resignthr:
R = (math.floor(resignthr * 100) - 1) / 100
else:
R = resignthr
else:
L = (math.floor(resignthr * 100) + 1) / 100
if (L == R):
print(("The highest the resign threshold should be set to: %0.2f")
% (R - 0.01))
|
