1 #include "chess.h"
2 #include "data.h"
3 /* last modified 01/09/15 */
4 /*
5 *******************************************************************************
6 * *
7 * Test() is used to test the program against a suite of test positions to *
8 * measure its performance on a particular machine, or to evaluate its skill *
9 * after modifying it in some way. *
10 * *
11 * The test is initiated by using the "test <filename>" command to read in *
12 * the suite of problems from file <filename>. The format of this file is *
13 * as follows: *
14 * *
15 * Setboard <forsythe-string>: This sets the board position using the usual *
16 * forsythe notation (see module SetBoard() in setc for a full ex- *
17 * planation of the syntax). *
18 * *
19 * Solution <move1> <move2> ... <moven>: this provides a solution move (or *
20 * set of solution moves if more than one is correct). If the search finds *
21 * one of these moves, then the prblem is counted as correct, otherwise it *
22 * is counted wrong. *
23 * *
24 * After reading these two lines, the program then searches to whatever time *
25 * or depth limit has been set, when it reaches the end-of-file condition or *
26 * when it reads a record containing the string "end" it then displays the *
27 * number correct and the number missed. *
28 * *
29 * There are two test modules here. Test() handles the specific Crafty test *
30 * data format (dates back to Cray Blitz days) while TestEPD() handles the *
31 * EPD-style test positions which is more concise. Other than the parsing *
32 * differences, these are identical modules. *
33 * *
34 *******************************************************************************
35 */
Test(char * filename,FILE * unsolved,int screen,int margin)36 void Test(char *filename, FILE * unsolved, int screen, int margin) {
37 TREE *const tree = block[0];
38 FILE *test_input;
39 uint64_t nodes = 0;
40 int i, move, right = 0, wrong = 0, correct, time = 0, len, nfailed = 0;
41 float avg_depth = 0.0;
42 char failed[8][4096], *eof, *delim;
43
44 /*
45 ************************************************************
46 * *
47 * Read in the position and then the solutions. After *
48 * executing a search to find the best move (according to *
49 * the program, anyway) compare it against the list of *
50 * solutions and count it right or wrong. *
51 * *
52 ************************************************************
53 */
54 if (!(test_input = fopen(filename, "r"))) {
55 printf("file %s does not exist.\n", filename);
56 return;
57 }
58 Print(4095, "\n");
59 eof = fgets(buffer, 4096, test_input);
60 if (!strstr(buffer, "title")) {
61 fclose(test_input);
62 TestEPD(filename, unsolved, screen, margin);
63 return;
64 }
65 if (book_file) {
66 fclose(book_file);
67 book_file = 0;
68 }
69 if (books_file) {
70 fclose(books_file);
71 books_file = 0;
72 }
73 fclose(test_input);
74 test_input = fopen(filename, "r");
75 while (FOREVER) {
76 eof = fgets(buffer, 4096, test_input);
77 strcpy(failed[nfailed++], buffer);
78 if (eof) {
79 delim = strchr(buffer, '\n');
80 if (delim)
81 *delim = 0;
82 delim = strchr(buffer, '\r');
83 if (delim)
84 *delim = ' ';
85 } else
86 break;
87 nargs = ReadParse(buffer, args, " \t;");
88 if (!strcmp(args[0], "end"))
89 break;
90 else if (!strcmp(args[0], "title")) {
91 Print(4095,
92 "=============================================="
93 "========================\n");
94 Print(4095, "! ");
95 len = 0;
96 for (i = 1; i < nargs; i++) {
97 Print(4095, "%s ", args[i]);
98 len += strlen(args[i]) + 1;
99 if (len > 65)
100 break;
101 }
102 for (i = len; i < 67; i++)
103 printf(" ");
104 Print(4095, "!\n");
105 Print(4095,
106 "=============================================="
107 "========================\n");
108 } else if (strcmp(args[0], "solution")) {
109 Option(tree);
110 } else {
111 number_of_solutions = 0;
112 solution_type = 0;
113 Print(4095, "solution ");
114 for (i = 1; i < nargs; i++) {
115 if (args[i][strlen(args[i]) - 1] == '?') {
116 solution_type = 1;
117 args[i][strlen(args[i]) - 1] = '\0';
118 } else if (*(args + i)[strlen(args[i]) - 1] == '!') {
119 solution_type = 0;
120 args[i][strlen(args[i]) - 1] = '\0';
121 }
122 move = InputMove(tree, 0, game_wtm, 0, 0, args[i]);
123 if (move) {
124 solutions[number_of_solutions] = move;
125 Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree,
126 0, game_wtm, move));
127 if (solution_type == 1)
128 Print(4095, "? ");
129 else
130 Print(4095, " ");
131 } else
132 DisplayChessBoard(stdout, tree->position);
133 }
134 Print(4095, "\n");
135 InitializeHashTables(0);
136 last_pv.pathd = 0;
137 thinking = 1;
138 tree->status[1] = tree->status[0];
139 Iterate(game_wtm, think, 0);
140 thinking = 0;
141 nodes += tree->nodes_searched;
142 avg_depth += (float) iteration;
143 time += (end_time - start_time);
144 correct = solution_type;
145 for (i = 0; i < number_of_solutions; i++) {
146 if (!solution_type) {
147 if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
148 correct = 1;
149 } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
150 correct = 0;
151 }
152 if (correct) {
153 right++;
154 Print(4095, "----------------------> solution correct (%d/%d).\n",
155 right, right + wrong);
156 } else {
157 wrong++;
158 Print(4095, "----------------------> solution incorrect (%d/%d).\n",
159 right, right + wrong);
160 if (unsolved)
161 for (i = 0; i < nfailed; i++)
162 fputs(failed[i], unsolved);
163 }
164 nfailed = 0;
165 }
166 }
167 /*
168 ************************************************************
169 * *
170 * Now print the results. *
171 * *
172 ************************************************************
173 */
174 if (right + wrong) {
175 Print(4095, "\n\n\n");
176 Print(4095, "test results summary:\n\n");
177 Print(4095, "total positions searched..........%12d\n", right + wrong);
178 Print(4095, "number right......................%12d\n", right);
179 Print(4095, "number wrong......................%12d\n", wrong);
180 Print(4095, "percentage right..................%12d\n",
181 right * 100 / (right + wrong));
182 Print(4095, "percentage wrong..................%12d\n",
183 wrong * 100 / (right + wrong));
184 Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes);
185 Print(4095, "average search depth..............%12.1f\n",
186 avg_depth / (right + wrong));
187 Print(4095, "nodes per second..................%12" PRIu64 "\n",
188 nodes * 100 / Max(time, 1));
189 Print(4095, "total time........................%12s\n",
190 DisplayTime(time));
191 }
192 input_stream = stdin;
193 early_exit = 99;
194 fclose(test_input);
195 }
196
197 /* last modified 06/26/15 */
198 /*
199 *******************************************************************************
200 * *
201 * TestEPD() is used to test the program against a suite of test positions *
202 * to measure its performance on a particular machine, or to evaluate its *
203 * skill after modifying it in some way. *
204 * *
205 * The test is initiated by using the "test <filename>" command to read in *
206 * the suite of problems from file <filename>. The format of this file is *
207 * as follows: *
208 * *
209 * <forsythe-string> am/bm move1 move2 etc; title "xxx" *
210 * *
211 * Am means "avoid move" and bm means "best move". Each test position may *
212 * have multiple moves to avoid or that are best, but both am and bm may not *
213 * appear on one position. *
214 * *
215 * The title is just a comment that is given in the program output to make *
216 * it easier to match output to specific positions. *
217 * *
218 * One new addition is the ability to take a set of EPD records and run a *
219 * search on each one. If the final evaluation is within some window, then *
220 * the input record is written out to a second file. This is used to screen *
221 * cluster-testing starting positions to weed out those that are so badly *
222 * unbalanced that one side always wins. *
223 * *
224 *******************************************************************************
225 */
TestEPD(char * filename,FILE * unsolved,int screen,int margin)226 void TestEPD(char *filename, FILE * unsolved, int screen, int margin) {
227 TREE *const tree = block[0];
228 FILE *test_input, *test_output = 0;
229 uint64_t nodes = 0;
230 int i, move, right = 0, wrong = 0, correct, time = 0, len, culled = 0, r =
231 0;
232 float avg_depth = 0.0;
233 char *eof, *mvs, *title, tbuffer[512], failed[4096];
234
235 /*
236 ************************************************************
237 * *
238 * Read in the position and then the solutions. After *
239 * executing a search to find the best move (according to *
240 * the program, anyway) compare it against the list of *
241 * solutions and count it right or wrong. *
242 * *
243 ************************************************************
244 */
245 if (!(test_input = fopen(filename, "r"))) {
246 printf("file %s does not exist.\n", filename);
247 return;
248 }
249 if (screen) {
250 char outfile[256];
251
252 strcpy(outfile, filename);
253 strcat(outfile, ".screened");
254 if (!(test_output = fopen(outfile, "w"))) {
255 printf("file %s cannot be opened for write.\n", filename);
256 return;
257 }
258 }
259 if (book_file) {
260 fclose(book_file);
261 book_file = 0;
262 }
263 if (books_file) {
264 fclose(books_file);
265 books_file = 0;
266 }
267 while (FOREVER) {
268 eof = fgets(buffer, 4096, test_input);
269 strcpy(failed, buffer);
270 Print(4095, "%s\n", buffer);
271 strcpy(tbuffer, buffer);
272 if (eof) {
273 char *delim;
274
275 delim = strchr(buffer, '\n');
276 if (delim)
277 *delim = 0;
278 delim = strchr(buffer, '\r');
279 if (delim)
280 *delim = ' ';
281 } else
282 break;
283 r++;
284 mvs = strstr(buffer, " sd ");
285 if (mvs) {
286 search_depth = atoi(mvs + 3);
287 *(mvs - 1) = 0;
288 Print(4095, "search depth %d\n", search_depth);
289 }
290 mvs = strstr(buffer, " bm ");
291 if (!mvs)
292 mvs = strstr(buffer, " am ");
293 if (!mvs && !screen)
294 Print(4095, "Warning. am/bm field missing, input string follows\n%s\n",
295 buffer);
296 if (mvs)
297 mvs++;
298 title = strstr(buffer, "id");
299 if (mvs)
300 *(mvs - 1) = 0;
301 if (title)
302 *(title - 1) = 0;
303 if (title) {
304 title = strchr(title, '\"') + 1;
305 if (title) {
306 if (strchr(title, '\"')) {
307 *strchr(title, '\"') = 0;
308 }
309 }
310 Print(4095,
311 "=============================================="
312 "========================\n");
313 Print(4095, "! ");
314 Print(4095, "%s ", title);
315 len = 66 - strlen(title);
316 for (i = 0; i < len; i++)
317 printf(" ");
318 Print(4095, "!\n");
319 Print(4095,
320 "=============================================="
321 "========================\n");
322 }
323 Option(tree);
324 if (mvs) {
325 nargs = ReadParse(mvs, args, " \t;");
326 number_of_solutions = 0;
327 solution_type = 0;
328 if (!strcmp(args[0], "am"))
329 solution_type = 1;
330 Print(4095, "solution ");
331 for (i = 1; i < nargs; i++) {
332 if (!strcmp(args[i], "c0"))
333 break;
334 move = InputMove(tree, 0, game_wtm, 0, 0, args[i]);
335 if (move) {
336 solutions[number_of_solutions] = move;
337 Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree,
338 0, game_wtm, move));
339 if (solution_type == 1)
340 Print(4095, "? ");
341 else
342 Print(4095, " ");
343 } else
344 DisplayChessBoard(stdout, tree->position);
345 }
346 }
347 Print(4095, "\n");
348 InitializeHashTables(0);
349 last_pv.pathd = 0;
350 thinking = 1;
351 tree->status[1] = tree->status[0];
352 Iterate(game_wtm, think, 0);
353 if (screen) {
354 if (Abs(last_root_value) < margin)
355 fwrite(tbuffer, 1, strlen(tbuffer), test_output);
356 else
357 culled++;
358 printf("record #%d, culled %d, score=%s \r", r, culled,
359 DisplayEvaluation(last_root_value, game_wtm));
360 fflush(stdout);
361 }
362 thinking = 0;
363 nodes += tree->nodes_searched;
364 avg_depth += (float) iteration;
365 time += (end_time - start_time);
366 if (!screen) {
367 correct = solution_type;
368 for (i = 0; i < number_of_solutions; i++) {
369 if (!solution_type) {
370 if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
371 correct = 1;
372 } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
373 correct = 0;
374 }
375 if (correct) {
376 right++;
377 Print(4095, "----------------------> solution correct (%d/%d).\n",
378 right, right + wrong);
379 } else {
380 wrong++;
381 Print(4095, "----------------------> solution incorrect (%d/%d).\n",
382 right, right + wrong);
383 if (unsolved)
384 fputs(failed, unsolved);
385 }
386 }
387 }
388 /*
389 ************************************************************
390 * *
391 * Now print the results. *
392 * *
393 ************************************************************
394 */
395 if (r) {
396 Print(4095, "\n\n\n");
397 Print(4095, "test results summary:\n\n");
398 Print(4095, "total positions searched..........%12d\n", r);
399 if (!screen) {
400 Print(4095, "number right......................%12d\n", right);
401 Print(4095, "number wrong......................%12d\n", wrong);
402 Print(4095, "percentage right..................%12d\n",
403 right * 100 / (right + wrong));
404 Print(4095, "percentage wrong..................%12d\n",
405 wrong * 100 / (right + wrong));
406 } else
407 Print(4095, "records excluded..................%12d\n", culled);
408
409 Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes);
410 Print(4095, "average search depth..............%12.1f\n", avg_depth / r);
411 Print(4095, "nodes per second..................%12" PRIu64 "\n",
412 nodes * 100 / Max(1, time));
413 Print(4095, "total time........................%12s\n",
414 DisplayTime(time));
415 }
416 input_stream = stdin;
417 early_exit = 99;
418 fclose(test_input);
419 }
420