1# -*- coding: UTF-8 -*- 2 3import shutil 4import collections 5import os 6from io import StringIO 7from os.path import getmtime 8import re 9import textwrap 10 11import pexpect 12 13from sqlalchemy import String 14 15from pychess.external.scoutfish import Scoutfish 16from pychess.external.chess_db import Parser 17 18from pychess.Utils.const import WHITE, BLACK, reprResult, FEN_START, FEN_EMPTY, \ 19 WON_RESIGN, DRAW, BLACKWON, WHITEWON, NORMALCHESS, DRAW_AGREE, FIRST_PAGE, PREV_PAGE, NEXT_PAGE, \ 20 ABORTED_REASONS, ADJOURNED_REASONS, ADJUDICATION_REASONS, WON_ADJUDICATION, DEATH_REASONS, CALLFLAG_REASONS, \ 21 RUNNING, TOOL_NONE, TOOL_CHESSDB, TOOL_SCOUTFISH 22from pychess.System import conf 23from pychess.System.Log import log 24from pychess.System.protoopen import PGN_ENCODING 25from pychess.System.prefix import getEngineDataPrefix 26from pychess.Utils.lutils.LBoard import LBoard 27from pychess.Utils.GameModel import GameModel 28from pychess.Utils.lutils.lmove import toSAN, parseAny, ParsingError 29from pychess.Utils.Move import Move 30from pychess.Utils.elo import get_elo_rating_change_pgn 31from pychess.Utils.logic import getStatus 32from pychess.Variants import name2variant, NormalBoard, variants 33from pychess.Utils import formatTime 34from pychess.Savers.ChessFile import ChessFile, LoadingError 35from pychess.Savers.database import col2label, TagDatabase, parseDateTag 36from pychess.System.cpu import get_cpu 37from pychess.Database import model as dbmodel 38from pychess.Database.PgnImport import TAG_REGEX, pgn2Const, PgnImport 39from pychess.Database.model import game, create_indexes, drop_indexes, metadata, ini_schema_version 40 41__label__ = _("Chess Game") 42__ending__ = "pgn" 43__append__ = True 44 45 46# token categories 47COMMENT_REST, COMMENT_BRACE, COMMENT_NAG, \ 48 VARIATION_START, VARIATION_END, \ 49 RESULT, FULL_MOVE, MOVE, MOVE_COMMENT = range(1, 10) 50 51pattern = re.compile(r""" 52 (\;.*?[\n\r]) # comment, rest of line style 53 |(\{.*?\}) # comment, between {} 54 |(\$[0-9]+) # comment, Numeric Annotation Glyph 55 |(\() # variation start 56 |(\)) # variation end 57 |(\*|1-0|0-1|1/2) # result (spec requires 1/2-1/2 for draw, but we want to tolerate simple 1/2 too) 58 |( 59 ([a-hKkQqRrBNnMmSsF][a-hxKQRBNMSF1-8+#=\-]{1,6} 60 |[PNBRQMSFK]@[a-h][1-8][+#]? # drop move 61 |o\-?o(?:\-?o)? # castling notation using letter 'o' with or without '-' 62 |O\-?O(?:\-?O)? # castling notation using letter 'O' with or without '-' 63 |0\-0(?:\-0)? # castling notation using zero with required '-' 64 |\-\-) # non standard '--' is used for null move inside variations 65 ([\?!]{1,2})* 66 ) # move (full, count, move with ?!, ?!) 67 """, re.VERBOSE | re.DOTALL) 68 69move_eval_re = re.compile(r"\[%eval\s+([+\-])?(?:#)?(\d+)(?:[,\.](\d{1,2}))?(?:/(\d{1,2}))?\]") 70move_time_re = re.compile(r"\[%emt\s+(\d:)?(\d{1,2}:)?(\d{1,4})(?:\.(\d{1,3}))?\]") 71 72# Chessbase style circles/arrows {[%csl Ra3][%cal Gc2c3,Rc3d4]} 73comment_circles_re = re.compile(r"\[%csl\s+((?:[RGBY]\w{2},?)+)\]") 74comment_arrows_re = re.compile(r"\[%cal\s+((?:[RGBY]\w{4},?)+)\]") 75 76# Mandatory tags (except "Result") 77mandatory_tags = ("Event", "Site", "Date", "Round", "White", "Black") 78 79 80def msToClockTimeTag(ms): 81 """ 82 Converts milliseconds to a chess clock time string in 'WhiteClock'/ 83 'BlackClock' PGN header format 84 """ 85 msec = ms % 1000 86 sec = ((ms - msec) % (1000 * 60)) / 1000 87 minute = ((ms - sec * 1000 - msec) % (1000 * 60 * 60)) / (1000 * 60) 88 hour = ((ms - minute * 1000 * 60 - sec * 1000 - msec) % 89 (1000 * 60 * 60 * 24)) / (1000 * 60 * 60) 90 return "%01d:%02d:%02d.%03d" % (hour, minute, sec, msec) 91 92 93def parseClockTimeTag(tag): 94 """ 95 Parses 'WhiteClock'/'BlackClock' PGN headers and returns the time the 96 player playing that color has left on their clock in milliseconds 97 """ 98 match = re.match(r"(\d{1,2}):(\d\d):(\d\d).(\d{1,3})", tag) 99 if match: 100 hour, minute, sec, msec = match.groups() 101 return int(msec) + int(sec) * 1000 + int(minute) * 60 * 1000 + int( 102 hour) * 60 * 60 * 1000 103 104 105def parseTimeControlTag(tag): 106 """ 107 Parses 'TimeControl' PGN header and returns the time and gain the 108 players have on game start in seconds 109 """ 110 match = re.match(r"^(\d+)\+?(\-?\d+)?$", tag) 111 if match: 112 secs, gain = match.groups() 113 return int(secs), int(gain) if gain is not None else 0, 0 114 else: 115 match = re.match(r"^(\d+)\/(\d+)$", tag) 116 if match: 117 moves, secs = match.groups() 118 return int(secs), 0, int(moves) 119 else: 120 return None 121 122 123def save(handle, model, position=None, flip=False): 124 """ Saves the game from GameModel to .pgn """ 125 processed_tags = [] 126 127 def write_tag(tag, value, roster=False): 128 nonlocal processed_tags 129 if tag in processed_tags or (not roster and not value): 130 return 131 try: 132 pval = str(value) 133 pval = pval.replace("\\", "\\\\") 134 pval = pval.replace("\"", "\\\"") 135 print('[%s "%s"]' % (tag, pval), file=handle) 136 except UnicodeEncodeError: 137 pval = bytes(pval, "utf-8").decode(PGN_ENCODING, errors="ignore") 138 print('[%s "%s"]' % (tag, pval), file=handle) 139 processed_tags = processed_tags + [tag] 140 141 # Mandatory ordered seven-tag roster 142 status = reprResult[model.status] 143 for tag in mandatory_tags: 144 value = model.tags[tag] 145 if tag == "Date": 146 y, m, d = parseDateTag(value) 147 y = "%04d" % y if y is not None else "????" 148 m = "%02d" % m if m is not None else "??" 149 d = "%02d" % d if d is not None else "??" 150 value = "%s.%s.%s" % (y, m, d) 151 elif value == "": 152 value = "?" 153 write_tag(tag, value, roster=True) 154 write_tag("Result", reprResult[model.status], roster=True) 155 156 # Variant 157 if model.variant.variant != NORMALCHESS: 158 write_tag("Variant", model.variant.cecp_name.capitalize()) 159 160 # Initial position 161 if model.boards[0].asFen() != FEN_START: 162 write_tag("SetUp", "1") 163 write_tag("FEN", model.boards[0].asFen()) 164 165 # Number of moves 166 write_tag("PlyCount", model.ply - model.lowply) 167 168 # Final position 169 if model.reason in ABORTED_REASONS: 170 value = "abandoned" 171 elif model.reason == WON_ADJUDICATION and model.isEngine2EngineGame(): 172 value = "rules infraction" 173 elif model.reason in ADJUDICATION_REASONS: 174 value = "adjudication" 175 elif model.reason in DEATH_REASONS: 176 value = "death" 177 elif model.reason in CALLFLAG_REASONS: 178 value = "time forfeit" 179 elif model.reason in ADJOURNED_REASONS or status == "*": 180 value = "unterminated" 181 else: 182 value = "normal" 183 write_tag("Termination", value) 184 185 # ELO and its variation 186 if conf.get("saveRatingChange"): 187 welo = model.tags["WhiteElo"] 188 belo = model.tags["BlackElo"] 189 if welo != "" and belo != "": 190 write_tag("WhiteRatingDiff", get_elo_rating_change_pgn(model, WHITE)) # Unofficial 191 write_tag("BlackRatingDiff", get_elo_rating_change_pgn(model, BLACK)) # Unofficial 192 193 # Time 194 if model.timed: 195 write_tag('WhiteClock', msToClockTimeTag(int(model.timemodel.getPlayerTime(WHITE) * 1000))) 196 write_tag('BlackClock', msToClockTimeTag(int(model.timemodel.getPlayerTime(BLACK) * 1000))) 197 198 # Write all the unprocessed tags 199 for tag in model.tags: 200 # Debug: print(">> %s = %s" % (tag, str(model.tags[tag]))) 201 write_tag(tag, model.tags[tag]) 202 203 # Discovery of the moves and comments 204 save_emt = conf.get("saveEmt") 205 save_eval = conf.get("saveEval") 206 result = [] 207 walk(model.boards[0].board, result, model, save_emt, save_eval) 208 209 # Alignment of the fetched elements 210 indented = conf.get("indentPgn") 211 if indented: 212 buffer = "" 213 depth = 0 214 crlf = False 215 for text in result: 216 # De/Indentation 217 crlf = (buffer[-1:] if len(buffer) > 0 else "") in ["\r", "\n"] 218 if text == "(": 219 depth += 1 220 if indented and not crlf: 221 buffer += os.linesep 222 crlf = True 223 # Space between each term 224 last = buffer[-1:] if len(buffer) > 0 else "" 225 crlf = last in ["\r", "\n"] 226 if not crlf and last != " " and last != "\t" and last != "(" and not text.startswith("\r") and not text.startswith("\n") and text != ")" and len(buffer) > 0: 227 buffer += " " 228 # New line for a new main move 229 if len(buffer) == 0 or (indented and depth == 0 and last != "\r" and last != "\n" and re.match(r"^[0-9]+\.", text) is not None): 230 buffer += os.linesep 231 crlf = True 232 # Alignment 233 if crlf and depth > 0: 234 for j in range(0, depth): 235 buffer += " " 236 # Term 237 buffer += text 238 if indented and text == ")": 239 buffer += os.linesep 240 crlf = True 241 depth -= 1 242 else: 243 # Add new line to separate tag section and movetext 244 print('', file=handle) 245 buffer = textwrap.fill(" ".join(result), width=80) 246 247 # Final 248 status = reprResult[model.status] 249 print(buffer, status, file=handle) 250 # Add new line to separate next game 251 print('', file=handle) 252 253 output = handle.getvalue() if isinstance(handle, StringIO) else "" 254 handle.close() 255 return output 256 257 258def walk(node, result, model, save_emt=False, save_eval=False, vari=False): 259 """Prepares a game data for .pgn storage. 260 Recursively walks the node tree to collect moves and comments 261 into a resulting movetext string. 262 263 Arguments: 264 node - list (a tree of lboards created by the pgn parser) 265 result - str (movetext strings)""" 266 267 while True: 268 if node is None: 269 break 270 271 # Initial game or variation comment 272 if node.prev is None: 273 for child in node.children: 274 if isinstance(child, str): 275 result.append("{%s}%s" % (child, os.linesep)) 276 node = node.next 277 continue 278 279 movecount = move_count(node, 280 black_periods=(save_emt or save_eval) and 281 "TimeControl" in model.tags) 282 if movecount is not None: 283 if movecount: 284 result.append(movecount) 285 move = node.lastMove 286 result.append(toSAN(node.prev, move)) 287 if (save_emt or save_eval) and not vari: 288 emt_eval = "" 289 if "TimeControl" in model.tags and save_emt: 290 elapsed = model.timemodel.getElapsedMoveTime( 291 node.plyCount - model.lowply) 292 emt_eval = "[%%emt %s]" % formatTime(elapsed, clk2pgn=True) 293 if node.plyCount in model.scores and save_eval: 294 moves, score, depth = model.scores[node.plyCount] 295 if node.color == BLACK: 296 score = -score 297 emt_eval += "[%%eval %0.2f/%s]" % (score / 100.0, depth) 298 if emt_eval: 299 result.append("{%s}" % emt_eval) 300 301 for nag in node.nags: 302 if nag: 303 result.append(nag) 304 305 for child in node.children: 306 if isinstance(child, str): 307 # comment 308 if child: 309 result.append("{%s}" % child) 310 else: 311 # variations 312 if node.fen_was_applied: 313 result.append("(") 314 walk(child[0], 315 result, 316 model, 317 save_emt, 318 save_eval, 319 vari=True) 320 result.append(")") 321 # variation after last played move is not valid pgn 322 # but we will save it as in comment 323 else: 324 result.append("{%s:" % _("Analyzer's primary variation")) 325 walk(child[0], 326 result, 327 model, 328 save_emt, 329 save_eval, 330 vari=True) 331 result.append("}") 332 333 if node.next: 334 node = node.next 335 else: 336 break 337 338 339def move_count(node, black_periods=False): 340 mvcount = None 341 if node.fen_was_applied: 342 ply = node.plyCount 343 if ply % 2 == 1: 344 mvcount = "%d." % (ply // 2 + 1) 345 # initial game move, or initial variation move 346 # it can be the same position as the main line! this is the reason using id() 347 elif node.prev.prev is None or id(node) != id( 348 node.prev.next) or black_periods: 349 mvcount = "%d..." % (ply // 2) 350 elif node.prev.children: 351 # move after real(not [%foo bar]) comment 352 need_mvcount = False 353 for child in node.prev.children: 354 if isinstance(child, str): 355 if not child.startswith("[%"): 356 need_mvcount = True 357 break 358 else: 359 need_mvcount = True 360 break 361 if need_mvcount: 362 mvcount = "%d..." % (ply // 2) 363 else: 364 mvcount = "" 365 else: 366 mvcount = "" 367 return mvcount 368 369 370def load(handle, progressbar=None): 371 return PGNFile(handle, progressbar) 372 373 374cpuinfo = get_cpu() 375MODERN = "-modern" if cpuinfo['popcnt'] else "" 376 377altpath = getEngineDataPrefix() 378 379scoutfish = "scoutfish_x%s%s%s" % (cpuinfo['bitness'], MODERN, cpuinfo['binext']) 380scoutfish_path = shutil.which(scoutfish, mode=os.X_OK, path=altpath) 381 382parser = "parser_x%s%s%s" % (cpuinfo['bitness'], MODERN, cpuinfo['binext']) 383chess_db_path = shutil.which(parser, mode=os.X_OK, path=altpath) 384 385 386class PGNFile(ChessFile): 387 def __init__(self, handle, progressbar=None): 388 ChessFile.__init__(self, handle) 389 self.handle = handle 390 self.progressbar = progressbar 391 self.pgn_is_string = isinstance(handle, StringIO) 392 393 if self.pgn_is_string: 394 self.games = [self.load_game_tags(), ] 395 else: 396 self.skip = 0 397 self.limit = 100 398 self.order_col = game.c.offset 399 self.is_desc = False 400 self.reset_last_seen() 401 402 # filter expressions to .sqlite .bin .scout 403 self.tag_query = None 404 self.fen = None 405 self.scout_query = None 406 407 self.scoutfish = None 408 self.chess_db = None 409 410 self.sqlite_path = os.path.splitext(self.path)[0] + '.sqlite' 411 self.engine = dbmodel.get_engine(self.sqlite_path) 412 self.tag_database = TagDatabase(self.engine) 413 414 self.games, self.offs_ply = self.get_records(0) 415 log.info("%s contains %s game(s)" % (self.path, self.count), extra={"task": "SQL"}) 416 417 def get_count(self): 418 """ Number of games in .pgn database """ 419 if self.pgn_is_string: 420 return len(self.games) 421 else: 422 return self.tag_database.count 423 count = property(get_count) 424 425 def get_size(self): 426 """ Size of .pgn file in bytes """ 427 return os.path.getsize(self.path) 428 size = property(get_size) 429 430 def close(self): 431 self.tag_database.close() 432 ChessFile.close(self) 433 434 def init_tag_database(self, importer=None): 435 """ Create/open .sqlite database of game header tags """ 436 # Import .pgn header tags to .sqlite database 437 438 sqlite_path = self.path.replace(".pgn", ".sqlite") 439 if os.path.isfile(self.path) and os.path.isfile(sqlite_path) and getmtime(self.path) > getmtime(sqlite_path): 440 metadata.drop_all(self.engine) 441 metadata.create_all(self.engine) 442 ini_schema_version(self.engine) 443 444 size = self.size 445 if size > 0 and self.tag_database.count == 0: 446 if size > 10000000: 447 drop_indexes(self.engine) 448 if self.progressbar is not None: 449 from gi.repository import GLib 450 GLib.idle_add(self.progressbar.set_text, _("Importing game headers...")) 451 if importer is None: 452 importer = PgnImport(self) 453 importer.initialize() 454 importer.do_import(self.path, progressbar=self.progressbar) 455 if size > 10000000 and not importer.cancel: 456 create_indexes(self.engine) 457 458 return importer 459 460 def init_chess_db(self): 461 """ Create/open polyglot .bin file with extra win/loss/draw stats 462 using chess_db parser from https://github.com/mcostalba/chess_db 463 """ 464 if chess_db_path is not None and self.path and self.size > 0: 465 try: 466 if self.progressbar is not None: 467 from gi.repository import GLib 468 GLib.idle_add(self.progressbar.set_text, _("Creating .bin index file...")) 469 self.chess_db = Parser(engine=(chess_db_path, )) 470 self.chess_db.open(self.path) 471 bin_path = os.path.splitext(self.path)[0] + '.bin' 472 if not os.path.isfile(bin_path): 473 log.debug("No valid games found in %s" % self.path) 474 self.chess_db = None 475 elif getmtime(self.path) > getmtime(bin_path): 476 self.chess_db.make() 477 except OSError as err: 478 self.chess_db = None 479 log.warning("Failed to sart chess_db parser. OSError %s %s" % (err.errno, err.strerror)) 480 except pexpect.TIMEOUT: 481 self.chess_db = None 482 log.warning("chess_db parser failed (pexpect.TIMEOUT)") 483 except pexpect.EOF: 484 self.chess_db = None 485 log.warning("chess_db parser failed (pexpect.EOF)") 486 487 def init_scoutfish(self): 488 """ Create/open .scout database index file to help querying 489 using scoutfish from https://github.com/mcostalba/scoutfish 490 """ 491 if scoutfish_path is not None and self.path and self.size > 0: 492 try: 493 if self.progressbar is not None: 494 from gi.repository import GLib 495 GLib.idle_add(self.progressbar.set_text, _("Creating .scout index file...")) 496 self.scoutfish = Scoutfish(engine=(scoutfish_path, )) 497 self.scoutfish.open(self.path) 498 scout_path = os.path.splitext(self.path)[0] + '.scout' 499 if getmtime(self.path) > getmtime(scout_path): 500 self.scoutfish.make() 501 except OSError as err: 502 self.scoutfish = None 503 log.warning("Failed to sart scoutfish. OSError %s %s" % (err.errno, err.strerror)) 504 except pexpect.TIMEOUT: 505 self.scoutfish = None 506 log.warning("scoutfish failed (pexpect.TIMEOUT)") 507 except pexpect.EOF: 508 self.scoutfish = None 509 log.warning("scoutfish failed (pexpect.EOF)") 510 511 def get_book_moves(self, fen): 512 """ Get move-games-win-loss-draw stat of fen position """ 513 rows = [] 514 if self.chess_db is not None: 515 move_stat = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen)) 516 for mstat in move_stat["moves"]: 517 rows.append((mstat["move"], int(mstat["games"]), int(mstat["wins"]), int(mstat["losses"]), int(mstat["draws"]))) 518 return rows 519 520 def has_position(self, fen): 521 # ChessDB (prioritary) 522 if self.chess_db is not None: 523 ret = self.chess_db.find("limit %s skip %s %s" % (1, 0, fen)) 524 if len(ret["moves"]) > 0: 525 return TOOL_CHESSDB, True 526 # Scoutfish (alternate by approximation) 527 if self.scoutfish is not None: 528 q = {"limit": 1, "skip": 0, "sub-fen": fen} 529 ret = self.scoutfish.scout(q) 530 if ret["match count"] > 0: 531 return TOOL_SCOUTFISH, True 532 return TOOL_NONE, False 533 534 def set_tag_order(self, order_col, is_desc): 535 self.order_col = order_col 536 self.is_desc = is_desc 537 self.tag_database.build_order_by(self.order_col, self.is_desc) 538 539 def reset_last_seen(self): 540 col_max = "ZZZ" if isinstance(self.order_col.type, String) else 2 ** 32 541 col_min = "" if isinstance(self.order_col.type, String) else -1 542 if self.is_desc: 543 self.last_seen = [(col_max, 2 ** 32)] 544 else: 545 self.last_seen = [(col_min, -1)] 546 547 def set_tag_filter(self, query): 548 """ Set (now prefixing) text and 549 create where clause we will use to query header tag .sqlite database 550 """ 551 self.tag_query = query 552 self.tag_database.build_where_tags(self.tag_query) 553 554 def set_fen_filter(self, fen): 555 """ Set fen string we will use to get game offsets from .bin database """ 556 if self.chess_db is not None and fen is not None and fen != FEN_START: 557 self.fen = fen 558 else: 559 self.fen = None 560 self.tag_database.build_where_offs8(None) 561 562 def set_scout_filter(self, query): 563 """ Set json string we will use to get game offsets from .scout database """ 564 if self.scoutfish is not None and query: 565 self.scout_query = query 566 else: 567 self.scout_query = None 568 self.tag_database.build_where_offs(None) 569 self.offs_ply = {} 570 571 def get_offs(self, skip, filtered_offs_list=None): 572 """ Get offsets from .scout database and 573 create where clause we will use to query header tag .sqlite database 574 """ 575 if self.scout_query: 576 limit = (10000 if self.tag_query else self.limit) + 1 577 self.scout_query["skip"] = skip 578 self.scout_query["limit"] = limit 579 move_stat = self.scoutfish.scout(self.scout_query) 580 581 offsets = [] 582 for mstat in move_stat["matches"]: 583 offs = mstat["ofs"] 584 if filtered_offs_list is None: 585 offsets.append(offs) 586 self.offs_ply[offs] = mstat["ply"][0] 587 elif offs in filtered_offs_list: 588 offsets.append(offs) 589 self.offs_ply[offs] = mstat["ply"][0] 590 591 if filtered_offs_list is not None: 592 # Continue scouting until we get enough good offset if needed 593 # print(0, move_stat["match count"], len(offsets)) 594 i = 1 595 while len(offsets) < self.limit and move_stat["match count"] == limit: 596 self.scout_query["skip"] = i * limit - 1 597 move_stat = self.scoutfish.scout(self.scout_query) 598 599 for mstat in move_stat["matches"]: 600 offs = mstat["ofs"] 601 if offs in filtered_offs_list: 602 offsets.append(offs) 603 self.offs_ply[offs] = mstat["ply"][0] 604 605 # print(i, move_stat["match count"], len(offsets)) 606 i += 1 607 608 if len(offsets) > self.limit: 609 self.tag_database.build_where_offs(offsets[:self.limit]) 610 else: 611 self.tag_database.build_where_offs(offsets) 612 613 def get_offs8(self, skip, filtered_offs_list=None): 614 """ Get offsets from .bin database and 615 create where clause we will use to query header tag .sqlite database 616 """ 617 if self.fen: 618 move_stat = self.chess_db.find("limit %s skip %s %s" % (self.limit, skip, self.fen)) 619 620 offsets = [] 621 for mstat in move_stat["moves"]: 622 offs = mstat["pgn offsets"] 623 if filtered_offs_list is None: 624 offsets += offs 625 elif offs in filtered_offs_list: 626 offsets += offs 627 628 if len(offsets) > self.limit: 629 self.tag_database.build_where_offs8(sorted(offsets)[:self.limit]) 630 else: 631 self.tag_database.build_where_offs8(sorted(offsets)) 632 633 def get_records(self, direction=FIRST_PAGE): 634 """ Get game header tag records from .sqlite database in paginated way """ 635 if direction == FIRST_PAGE: 636 self.skip = 0 637 self.reset_last_seen() 638 elif direction == NEXT_PAGE: 639 if not self.tag_query: 640 self.skip += self.limit 641 elif direction == PREV_PAGE: 642 if len(self.last_seen) == 2: 643 self.reset_last_seen() 644 elif len(self.last_seen) > 2: 645 self.last_seen = self.last_seen[:-2] 646 647 if not self.tag_query and self.skip >= self.limit: 648 self.skip -= self.limit 649 650 if self.fen: 651 self.reset_last_seen() 652 653 filtered_offs_list = None 654 if self.tag_query and (self.fen or self.scout_query): 655 filtered_offs_list = self.tag_database.get_offsets_for_tags(self.last_seen[-1]) 656 657 if self.fen: 658 self.get_offs8(self.skip, filtered_offs_list=filtered_offs_list) 659 660 if self.scout_query: 661 self.get_offs(self.skip, filtered_offs_list=filtered_offs_list) 662 # No game satisfied scout_query 663 if self.tag_database.where_offs is None: 664 return [], {} 665 666 records = self.tag_database.get_records(self.last_seen[-1], self.limit) 667 668 if records: 669 self.last_seen.append((records[-1][col2label[self.order_col]], records[-1]["Offset"])) 670 return records, self.offs_ply 671 else: 672 return [], {} 673 674 def load_game_tags(self): 675 """ Reads header tags from pgn if pgn is a one game only StringIO object """ 676 677 header = collections.defaultdict(str) 678 header["Id"] = 0 679 header["Offset"] = 0 680 for line in self.handle.readlines(): 681 line = line.strip() 682 if line.startswith('[') and line.endswith(']'): 683 tag_match = TAG_REGEX.match(line) 684 if tag_match: 685 value = tag_match.group(2) 686 value = value.replace("\\\"", "\"") 687 value = value.replace("\\\\", "\\") 688 header[tag_match.group(1)] = value 689 else: 690 break 691 return header 692 693 def loadToModel(self, rec, position=-1, model=None): 694 """ Parse game text and load game record header tags to a GameModel object """ 695 696 if model is None: 697 model = GameModel() 698 699 if self.pgn_is_string: 700 rec = self.games[0] 701 702 # Load mandatory tags 703 for tag in mandatory_tags: 704 model.tags[tag] = rec[tag] 705 706 # Load other tags 707 for tag in ('WhiteElo', 'BlackElo', 'ECO', 'TimeControl', 'Annotator'): 708 model.tags[tag] = rec[tag] 709 710 if self.pgn_is_string: 711 for tag in rec: 712 if isinstance(rec[tag], str) and rec[tag]: 713 model.tags[tag] = rec[tag] 714 else: 715 model.info = self.tag_database.get_info(rec) 716 extra_tags = self.tag_database.get_exta_tags(rec) 717 for et in extra_tags: 718 model.tags[et['tag_name']] = et['tag_value'] 719 720 if self.pgn_is_string: 721 variant = rec["Variant"].capitalize() 722 else: 723 variant = self.get_variant(rec) 724 725 if model.tags['TimeControl']: 726 tc = parseTimeControlTag(model.tags['TimeControl']) 727 if tc is not None: 728 secs, gain, moves = tc 729 model.timed = True 730 model.timemodel.secs = secs 731 model.timemodel.gain = gain 732 model.timemodel.minutes = secs / 60 733 model.timemodel.moves = moves 734 for tag, color in (('WhiteClock', WHITE), ('BlackClock', BLACK)): 735 if tag in model.tags: 736 try: 737 millisec = parseClockTimeTag(model.tags[tag]) 738 # We need to fix when FICS reports negative clock time like this 739 # [TimeControl "180+0"] 740 # [WhiteClock "0:00:15.867"] 741 # [BlackClock "23:59:58.820"] 742 start_sec = ( 743 millisec - 24 * 60 * 60 * 1000 744 ) / 1000. if millisec > 23 * 60 * 60 * 1000 else millisec / 1000. 745 model.timemodel.intervals[color][0] = start_sec 746 except ValueError: 747 raise LoadingError( 748 "Error parsing '%s'" % tag) 749 fenstr = rec["FEN"] 750 751 if variant: 752 if variant not in name2variant: 753 raise LoadingError("Unknown variant %s" % variant) 754 755 model.tags["Variant"] = variant 756 # Fixes for some non statndard Chess960 .pgn 757 if (fenstr is not None) and variant == "Fischerandom": 758 parts = fenstr.split() 759 parts[0] = parts[0].replace(".", "/").replace("0", "") 760 if len(parts) == 1: 761 parts.append("w") 762 parts.append("-") 763 parts.append("-") 764 fenstr = " ".join(parts) 765 766 model.variant = name2variant[variant] 767 board = LBoard(model.variant.variant) 768 else: 769 model.variant = NormalBoard 770 board = LBoard() 771 772 if fenstr: 773 try: 774 board.applyFen(fenstr) 775 model.tags["FEN"] = fenstr 776 except SyntaxError as err: 777 board.applyFen(FEN_EMPTY) 778 raise LoadingError( 779 _("The game can't be loaded, because of an error parsing FEN"), 780 err.args[0]) 781 else: 782 board.applyFen(FEN_START) 783 784 boards = [board] 785 786 del model.moves[:] 787 del model.variations[:] 788 789 self.error = None 790 movetext = self.get_movetext(rec) 791 boards = self.parse_movetext(movetext, boards[0], position) 792 793 # The parser built a tree of lboard objects, now we have to 794 # create the high level Board and Move lists... 795 796 for board in boards: 797 if board.lastMove is not None: 798 model.moves.append(Move(board.lastMove)) 799 800 self.has_emt = False 801 self.has_eval = False 802 803 def _create_board(model, node): 804 if node.prev is None: 805 # initial game board 806 board = model.variant(setup=node.asFen(), lboard=node) 807 else: 808 move = Move(node.lastMove) 809 try: 810 board = node.prev.pieceBoard.move(move, lboard=node) 811 except Exception: 812 raise LoadingError( 813 _("Invalid move."), 814 "%s%s" % (move_count(node, black_periods=True), move)) 815 816 return board 817 818 def walk(model, node, path): 819 boards = path 820 stack = [] 821 current = node 822 823 while current is not None: 824 board = _create_board(model, current) 825 boards.append(board) 826 stack.append(current) 827 current = current.next 828 else: 829 model.variations.append(list(boards)) 830 831 while stack: 832 current = stack.pop() 833 boards.pop() 834 for child in current.children: 835 if isinstance(child, list): 836 if len(child) > 1: 837 # non empty variation, go walk 838 walk(model, child[1], list(boards)) 839 else: 840 if not self.has_emt: 841 self.has_emt = child.find("%emt") >= 0 842 if not self.has_eval: 843 self.has_eval = child.find("%eval") >= 0 844 845 # Collect all variation paths into a list of board lists 846 # where the first one will be the boards of mainline game. 847 # model.boards will allways point to the current shown variation 848 # which will be model.variations[0] when we are in the mainline. 849 walk(model, boards[0], []) 850 model.boards = model.variations[0] 851 self.has_emt = self.has_emt and model.timed 852 if self.has_emt or self.has_eval: 853 if self.has_emt: 854 blacks = len(model.moves) // 2 855 whites = len(model.moves) - blacks 856 857 model.timemodel.intervals = [ 858 [model.timemodel.intervals[0][0]] * (whites + 1), 859 [model.timemodel.intervals[1][0]] * (blacks + 1), 860 ] 861 model.timemodel.intervals[0][0] = secs 862 model.timemodel.intervals[1][0] = secs 863 for ply, board in enumerate(boards): 864 for child in board.children: 865 if isinstance(child, str): 866 if self.has_emt: 867 match = move_time_re.search(child) 868 if match: 869 movecount, color = divmod(ply + 1, 2) 870 hour, minute, sec, msec = match.groups() 871 prev = model.timemodel.intervals[color][ 872 movecount - 1] 873 hour = 0 if hour is None else int(hour[:-1]) 874 minute = 0 if minute is None else int(minute[:-1]) 875 msec = 0 if msec is None else int(msec) 876 msec += int(sec) * 1000 + int(minute) * 60 * 1000 + int(hour) * 60 * 60 * 1000 877 model.timemodel.intervals[color][movecount] = prev - msec / 1000. + gain 878 879 if self.has_eval: 880 match = move_eval_re.search(child) 881 if match: 882 sign, num, fraction, depth = match.groups() 883 sign = 1 if sign is None or sign == "+" else -1 884 num = int(num) 885 fraction = 0 if fraction is None else int( 886 fraction) 887 value = sign * (num * 100 + fraction) 888 depth = "" if depth is None else depth 889 if board.color == BLACK: 890 value = -value 891 model.scores[ply] = ("", value, depth) 892 log.debug("pgn.loadToModel: intervals %s" % 893 model.timemodel.intervals) 894 895 # Find the physical status of the game 896 model.status, model.reason = getStatus(model.boards[-1]) 897 898 # Apply result from .pgn if the last position was loaded 899 if position == -1 or len(model.moves) == position - model.lowply: 900 if self.pgn_is_string: 901 result = rec["Result"] 902 if result in pgn2Const: 903 status = pgn2Const[result] 904 else: 905 status = RUNNING 906 else: 907 status = rec["Result"] 908 909 if status in (WHITEWON, BLACKWON) and status != model.status: 910 model.status = status 911 model.reason = WON_RESIGN 912 elif status == DRAW and status != model.status: 913 model.status = DRAW 914 model.reason = DRAW_AGREE 915 916 if model.timed: 917 model.timemodel.movingColor = model.boards[-1].color 918 919 # If parsing gave an error we throw it now, to enlarge our possibility 920 # of being able to continue the game from where it failed. 921 if self.error: 922 raise self.error 923 924 return model 925 926 def parse_movetext(self, string, board, position, variation=False): 927 """Recursive parses a movelist part of one game. 928 929 Arguments: 930 srting - str (movelist) 931 board - lboard (initial position) 932 position - int (maximum ply to parse) 933 variation- boolean (True if the string is a variation)""" 934 935 boards = [] 936 boards_append = boards.append 937 938 last_board = board 939 if variation: 940 # this board used only to hold initial variation comments 941 boards_append(LBoard(board.variant)) 942 else: 943 # initial game board 944 boards_append(board) 945 946 # status = None 947 parenthesis = 0 948 v_string = "" 949 v_last_board = None 950 for m in re.finditer(pattern, string): 951 group, text = m.lastindex, m.group(m.lastindex) 952 if parenthesis > 0: 953 v_string += ' ' + text 954 955 if group == VARIATION_END: 956 parenthesis -= 1 957 if parenthesis == 0: 958 if last_board.prev is None: 959 errstr1 = _("Error parsing %(mstr)s") % {"mstr": string} 960 self.error = LoadingError(errstr1, "") 961 return boards # , status 962 963 v_last_board.children.append( 964 self.parse_movetext(v_string[:-1], last_board.prev, position, variation=True)) 965 v_string = "" 966 continue 967 968 elif group == VARIATION_START: 969 parenthesis += 1 970 if parenthesis == 1: 971 v_last_board = last_board 972 973 if parenthesis == 0: 974 if group == FULL_MOVE: 975 if not variation: 976 if position != -1 and last_board.plyCount >= position: 977 break 978 979 mstr = m.group(MOVE) 980 try: 981 lmove = parseAny(last_board, mstr) 982 except ParsingError as err: 983 # TODO: save the rest as comment 984 # last_board.children.append(string[m.start():]) 985 notation, reason, boardfen = err.args 986 ply = last_board.plyCount 987 if ply % 2 == 0: 988 moveno = "%d." % (ply // 2 + 1) 989 else: 990 moveno = "%d..." % (ply // 2 + 1) 991 errstr1 = _( 992 "The game can't be read to end, because of an error parsing move %(moveno)s '%(notation)s'.") % { 993 'moveno': moveno, 994 'notation': notation} 995 errstr2 = _("The move failed because %s.") % reason 996 self.error = LoadingError(errstr1, errstr2) 997 break 998 except Exception: 999 ply = last_board.plyCount 1000 if ply % 2 == 0: 1001 moveno = "%d." % (ply // 2 + 1) 1002 else: 1003 moveno = "%d..." % (ply // 2 + 1) 1004 errstr1 = _( 1005 "Error parsing move %(moveno)s %(mstr)s") % { 1006 "moveno": moveno, 1007 "mstr": mstr} 1008 self.error = LoadingError(errstr1, "") 1009 break 1010 1011 new_board = last_board.clone() 1012 new_board.applyMove(lmove) 1013 1014 if m.group(MOVE_COMMENT): 1015 new_board.nags.append(symbol2nag(m.group( 1016 MOVE_COMMENT))) 1017 1018 new_board.prev = last_board 1019 1020 # set last_board next, except starting a new variation 1021 if variation and last_board == board: 1022 boards[0].next = new_board 1023 else: 1024 last_board.next = new_board 1025 1026 boards_append(new_board) 1027 last_board = new_board 1028 1029 elif group == COMMENT_REST: 1030 last_board.children.append(text[1:]) 1031 1032 elif group == COMMENT_BRACE: 1033 comm = text.replace('{\r\n', '{').replace('\r\n}', '}') 1034 # Preserve new lines of lichess study comments 1035 if self.path is not None and "lichess_study_" in self.path: 1036 comment = comm[1:-1] 1037 else: 1038 comm = comm[1:-1].splitlines() 1039 comment = ' '.join([line.strip() for line in comm]) 1040 if variation and last_board == board: 1041 # initial variation comment 1042 boards[0].children.append(comment) 1043 else: 1044 last_board.children.append(comment) 1045 1046 elif group == COMMENT_NAG: 1047 last_board.nags.append(text) 1048 1049 # TODO 1050 elif group == RESULT: 1051 # if text == "1/2": 1052 # status = reprResult.index("1/2-1/2") 1053 # else: 1054 # status = reprResult.index(text) 1055 break 1056 1057 else: 1058 print("Unknown:", text) 1059 1060 return boards # , status 1061 1062 def get_movetext(self, rec): 1063 self.handle.seek(rec["Offset"]) 1064 in_comment = False 1065 lines = [] 1066 line = self.handle.readline() 1067 if not line.strip(): 1068 line = self.handle.readline() 1069 1070 while line: 1071 # escape non-PGN data line 1072 if line.startswith("%"): 1073 line = self.handle.readline() 1074 continue 1075 1076 # header tag line 1077 if not in_comment and line.startswith("["): 1078 line = self.handle.readline() 1079 continue 1080 1081 # update in_comment state 1082 if (not in_comment and "{" in line) or (in_comment and "}" in line): 1083 in_comment = line.rfind("{") > line.rfind("}") 1084 1085 # if there is something add it 1086 if line.strip(): 1087 if not self.pgn_is_string and self.handle.pgn_encoding != PGN_ENCODING: 1088 line = line.encode(PGN_ENCODING).decode(self.handle.pgn_encoding) 1089 lines.append(line) 1090 line = self.handle.readline() 1091 # if line is empty it should be the game separator line except... 1092 elif len(lines) == 0 or in_comment: 1093 if in_comment: 1094 lines.append(line) 1095 line = self.handle.readline() 1096 else: 1097 break 1098 return "".join(lines) 1099 1100 def get_variant(self, rec): 1101 variant = rec["Variant"] 1102 return variants[variant].cecp_name.capitalize() if variant else "" 1103 1104 1105nag2symbolDict = { 1106 "$0": "", 1107 "$1": "!", 1108 "$2": "?", 1109 "$3": "!!", 1110 "$4": "??", 1111 "$5": "!?", 1112 "$6": "?!", 1113 "$7": "□", # forced move 1114 "$8": "□", 1115 "$9": "??", 1116 "$10": "=", 1117 "$11": "=", 1118 "$12": "=", 1119 "$13": "∞", # unclear 1120 "$14": "+=", 1121 "$15": "=+", 1122 "$16": "±", 1123 "$17": "∓", 1124 "$18": "+-", 1125 "$19": "-+", 1126 "$20": "+--", 1127 "$21": "--+", 1128 "$22": "⨀", # zugzwang 1129 "$23": "⨀", 1130 "$24": "◯", # space 1131 "$25": "◯", 1132 "$26": "◯", 1133 "$27": "◯", 1134 "$28": "◯", 1135 "$29": "◯", 1136 "$32": "⟳", # development 1137 "$33": "⟳", 1138 "$36": "↑", # initiative 1139 "$37": "↑", 1140 "$40": "→", # attack 1141 "$41": "→", 1142 "$44": "~=", # compensation 1143 "$45": "=~", 1144 "$132": "⇆", # counterplay 1145 "$133": "⇆", 1146 "$136": "⨁", # time 1147 "$137": "⨁", 1148 "$138": "⨁", 1149 "$139": "⨁", 1150 "$140": "∆", # with the idea 1151 "$141": "∇", # aimed against 1152 "$142": "⌓", # better is 1153 "$146": "N", # novelty 1154} 1155 1156symbol2nagDict = {} 1157for k, v in nag2symbolDict.items(): 1158 if v not in symbol2nagDict: 1159 symbol2nagDict[v] = k 1160 1161 1162def nag2symbol(nag): 1163 return nag2symbolDict.get(nag, nag) 1164 1165 1166def symbol2nag(symbol): 1167 return symbol2nagDict[symbol] 1168