1import os
2import re
3import json
4from urllib.request import Request, urlopen
5from urllib.parse import urlparse, parse_qs, unquote, urlencode
6from html import unescape
7from html.parser import HTMLParser
8import asyncio
9import websockets
10from base64 import b64decode
11import string
12from random import choice, randint
13
14from pychess import VERSION
15from pychess.Utils.const import CRAZYHOUSECHESS, FISCHERRANDOMCHESS, reprResult
16from pychess.Utils.lutils.LBoard import LBoard
17from pychess.Utils.lutils.lmove import parseAny, toSAN
18from pychess.System.Log import log
19
20# import pdb
21# def _(p):
22#     return p
23# VERSION = '1.0'
24# import ssl
25# ssl._create_default_https_context = ssl._create_unverified_context  # Chess24, ICCF
26
27
28TYPE_NONE, TYPE_GAME, TYPE_STUDY, TYPE_PUZZLE, TYPE_EVENT = range(5)
29CHESS960 = 'Fischerandom'
30DEFAULT_BOARD = 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1'
31
32CAT_DL = _('Download link')
33CAT_HTML = _('HTML parsing')
34CAT_API = _('Application programming interface')
35CAT_MISC = _('Various techniques')
36CAT_WS = _('Websockets')
37
38
39# Abstract class to download a game from the Internet
40class InternetGameInterface:
41    # Internal
42    def __init__(self):
43        ''' Initialize the common data that can be used in ALL the sub-classes. '''
44        self.id = None
45        self.allow_extra = False
46        self.userAgent = 'PyChess %s' % VERSION
47        self.use_an = True  # To rebuild a readable PGN where possible
48
49    def is_enabled(self):
50        ''' To disable a chess provider temporarily, override this method in the sub-class. '''
51        return True
52
53    def get_game_id(self):
54        ''' Return the unique identifier of the game that was detected after a successful call to assign_game().
55            The value is None if no game was found earlier. '''
56        return self.id
57
58    def reacts_to(self, url, host):
59        ''' Return True if the URL belongs to the HOST. The sub-domains other than "www" are not supported.
60            The method is used to accept any URL when a unique identifier cannot be extracted by assign_game(). '''
61        # Verify the hostname
62        if url is None:
63            return False
64        parsed = urlparse(url)
65        if parsed.netloc.lower() not in ['www.' + host.lower(), host.lower()]:
66            return False
67
68        # Any page is valid
69        self.id = url
70        return True
71
72    def json_loads(self, data):
73        ''' Load a JSON and handle the errors.
74            The value None is returned when the data are not relevant or misbuilt. '''
75        try:
76            if data in [None, '']:
77                return None
78            return json.loads(data)
79        except ValueError:
80            return None
81
82    def json_field(self, data, path, default=''):
83        ''' Conveniently read a field from a JSON data. The PATH is a key like "node1/node2/key".
84            A blank string is returned in case of error. '''
85        if data in [None, '']:
86            return ''
87        keys = path.split('/')
88        value = data
89        for key in keys:
90            if key.startswith('[') and key.endswith(']'):
91                try:
92                    value = value[int(key[1:-1])]
93                except (ValueError, TypeError, IndexError):
94                    return ''
95            else:
96                if key in value:
97                    value = value[key]
98                else:
99                    return ''
100        return default if value in [None, ''] else value
101
102    def read_data(self, response):
103        ''' Read the data from an HTTP request and execute the charset conversion.
104            The value None is returned in case of error. '''
105        # Check
106        if response is None:
107            return None
108        bytes = response.read()
109
110        # Decode
111        cs = response.info().get_content_charset()
112        if cs is not None:
113            data = bytes.decode(cs)
114        else:
115            try:
116                data = bytes.decode('utf-8')
117            except Exception:
118                try:
119                    data = bytes.decode('latin-1')
120                except Exception:
121                    log.debug('Error in the decoding of the data')
122                    return None
123
124        # Result
125        data = data.replace("\ufeff", '').replace("\r", '').strip()
126        if data == '':
127            return None
128        else:
129            return data
130
131    def download(self, url, userAgent=False):
132        ''' Download the URL from the Internet.
133            The USERAGENT is requested by some websites to make sure that you are not a bot.
134            The value None is returned in case of error. '''
135        # Check
136        if url in [None, '']:
137            return None
138
139        # Download
140        try:
141            log.debug('Downloading game: %s' % url)
142            if userAgent:
143                req = Request(url, headers={'User-Agent': self.userAgent})
144                response = urlopen(req)
145            else:
146                response = urlopen(url)
147            return self.read_data(response)
148        except Exception as exception:
149            log.debug('Exception raised: %s' % str(exception))
150            return None
151
152    def download_list(self, links, userAgent=False):
153        ''' Download and concatenate the URL given in the array LINKS.
154            The USERAGENT is requested by some websites to make sure that you are not a bot.
155            The number of downloads is limited to 10.
156            The downloads that failed are dropped silently.
157            The value None is returned in case of no data or error. '''
158        pgn = ''
159        for i, link in enumerate(links):
160            data = self.download(link, userAgent)
161            if data not in [None, '']:
162                pgn += '%s\n\n' % data
163            if i >= 10:                             # Anti-flood
164                break
165        if pgn == '':
166            return None
167        else:
168            return pgn
169
170    def async_from_sync(self, coro):
171        ''' The method is used for the WebSockets technique to call an asynchronous task from a synchronous task. '''
172        # TODO Not working under Linux while PyChess GUI is running
173        curloop = asyncio.get_event_loop()
174        loop = asyncio.new_event_loop()
175        asyncio.set_event_loop(loop)
176        result = loop.run_until_complete(coro)
177        loop.close()
178        asyncio.set_event_loop(curloop)
179        return result
180
181    def send_xhr(self, url, postData, userAgent=False):
182        ''' Call a target URL by submitting the POSTDATA.
183            The USERAGENT is requested by some websites to make sure that you are not a bot.
184            The value None is returned in case of error. '''
185        # Check
186        if url in [None, ''] or postData in [None, '']:
187            return None
188
189        # Call data
190        try:
191            log.debug('Calling API: %s' % url)
192            if userAgent:
193                req = Request(url, urlencode(postData).encode(), headers={'User-Agent': self.userAgent})
194            else:
195                req = Request(url, urlencode(postData).encode())
196            response = urlopen(req)
197            return self.read_data(response)
198        except Exception as exception:
199            log.debug('Exception raised: %s' % str(exception))
200            return None
201
202    def rebuild_pgn(self, game):
203        ''' Return an object in PGN format.
204            The keys starting with "_" are dropped silently.
205            The key "_url" becomes the first comment.
206            The key "_moves" contains the moves.
207            The key "_reason" becomes the last comment. '''
208        # Check
209        if game is None or game == '' or '_moves' not in game or game['_moves'] == '':
210            return None
211
212        # Header
213        pgn = ''
214        for e in game:
215            if e[:1] != '_' and game[e] not in [None, '']:
216                pgn += '[%s "%s"]\n' % (e, game[e])
217        if pgn == '':
218            pgn = '[Annotator "PyChess %s"]\n' % VERSION
219        pgn += "\n"
220
221        # Body
222        if '_url' in game:
223            pgn += "{%s}\n" % game['_url']
224        if '_moves' in game:
225            pgn += '%s ' % game['_moves']
226        if '_reason' in game:
227            pgn += '{%s} ' % game['_reason']
228        if 'Result' in game:
229            pgn += '%s ' % game['Result']
230        return pgn.strip()
231
232    def sanitize(self, pgn):
233        ''' Modify the PGN output to comply with the expected format '''
234        # Check
235        if pgn in [None, '']:
236            return None
237
238        # Verify that it starts with the correct magic character (ex.: "<" denotes an HTML content, "[" a chess game, etc...)
239        pgn = pgn.strip()
240        if not pgn.startswith('['):
241            return None
242
243        # Reorganize the spaces to bypass Scoutfish's limitation
244        while (True):
245            lc = len(pgn)
246            pgn = pgn.replace("\n\n\n", "\n\n")
247            if len(pgn) == lc:
248                break
249
250        # Extract the first game
251        pos = pgn.find("\n\n[")  # TODO Support in-memory database to load several games at once
252        if pos != -1:
253            pgn = pgn[:pos]
254
255        # Return the PGN with the local crlf
256        return pgn.replace("\n", os.linesep)
257
258    def stripHtml(self, input):
259        ''' This method removes any HTML mark from the input parameter '''
260        rxp = re.compile(r'<\/?[^<]+>', re.IGNORECASE)
261        return rxp.sub('', input)
262
263    # External
264    def get_description(self):
265        ''' (Abstract) Name of the chess provider written as "Chess provider -- Technique used" '''
266        pass
267
268    def assign_game(self, url):
269        ''' (Abstract) Detect the unique identifier of URL '''
270        pass
271
272    def download_game(self):
273        ''' (Abstract) Download the game identified earlier by assign_game() '''
274        pass
275
276
277# Lichess.org
278class InternetGameLichess(InternetGameInterface):
279    def __init__(self):
280        InternetGameInterface.__init__(self)
281        self.url_type = TYPE_NONE
282        self.url_tld = 'org'
283
284    def get_description(self):
285        return 'Lichess.org -- %s' % CAT_DL
286
287    def assign_game(self, url):
288        # Retrieve the ID of the broadcast
289        rxp = re.compile(r'^https?:\/\/([\S]+\.)?lichess\.(org|dev)\/broadcast\/[a-z0-9\-]+\/([a-z0-9]+)[\/\?\#]?', re.IGNORECASE)
290        m = rxp.match(url)
291        if m is not None:
292            gid = m.group(3)
293            if len(gid) == 8:
294                self.url_type = TYPE_STUDY
295                self.id = gid
296                self.url_tld = m.group(2)
297                return True
298
299        # Retrieve the ID of the practice
300        rxp = re.compile(r'^https?:\/\/([\S]+\.)?lichess\.(org|dev)\/practice\/[\w\-\/]+\/([a-z0-9]+\/[a-z0-9]+)(\.pgn)?\/?([\S\/]+)?$', re.IGNORECASE)
301        m = rxp.match(url)
302        if m is not None:
303            gid = m.group(3)
304            if len(gid) == 17:
305                self.url_type = TYPE_STUDY
306                self.id = gid
307                self.url_tld = m.group(2)
308                return True
309
310        # Retrieve the ID of the study
311        rxp = re.compile(r'^https?:\/\/([\S]+\.)?lichess\.(org|dev)\/study\/([a-z0-9]+(\/[a-z0-9]+)?)(\.pgn)?\/?([\S\/]+)?$', re.IGNORECASE)
312        m = rxp.match(url)
313        if m is not None:
314            gid = m.group(3)
315            if len(gid) in [8, 17]:
316                self.url_type = TYPE_STUDY
317                self.id = gid
318                self.url_tld = m.group(2)
319                return True
320
321        # Retrieve the ID of the puzzle
322        rxp = re.compile(r'^https?:\/\/([\S]+\.)?lichess\.(org|dev)\/training\/([0-9]+|daily)[\/\?\#]?', re.IGNORECASE)
323        m = rxp.match(url)
324        if m is not None:
325            gid = m.group(3)
326            if (gid.isdigit() and gid != '0') or gid == 'daily':
327                self.url_type = TYPE_PUZZLE
328                self.id = gid
329                self.url_tld = m.group(2)
330                return True
331
332        # Retrieve the ID of the game
333        rxp = re.compile(r'^https?:\/\/([\S]+\.)?lichess\.(org|dev)\/(game\/export\/|embed\/)?([a-z0-9]+)\/?([\S\/]+)?$', re.IGNORECASE)  # More permissive
334        m = rxp.match(url)
335        if m is not None:
336            gid = m.group(4)
337            if len(gid) == 8:
338                self.url_type = TYPE_GAME
339                self.id = gid
340                self.url_tld = m.group(2)
341                return True
342
343        # Nothing found
344        return False
345
346    def query_api(self, path):
347        response = urlopen(Request('https://lichess.%s%s' % (self.url_tld, path), headers={'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/vnd.lichess.v4+json'}))
348        bourne = self.read_data(response)
349        return self.json_loads(bourne)
350
351    def download_game(self):
352        # Check
353        if None in [self.id, self.url_tld]:
354            return None
355
356        # Logic for the games
357        if self.url_type == TYPE_GAME:
358            # Download the finished game
359            api = self.query_api('/import/master/%s/white' % self.id)
360            game = self.json_field(api, 'game')
361            if 'winner' in game:
362                url = 'https://lichess.%s/game/export/%s?literate=1' % (self.url_tld, self.id)
363                return self.download(url)
364            else:
365                if not self.allow_extra and game['rated']:
366                    return None
367
368            # Rebuild the PGN file
369            game = {}
370            game['_url'] = 'https://lichess.%s%s' % (self.url_tld, self.json_field(api, 'url/round'))
371            game['Variant'] = self.json_field(api, 'game/variant/name')
372            game['FEN'] = self.json_field(api, 'game/initialFen')
373            game['SetUp'] = '1'
374            game['White'] = self.json_field(api, 'player/name', self.json_field(api, 'player/user/username', 'Anonymous'))
375            game['WhiteElo'] = self.json_field(api, 'player/rating')
376            game['Black'] = self.json_field(api, 'opponent/name', self.json_field(api, 'opponent/user/username', 'Anonymous'))
377            game['BlackElo'] = self.json_field(api, 'opponent/rating')
378            if self.json_field(api, 'clock') != '':
379                game['TimeControl'] = '%d+%d' % (self.json_field(api, 'clock/initial'), self.json_field(api, 'clock/increment'))
380            else:
381                game['TimeControl'] = '%dd' % (self.json_field(api, 'correspondence/increment') // 86400)
382            game['Result'] = '*'
383            game['_moves'] = ''
384            moves = self.json_field(api, 'steps')
385            for move in moves:
386                if move['ply'] > 0:
387                    game['_moves'] += ' %s' % move['san']
388            return self.rebuild_pgn(game)
389
390        # Logic for the studies
391        elif self.url_type == TYPE_STUDY:
392            url = 'https://lichess.%s/study/%s.pgn' % (self.url_tld, self.id)
393            return self.download(url, userAgent=True)
394
395        # Logic for the puzzles
396        elif self.url_type == TYPE_PUZZLE:
397            # The API doesn't provide the history of the moves
398            # chessgame = self.query_api('/training/%s/load' % self.id)
399
400            # Fetch the puzzle
401            url = 'https://lichess.%s/training/%s' % (self.url_tld, self.id)
402            page = self.download(url)
403            if page is None:
404                return None
405
406            # Extract the JSON
407            page = page.replace("\n", '')
408            pos1 = page.find("lichess.puzzle =")
409            if pos1 == -1:
410                return None
411            pos1 = page.find('"game"', pos1 + 1)
412            if pos1 == -1:
413                return None
414            c = 1
415            pos2 = pos1
416            while pos2 < len(page):
417                pos2 += 1
418                if page[pos2] == '{':
419                    c += 1
420                if page[pos2] == '}':
421                    c -= 1
422                if c == 0:
423                    break
424            if c != 0:
425                return None
426
427            # Header
428            bourne = page[pos1 - 1:pos2 + 1]
429            chessgame = self.json_loads(bourne)
430            puzzle = self.json_field(chessgame, 'puzzle')
431            if puzzle == '':
432                return None
433            game = {}
434            game['_url'] = 'https://lichess.%s/%s#%s' % (self.url_tld, self.json_field(puzzle, 'gameId'), self.json_field(puzzle, 'initialPly'))
435            game['Site'] = 'lichess.%s' % self.url_tld
436            rating = self.json_field(puzzle, 'rating')
437            game['Event'] = 'Puzzle %d, rated %s' % (self.json_field(puzzle, 'id'), rating)
438            game['Result'] = '*'
439            game['X_ID'] = self.json_field(puzzle, 'id')
440            game['X_TimeControl'] = self.json_field(chessgame, 'game/clock')
441            game['X_Rating'] = rating
442            game['X_Attempts'] = self.json_field(puzzle, 'attempts')
443            game['X_Vote'] = self.json_field(puzzle, 'vote')
444
445            # Players
446            players = self.json_field(chessgame, 'game/players')
447            if not isinstance(players, list):
448                return None
449            for p in players:
450                if p['color'] == 'white':
451                    t = 'White'
452                elif p['color'] == 'black':
453                    t = 'Black'
454                else:
455                    return None
456                pos1 = p['name'].find(' (')
457                if pos1 == -1:
458                    game[t] = p['name']
459                else:
460                    game[t] = p['name'][:pos1]
461                    game[t + 'Elo'] = p['name'][pos1 + 2:-1]
462
463            # Moves
464            moves = self.json_field(chessgame, 'game/treeParts')
465            if not isinstance(moves, list):
466                return None
467            game['_moves'] = ''
468            for m in moves:
469                if m['ply'] in [0, '0']:
470                    game['SetUp'] = '1'
471                    game['FEN'] = m['fen']
472                else:
473                    game['_moves'] += '%s ' % m['san']
474
475            # Solution
476            game['_moves'] += ' {Solution: '
477            puzzle = self.json_field(puzzle, 'branch')
478            while True:
479                game['_moves'] += '%s ' % self.json_field(puzzle, 'san')
480                puzzle = self.json_field(puzzle, 'children')
481                if len(puzzle) == 0:
482                    break
483                puzzle = puzzle[0]
484            game['_moves'] += '}'
485
486            # Rebuild the PGN game
487            return self.rebuild_pgn(game)
488
489        else:
490            assert(False)
491
492
493# ChessGames.com
494class InternetGameChessgames(InternetGameInterface):
495    def __init__(self):
496        InternetGameInterface.__init__(self)
497        self.computer = False
498
499    def get_description(self):
500        return 'ChessGames.com -- %s' % CAT_DL
501
502    def assign_game(self, url):
503        # Verify the hostname
504        parsed = urlparse(url)
505        if parsed.netloc.lower() not in ['www.chessgames.com', 'chessgames.com']:
506            return False
507
508        # Read the arguments
509        args = parse_qs(parsed.query)
510        if 'gid' in args:
511            gid = args['gid'][0]
512            if gid.isdigit() and gid != '0':
513                self.id = gid
514                self.computer = ('comp' in args) and (args['comp'][0] == '1')
515                return True
516        return False
517
518    def download_game(self):
519        # Check
520        if self.id is None:
521            return None
522
523        # First try with computer analysis
524        url = 'http://www.chessgames.com/pgn/pychess.pgn?gid=' + self.id
525        if self.computer:
526            pgn = self.download(url + '&comp=1')
527            if pgn in [None, ''] or 'NO SUCH GAME' in pgn:
528                self.computer = False
529            else:
530                return pgn
531
532        # Second try without computer analysis
533        if not self.computer:
534            pgn = self.download(url)
535            if pgn in [None, ''] or 'NO SUCH GAME' in pgn:
536                return None
537            else:
538                return pgn
539
540
541# FicsGames.org
542class InternetGameFicsgames(InternetGameInterface):
543    def get_description(self):
544        return 'FicsGames.org -- %s' % CAT_DL
545
546    def assign_game(self, url):
547        # Verify the URL
548        parsed = urlparse(url)
549        if parsed.netloc.lower() not in ['www.ficsgames.org', 'ficsgames.org'] or 'show' not in parsed.path.lower():
550            return False
551
552        # Read the arguments
553        args = parse_qs(parsed.query)
554        if 'ID' in args:
555            gid = args['ID'][0]
556            if gid.isdigit() and gid != '0':
557                self.id = gid
558                return True
559        return False
560
561    def download_game(self):
562        # Check
563        if self.id is None:
564            return None
565
566        # Download
567        pgn = self.download('http://ficsgames.org/cgi-bin/show.cgi?ID=%s;action=save' % self.id)
568        if pgn in [None, ''] or 'not found in GGbID' in pgn:
569            return None
570        else:
571            return pgn
572
573
574# ChessTempo.com
575class InternetGameChesstempo(InternetGameInterface):
576    def get_description(self):
577        return 'ChessTempo.com -- %s' % CAT_DL
578
579    def assign_game(self, url):
580        rxp = re.compile(r'^https?:\/\/(\S+\.)?chesstempo\.com\/gamedb\/game\/(\d+)\/?([\S\/]+)?$', re.IGNORECASE)
581        m = rxp.match(url)
582        if m is not None:
583            gid = str(m.group(2))
584            if gid.isdigit() and gid != '0':
585                self.id = gid
586                return True
587        return False
588
589    def download_game(self):
590        # Check
591        if self.id is None:
592            return None
593
594        # Download
595        pgn = self.download('http://chesstempo.com/requests/download_game_pgn.php?gameids=%s' % self.id, userAgent=True)  # Else a random game is retrieved
596        if pgn is None or len(pgn) <= 128:
597            return None
598        else:
599            return pgn
600
601
602# Chess24.com
603class InternetGameChess24(InternetGameInterface):
604    def get_description(self):
605        return 'Chess24.com -- %s' % CAT_HTML
606
607    def assign_game(self, url):
608        rxp = re.compile(r'^https?:\/\/chess24\.com\/[a-z]+\/(analysis|game|download-game)\/([a-z0-9\-_]+)[\/\?\#]?', re.IGNORECASE)
609        m = rxp.match(url)
610        if m is not None:
611            gid = str(m.group(2))
612            if len(gid) == 22:
613                self.id = gid
614                return True
615        return False
616
617    def download_game(self):
618        # Download the page
619        if self.id is None:
620            return None
621        url = 'https://chess24.com/en/game/%s' % self.id
622        page = self.download(url, userAgent=True)  # Else HTTP 403 Forbidden
623        if page is None:
624            return None
625
626        # Extract the JSON of the game
627        lines = page.split("\n")
628        for line in lines:
629            line = line.strip()
630            pos1 = line.find('.initGameSession({')
631            pos2 = line.find('});', pos1)
632            if -1 in [pos1, pos2]:
633                continue
634
635            # Read the game from JSON
636            bourne = self.json_loads(line[pos1 + 17:pos2 + 1])
637            chessgame = self.json_field(bourne, 'chessGame')
638            moves = self.json_field(chessgame, 'moves')
639            if '' in [chessgame, moves]:
640                continue
641
642            # Build the header of the PGN file
643            game = {}
644            game['_moves'] = ''
645            game['_url'] = url
646            game['Event'] = self.json_field(chessgame, 'meta/Event')
647            game['Site'] = self.json_field(chessgame, 'meta/Site')
648            game['Date'] = self.json_field(chessgame, 'meta/Date')
649            game['Round'] = self.json_field(chessgame, 'meta/Round')
650            game['White'] = self.json_field(chessgame, 'meta/White/Name')
651            game['WhiteElo'] = self.json_field(chessgame, 'meta/White/Elo')
652            game['Black'] = self.json_field(chessgame, 'meta/Black/Name')
653            game['BlackElo'] = self.json_field(chessgame, 'meta/Black/Elo')
654            game['Result'] = self.json_field(chessgame, 'meta/Result')
655
656            # Build the PGN
657            board = LBoard(variant=FISCHERRANDOMCHESS)
658            head_complete = False
659            for move in moves:
660                # Info from the knot
661                kid = self.json_field(move, 'knotId')
662                if kid == '':
663                    break
664                kmove = self.json_field(move, 'move')
665
666                # FEN initialization
667                if kid == 0:
668                    kfen = self.json_field(move, 'fen')
669                    if kfen == '':
670                        break
671                    try:
672                        board.applyFen(kfen)
673                    except Exception:
674                        return None
675                    game['Variant'] = CHESS960
676                    game['SetUp'] = '1'
677                    game['FEN'] = kfen
678                    head_complete = True
679                else:
680                    if not head_complete:
681                        return None
682
683                    # Execution of the move
684                    if kmove == '':
685                        break
686                    try:
687                        if self.use_an:
688                            kmove = parseAny(board, kmove)
689                            game['_moves'] += toSAN(board, kmove) + ' '
690                            board.applyMove(kmove)
691                        else:
692                            game['_moves'] += kmove + ' '
693                    except Exception:
694                        return None
695
696            # Rebuild the PGN game
697            return self.rebuild_pgn(game)
698        return None
699
700
701# 365chess.com
702class InternetGame365chess(InternetGameInterface):
703    def get_description(self):
704        return '365chess.com -- %s' % CAT_HTML
705
706    def assign_game(self, url):
707        # Verify the URL
708        parsed = urlparse(url)
709        if parsed.netloc.lower() not in ['www.365chess.com', '365chess.com']:
710            return False
711        ppl = parsed.path.lower()
712        if ppl == '/game.php':
713            key = 'gid'
714        elif ppl == '/view_game.php':
715            key = 'g'
716        else:
717            return False
718
719        # Read the arguments
720        args = parse_qs(parsed.query)
721        if key in args:
722            gid = args[key][0]
723            if gid.isdigit() and gid != '0':
724                self.id = gid
725                return True
726        return False
727
728    def download_game(self):
729        # Download
730        if self.id is None:
731            return None
732        url = 'https://www.365chess.com/game.php?gid=%s' % self.id
733        page = self.download(url)
734        if page is None:
735            return None
736
737        # Played moves
738        game = {}
739        pos1 = page.find("chess_game.Init({")
740        pos1 = page.find(",pgn:'", pos1)
741        pos2 = page.find("'", pos1 + 6)
742        if -1 in [pos1, pos2]:
743            return None
744        game['_moves'] = page[pos1 + 6:pos2]
745
746        # Result
747        result = game['_moves'].split(' ')[-1]
748        if result in reprResult:
749            game['Result'] = result
750            game['_moves'] = " ".join(game['_moves'].split(' ')[0:-1])
751
752        # Header
753        game['_url'] = url
754        lines = page.replace("<td", "\n<td").split("\n")
755        rxp = re.compile(r'^([\w\-,\s]+)(\(([0-9]+)\))? vs\. ([\w\-,\s]+)(\(([0-9]+)\))?$', re.IGNORECASE)
756        game['White'] = _('Unknown')
757        game['Black'] = _('Unknown')
758        for line in lines:
759            line = line.strip()
760
761            # Event
762            for tag in ['Event', 'Site', 'Date', 'Round']:
763                if tag + ':' in line:
764                    pos1 = line.find(tag + ':')
765                    pos1 = line.find(' ', pos1)
766                    pos2 = line.find('<', pos1)
767                    if -1 not in [pos1, pos2]:
768                        v = line[pos1 + 1:pos2]
769                        if tag == 'Date':
770                            v = '%s.%s.%s' % (v[-4:], v[:2], v[3:5])  # mm/dd/yyyy --> yyyy.mm.dd
771                        game[tag] = v
772
773            # Players
774            line = self.stripHtml(line).strip()
775            m = rxp.match(line)
776            if m is not None:
777                game['White'] = str(m.group(1)).strip()
778                if m.group(3) is not None:
779                    game['WhiteElo'] = str(m.group(3)).strip()
780                game['Black'] = str(m.group(4)).strip()
781                if m.group(6) is not None:
782                    game['BlackElo'] = str(m.group(6)).strip()
783
784        # Rebuild the PGN game
785        return self.rebuild_pgn(game)
786
787
788# ChessPastebin.com
789class InternetGameChesspastebin(InternetGameInterface):
790    def get_description(self):
791        return 'ChessPastebin.com -- %s' % CAT_HTML
792
793    def assign_game(self, url):
794        return self.reacts_to(url, 'chesspastebin.com')
795
796    def download_game(self):
797        # Download
798        if self.id is None:
799            return None
800        page = self.download(self.id)
801        if page is None:
802            return None
803
804        # Extract the game ID
805        rxp = re.compile(r'.*?\<div id=\"([0-9]+)_board\"\>\<\/div\>.*?', flags=re.IGNORECASE)
806        m = rxp.match(page.replace("\n", ''))
807        if m is None:
808            return None
809        gid = m.group(1)
810
811        # Definition of the parser
812        class chesspastebinparser(HTMLParser):
813            def __init__(self):
814                HTMLParser.__init__(self)
815                self.tag_ok = False
816                self.pgn = None
817
818            def handle_starttag(self, tag, attrs):
819                if tag.lower() == 'div':
820                    for k, v in attrs:
821                        if k.lower() == 'id' and v == gid:
822                            self.tag_ok = True
823
824            def handle_data(self, data):
825                if self.pgn is None and self.tag_ok:
826                    self.pgn = data
827
828        # Read the PGN
829        parser = chesspastebinparser()
830        parser.feed(page)
831        pgn = parser.pgn
832        if pgn is not None:  # Any game must start with '[' to be considered further as valid
833            pgn = pgn.strip()
834            if not pgn.startswith('['):
835                pgn = "[Annotator \"ChessPastebin.com\"]\n%s" % pgn
836        return pgn
837
838
839# ChessBomb.com
840class InternetGameChessbomb(InternetGameInterface):
841    def get_description(self):
842        return 'ChessBomb.com -- %s' % CAT_HTML
843
844    def assign_game(self, url):
845        return self.reacts_to(url, 'chessbomb.com')
846
847    def download_game(self):
848        # Download
849        if self.id is None:
850            return None
851        url = self.id
852        page = self.download(url, userAgent=True)  # Else HTTP 403 Forbidden
853        if page is None:
854            return None
855
856        # Definition of the parser
857        class chessbombparser(HTMLParser):
858            def __init__(self):
859                HTMLParser.__init__(self)
860                self.last_tag = None
861                self.json = None
862
863            def handle_starttag(self, tag, attrs):
864                self.last_tag = tag.lower()
865
866            def handle_data(self, data):
867                if self.json is None and self.last_tag == 'script':
868                    pos1 = data.find('cbConfigData')
869                    if pos1 == -1:
870                        return
871                    pos1 = data.find('"', pos1)
872                    pos2 = data.find('"', pos1 + 1)
873                    if -1 not in [pos1, pos2]:
874                        try:
875                            bourne = b64decode(data[pos1 + 1:pos2]).decode().strip()
876                            self.json = json.loads(bourne)
877                        except Exception:
878                            self.json = None
879                            return
880
881        # Get the JSON
882        parser = chessbombparser()
883        parser.feed(page)
884        if parser.json is None:
885            return None
886
887        # Interpret the JSON
888        header = self.json_field(parser.json, 'gameData/game')
889        room = self.json_field(parser.json, 'gameData/room')
890        moves = self.json_field(parser.json, 'gameData/moves')
891        if '' in [header, room, moves]:
892            return None
893
894        game = {}
895        game['_url'] = url
896        game['Event'] = self.json_field(room, 'name')
897        game['Site'] = self.json_field(room, 'officialUrl')
898        game['Date'] = self.json_field(header, 'startAt')[:10]
899        game['Round'] = self.json_field(header, 'roundSlug')
900        game['White'] = self.json_field(header, 'white/name')
901        game['WhiteElo'] = self.json_field(header, 'white/elo')
902        game['Black'] = self.json_field(header, 'black/name')
903        game['BlackElo'] = self.json_field(header, 'black/elo')
904        game['Result'] = self.json_field(header, 'result')
905
906        game['_moves'] = ''
907        for move in moves:
908            move = self.json_field(move, 'cbn')
909            pos1 = move.find('_')
910            if pos1 == -1:
911                break
912            game['_moves'] += move[pos1 + 1:] + ' '
913
914        # Rebuild the PGN game
915        return self.rebuild_pgn(game)
916
917
918# TheChessWorld.com
919class InternetGameThechessworld(InternetGameInterface):
920    def get_description(self):
921        return 'TheChessWorld.com -- %s' % CAT_DL
922
923    def assign_game(self, url):
924        return self.reacts_to(url, 'thechessworld.com')
925
926    def download_game(self):
927        # Check
928        if self.id is None:
929            return None
930
931        # Find the links
932        links = []
933        if self.id.lower().endswith('.pgn'):
934            links.append(self.id)
935        else:
936            # Download the page
937            data = self.download(self.id)
938            if data is None:
939                return None
940
941            # Finds the games
942            rxp = re.compile(".*pgn_uri:.*'([^']+)'.*", re.IGNORECASE)
943            lines = data.split("\n")
944            for line in lines:
945                m = rxp.match(line)
946                if m is not None:
947                    links.append('https://www.thechessworld.com' + m.group(1))
948
949        # Collect the games
950        return self.download_list(links)
951
952
953# Chess.org
954class InternetGameChessOrg(InternetGameInterface):
955    def get_description(self):
956        return 'Chess.org -- %s' % CAT_WS
957
958    def assign_game(self, url):
959        rxp = re.compile(r'^https?:\/\/chess\.org\/play\/([a-f0-9\-]+)[\/\?\#]?', re.IGNORECASE)
960        m = rxp.match(url)
961        if m is not None:
962            id = str(m.group(1))
963            if len(id) == 36:
964                self.id = id
965                return True
966        return False
967
968    def download_game(self):
969        # Check
970        if self.id is None:
971            return None
972
973        # Fetch the page to retrieve the encrypted user name
974        url = 'https://chess.org/play/%s' % self.id
975        page = self.download(url)
976        if page is None:
977            return None
978        lines = page.split("\n")
979        name = ''
980        for line in lines:
981            pos1 = line.find('encryptedUsername')
982            if pos1 != -1:
983                pos1 = line.find("'", pos1)
984                pos2 = line.find("'", pos1 + 1)
985                if pos2 > pos1:
986                    name = line[pos1 + 1:pos2]
987                    break
988        if name == '':
989            return None
990
991        # Random elements to get a unique URL
992        rndI = randint(1, 1000)
993        rndS = ''.join(choice(string.ascii_lowercase) for i in range(8))
994
995        # Open a websocket to retrieve the chess data
996        @asyncio.coroutine
997        def coro():
998            url = 'wss://chess.org:443/play-sockjs/%d/%s/websocket' % (rndI, rndS)
999            log.debug('Websocket connecting to %s' % url)
1000            ws = yield from websockets.connect(url, origin="https://chess.org:443")
1001            try:
1002                # Server: Hello
1003                data = yield from ws.recv()
1004                if data != 'o':  # Open
1005                    yield from ws.close()
1006                    return None
1007
1008                # Client: I am XXX, please open the game YYY
1009                yield from ws.send('["%s %s"]' % (name, self.id))
1010                data = yield from ws.recv()
1011
1012                # Server: some data
1013                if data[:1] != 'a':
1014                    yield from ws.close()
1015                    return None
1016                return data[3:-2]
1017            finally:
1018                yield from ws.close()
1019
1020        data = self.async_from_sync(coro())
1021        if data is None or data == '':
1022            return None
1023
1024        # Parses the game
1025        chessgame = self.json_loads(data.replace('\\"', '"'))
1026        game = {}
1027        game['_url'] = url
1028        board = LBoard(variant=FISCHERRANDOMCHESS)
1029
1030        # Player info
1031        if self.json_field(chessgame, 'creatorColor') == '1':  # White=1, Black=0
1032            creator = 'White'
1033            opponent = 'Black'
1034        else:
1035            creator = 'Black'
1036            opponent = 'White'
1037        game[creator] = self.json_field(chessgame, 'creatorId')
1038        elo = self.json_field(chessgame, 'creatorPoint')
1039        if elo not in ['', '0', 0]:
1040            game[creator + 'Elo'] = elo
1041        game[opponent] = self.json_field(chessgame, 'opponentId')
1042        elo = self.json_field(chessgame, 'opponentPoint')
1043        if elo not in ['', '0', 0]:
1044            game[opponent + 'Elo'] = elo
1045
1046        # Game info
1047        startPos = self.json_field(chessgame, 'startPos')
1048        if startPos not in ['', 'startpos']:
1049            game['SetUp'] = '1'
1050            game['FEN'] = startPos
1051            game['Variant'] = CHESS960
1052            try:
1053                board.applyFen(startPos)
1054            except Exception:
1055                return None
1056        else:
1057            board.applyFen('rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w AHah - 0 1')
1058        time = self.json_field(chessgame, 'timeLimitSecs')
1059        inc = self.json_field(chessgame, 'timeBonusSecs')
1060        if '' not in [time, inc]:
1061            game['TimeControl'] = '%s+%s' % (time, inc)
1062        resultTable = [(0, '*', 'Game started'),                 # ALIVE
1063                       (1, '1-0', 'White checkmated'),           # WHITE_MATE
1064                       (2, '0-1', 'Black checkmated'),           # BLACK_MATE
1065                       (3, '1/2-1/2', 'White stalemated'),       # WHITE_STALEMATE
1066                       (4, '1/2-1/2', 'Black stalemated'),       # BLACK_STALEMATE
1067                       (5, '1/2-1/2', 'Insufficient material'),  # DRAW_NO_MATE
1068                       (6, '1/2-1/2', '50-move rule'),           # DRAW_50
1069                       (7, '1/2-1/2', 'Threefold repetition'),   # DRAW_REP
1070                       (8, '1/2-1/2', 'Mutual agreement'),       # DRAW_AGREE
1071                       (9, '0-1', 'White resigned'),             # WHITE_RESIGN
1072                       (10, '1-0', 'Black resigned'),            # BLACK_RESIGN
1073                       (11, '0-1', 'White canceled'),            # WHITE_CANCEL
1074                       (12, '1-0', 'Black canceled'),            # BLACK_CANCEL
1075                       (13, '1-0', 'White out of time'),         # WHITE_NO_TIME
1076                       (14, '0-1', 'Black out of time'),         # BLACK_NO_TIME
1077                       (15, '*', 'Not started')]                 # NOT_STARTED
1078        state = self.json_field(chessgame, 'state')
1079        result = '*'
1080        reason = 'Unknown reason %d' % state
1081        for rtID, rtScore, rtMsg in resultTable:
1082            if rtID == state:
1083                result = rtScore
1084                reason = rtMsg
1085                break
1086        game['Result'] = result
1087        game['_reason'] = reason
1088
1089        # Moves
1090        game['_moves'] = ''
1091        moves = self.json_field(chessgame, 'lans')
1092        if moves == '':
1093            return None
1094        moves = moves.split(' ')
1095        for move in moves:
1096            try:
1097                if self.use_an:
1098                    move = parseAny(board, move)
1099                    game['_moves'] += toSAN(board, move) + ' '
1100                    board.applyMove(move)
1101                else:
1102                    game['_moves'] += move + ' '
1103            except Exception:
1104                return None
1105
1106        # Rebuild the PGN game
1107        return self.rebuild_pgn(game)
1108
1109
1110# Europe-Echecs.com
1111class InternetGameEuropeechecs(InternetGameInterface):
1112    def __init__(self):
1113        InternetGameInterface.__init__(self)
1114
1115    def get_description(self):
1116        return 'Europe-Echecs.com -- %s' % CAT_DL
1117
1118    def assign_game(self, url):
1119        return self.reacts_to(url, 'europe-echecs.com')
1120
1121    def download_game(self):
1122        # Check
1123        if self.id is None:
1124            return None
1125
1126        # Find the links
1127        links = []
1128        if self.id.lower().endswith('.pgn'):
1129            links.append(self.id)
1130        else:
1131            # Download the page
1132            page = self.download(self.id)
1133            if page is None:
1134                return None
1135
1136            # Find the chess widgets
1137            rxp = re.compile(r".*class=\"cbwidget\"\s+id=\"([0-9a-f]+)_container\".*", re.IGNORECASE)
1138            lines = page.split("\n")
1139            for line in lines:
1140                m = rxp.match(line)
1141                if m is not None:
1142                    links.append('https://www.europe-echecs.com/embed/doc_%s.pgn' % m.group(1))
1143
1144        # Collect the games
1145        return self.download_list(links)
1146
1147
1148# GameKnot.com
1149class InternetGameGameknot(InternetGameInterface):
1150    def __init__(self):
1151        InternetGameInterface.__init__(self)
1152        self.url_type = TYPE_NONE
1153
1154    def get_description(self):
1155        return 'GameKnot.com -- %s' % CAT_HTML
1156
1157    def assign_game(self, url):
1158        # Verify the hostname
1159        parsed = urlparse(url.lower())
1160        if parsed.netloc not in ['www.gameknot.com', 'gameknot.com']:
1161            return False
1162
1163        # Verify the page
1164        if parsed.path == '/analyze-board.pl':
1165            ttype = TYPE_GAME
1166            tkey = 'bd'
1167        elif parsed.path == '/chess-puzzle.pl':
1168            ttype = TYPE_PUZZLE
1169            tkey = 'pz'
1170        else:
1171            return False
1172
1173        # Read the arguments
1174        args = parse_qs(parsed.query)
1175        if tkey in args:
1176            gid = args[tkey][0]
1177            if gid.isdigit() and gid != '0':
1178                self.id = gid
1179                self.url_type = ttype
1180                return True
1181        return False
1182
1183    def download_game(self):
1184        # Check
1185        if self.url_type not in [TYPE_GAME, TYPE_PUZZLE] or self.id is None:
1186            return None
1187
1188        # Download
1189        if self.url_type == TYPE_GAME:
1190            url = 'https://gameknot.com/analyze-board.pl?bd=%s' % self.id
1191        elif self.url_type == TYPE_PUZZLE:
1192            url = 'https://gameknot.com/chess-puzzle.pl?pz=%s' % self.id
1193        else:
1194            assert(False)
1195        page = self.download(url, userAgent=True)
1196        if page is None:
1197            return None
1198
1199        # Library
1200        def extract_variables(page, structure):
1201            game = {}
1202            for var, type, tag in structure:
1203                game[tag] = ''
1204            lines = page.split(';')
1205            for line in lines:
1206                for var, type, tag in structure:
1207                    pos1 = line.find(var)
1208                    if pos1 == -1:
1209                        continue
1210                    if type == 's':
1211                        pos1 = line.find("'", pos1 + 1)
1212                        pos2 = line.find("'", pos1 + 1)
1213                        if pos2 > pos1:
1214                            game[tag] = line[pos1 + 1:pos2]
1215                    elif type == 'i':
1216                        pos1 = line.find('=', pos1 + 1)
1217                        if pos1 != -1:
1218                            txt = line[pos1 + 1:].strip()
1219                            if txt not in ['', '0']:
1220                                game[tag] = txt
1221                    else:
1222                        assert(False)
1223            return game
1224
1225        # Logic for the puzzles
1226        if self.url_type == TYPE_PUZZLE:
1227            structure = [('puzzle_id', 'i', '_id'),
1228                         ('puzzle_fen', 's', 'FEN'),
1229                         ('load_solution(', 's', '_solution')]
1230            game = extract_variables(page, structure)
1231            game['_url'] = 'https://gameknot.com/chess-puzzle.pl?pz=%s' % game['_id']
1232            game['White'] = _('White')
1233            game['Black'] = _('Black')
1234            game['Result'] = '*'
1235            if game['FEN'] != '':
1236                game['SetUp'] = '1'
1237            if game['_solution'] != '':
1238                list = game['_solution'].split('|')
1239                game['_moves'] = ' {Solution:'
1240                nextid = '0'
1241                for item in list:
1242                    item = item.split(',')
1243                    # 0 = identifier of the move
1244                    # 1 = player
1245                    # 2 = identifier of the previous move
1246                    # 3 = count of following moves
1247                    # 4 = algebraic notation of the move
1248                    # 5 = UCI notation of the move
1249                    # 6 = ?
1250                    # 7 = identifier of the next move
1251                    # > = additional moves for the current line
1252                    curid = item[0]
1253                    if curid != nextid:
1254                        continue
1255                    if len(item) == 4:
1256                        break
1257                    nextid = item[7]
1258                    if self.use_an:
1259                        move = item[4]
1260                    else:
1261                        move = item[5]
1262                    game['_moves'] += ' %s' % move
1263                game['_moves'] += '}'
1264
1265        # Logic for the games
1266        elif self.url_type == TYPE_GAME:
1267            # Header
1268            structure = [('anbd_movelist', 's', '_moves'),
1269                         ('anbd_result', 'i', 'Result'),
1270                         ('anbd_player_w', 's', 'White'),
1271                         ('anbd_player_b', 's', 'Black'),
1272                         ('anbd_rating_w', 'i', 'WhiteElo'),
1273                         ('anbd_rating_b', 'i', 'BlackElo'),
1274                         ('anbd_title', 's', 'Event'),
1275                         ('anbd_timestamp', 's', 'Date'),
1276                         ('export_web_input_result_text', 's', '_reason')]
1277            game = extract_variables(page, structure)
1278            if game['Result'] == '1':
1279                game['Result'] = '1-0'
1280            elif game['Result'] == '2':
1281                game['Result'] = '1/2-1/2'
1282            elif game['Result'] == '3':
1283                game['Result'] = '0-1'
1284            else:
1285                game['Result'] = '*'
1286
1287            # Body
1288            board = LBoard()
1289            board.applyFen(DEFAULT_BOARD)
1290            moves = game['_moves'].split('-')
1291            game['_moves'] = ''
1292            for move in moves:
1293                if move == '':
1294                    break
1295                try:
1296                    if self.use_an:
1297                        kmove = parseAny(board, move)
1298                        game['_moves'] += toSAN(board, kmove) + ' '
1299                        board.applyMove(kmove)
1300                    else:
1301                        game['_moves'] += move + ' '
1302                except Exception:
1303                    return None
1304
1305        # Rebuild the PGN game
1306        return unquote(self.rebuild_pgn(game))
1307
1308
1309# Chess.com
1310class InternetGameChessCom(InternetGameInterface):
1311    def __init__(self):
1312        InternetGameInterface.__init__(self)
1313        self.url_type = None
1314
1315    def get_description(self):
1316        return 'Chess.com -- %s' % CAT_HTML
1317
1318    def assign_game(self, url):
1319        rxp = re.compile(r'^https?:\/\/([\S]+\.)?chess\.com\/([a-z\/]+)?(live|daily)\/([a-z\/]+)?([0-9]+)[\/\?\#]?', re.IGNORECASE)
1320        m = rxp.match(url)
1321        if m is not None:
1322            self.url_type = m.group(3)
1323            self.id = m.group(5)
1324            return True
1325        return False
1326
1327    def download_game(self):
1328        # Check
1329        if None in [self.id, self.url_type]:
1330            return None
1331
1332        # Download
1333        url = 'https://www.chess.com/%s/game/%s' % (self.url_type, self.id)
1334        page = self.download(url, userAgent=True)  # Else 403 Forbidden
1335        if page is None:
1336            return None
1337
1338        # Extract the JSON
1339        bourne = ''
1340        pos1 = page.find('window.chesscom.dailyGame')
1341        if pos1 != -1:
1342            pos1 = page.find('(', pos1)
1343            pos2 = page.find(')', pos1 + 1)
1344            if pos2 > pos1:
1345                bourne = page[pos1 + 2:pos2 - 1].replace('\\\\\\/', '/').replace('\\"', '"')
1346        if bourne == '':
1347            return None
1348        chessgame = self.json_loads(bourne)
1349        if not self.allow_extra and self.json_field(chessgame, 'game/isRated') and not self.json_field(chessgame, 'game/isFinished'):
1350            return None
1351        chessgame = self.json_field(chessgame, 'game')
1352        if chessgame == '':
1353            return None
1354
1355        # Header
1356        headers = self.json_field(chessgame, 'pgnHeaders')
1357        if headers == '':
1358            game = {}
1359        else:
1360            game = headers
1361        if 'Variant' in game and game['Variant'] == 'Chess960':
1362            game['Variant'] = CHESS960
1363        game['_url'] = url
1364
1365        # Body
1366        moves = self.json_field(chessgame, 'moveList')
1367        if moves == '':
1368            return None
1369        game['_moves'] = ''
1370        if 'Variant' in game and game['Variant'] == 'Crazyhouse':
1371            board = LBoard(variant=CRAZYHOUSECHESS)
1372        else:
1373            board = LBoard(variant=FISCHERRANDOMCHESS)
1374        if 'FEN' in game:
1375            board.applyFen(game['FEN'])
1376        else:
1377            board.applyFen(DEFAULT_BOARD)
1378        while len(moves) > 0:
1379            def decode(move):
1380                # Mapping
1381                map = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!?{~}(^)[_]@#$,./&-*++='
1382                pieces = 'qnrbkp'
1383
1384                # Analyze the move
1385                sFrom = sTo = sPromo = sDrop = ''
1386                posFrom = map.index(move[:1])
1387                posTo = map.index(move[1:])
1388                if posTo > 63:
1389                    sPromo = pieces[(posTo - 64) // 3]
1390                    posTo = posFrom + (-8 if posFrom < 16 else 8) + (posTo - 1) % 3 - 1
1391                if posFrom > 75:
1392                    sDrop = pieces[posFrom - 79].upper() + '@'
1393                else:
1394                    sFrom = map[posFrom % 8] + str((posFrom // 8 + 1))
1395                sTo = map[posTo % 8] + str((posTo // 8 + 1))
1396                return '%s%s%s%s' % (sDrop, sFrom, sTo, sPromo)
1397
1398            move = decode(moves[:2])
1399            moves = moves[2:]
1400            try:
1401                if self.use_an:
1402                    kmove = parseAny(board, move)
1403                    game['_moves'] += toSAN(board, kmove) + ' '
1404                    board.applyMove(kmove)
1405                else:
1406                    game['_moves'] += move + ' '
1407            except Exception:
1408                return None
1409
1410        # Final PGN
1411        return self.rebuild_pgn(game)
1412
1413
1414# Schach-Spielen.eu
1415class InternetGameSchachspielen(InternetGameInterface):
1416    def get_description(self):
1417        return 'Schach-Spielen.eu -- %s' % CAT_HTML
1418
1419    def assign_game(self, url):
1420        rxp = re.compile(r'^https?:\/\/(www\.)?schach-spielen\.eu\/(game|analyse)\/([a-z0-9]+)[\/\?\#]?', re.IGNORECASE)
1421        m = rxp.match(url)
1422        if m is not None:
1423            gid = m.group(3)
1424            if len(gid) == 8:
1425                self.id = gid
1426                return True
1427        return False
1428
1429    def download_game(self):
1430        # Download
1431        if self.id is None:
1432            return None
1433        page = self.download('https://www.schach-spielen.eu/analyse/%s' % self.id)
1434        if page is None:
1435            return None
1436
1437        # Definition of the parser
1438        class schachspielenparser(HTMLParser):
1439            def __init__(self):
1440                HTMLParser.__init__(self)
1441                self.tag_ok = False
1442                self.pgn = None
1443
1444            def handle_starttag(self, tag, attrs):
1445                if tag.lower() == 'textarea':
1446                    for k, v in attrs:
1447                        if k.lower() == 'id' and v == 'pgnText':
1448                            self.tag_ok = True
1449
1450            def handle_data(self, data):
1451                if self.pgn is None and self.tag_ok:
1452                    self.pgn = data
1453
1454        # Read the PGN
1455        parser = schachspielenparser()
1456        parser.feed(page)
1457        pgn = parser.pgn
1458        if pgn is not None:
1459            pgn = pgn.replace('[Variant "chess960"]', '[Variant "%s"]' % CHESS960)
1460        return pgn
1461
1462
1463# RedHotPawn.com
1464class InternetGameRedhotpawn(InternetGameInterface):
1465    def __init__(self):
1466        InternetGameInterface.__init__(self)
1467        self.url_type = None
1468
1469    def get_description(self):
1470        return 'RedHotPawn.com -- %s' % CAT_HTML
1471
1472    def assign_game(self, url):
1473        # Verify the URL
1474        parsed = urlparse(url)
1475        if parsed.netloc.lower() not in ['www.redhotpawn.com', 'redhotpawn.com']:
1476            return False
1477
1478        # Verify the path
1479        ppl = parsed.path.lower()
1480        if 'chess-game-' in ppl:
1481            ttype = TYPE_GAME
1482            key = 'gameid'
1483        elif 'chess-puzzle-' in ppl:
1484            ttype = TYPE_PUZZLE
1485            if 'chess-puzzle-serve' in url.lower():
1486                self.url_type = ttype
1487                self.id = url
1488                return True
1489            else:
1490                key = 'puzzleid'
1491        else:
1492            return False
1493
1494        # Read the arguments
1495        args = parse_qs(parsed.query)
1496        if key in args:
1497            gid = args[key][0]
1498            if gid.isdigit() and gid != '0':
1499                self.url_type = ttype
1500                self.id = gid
1501                return True
1502        return False
1503
1504    def download_game(self):
1505        # Download
1506        if self.id is None:
1507            return None
1508        if self.url_type == TYPE_GAME:
1509            url = 'https://www.redhotpawn.com/pagelet/view/game-pgn.php?gameid=%s' % self.id
1510        elif self.url_type == TYPE_PUZZLE:
1511            if '://' in self.id:
1512                url = self.id
1513                event = _('Puzzle')
1514            else:
1515                url = 'https://www.redhotpawn.com/chess-puzzles/chess-puzzle-solve.php?puzzleid=%s' % self.id
1516                event = _('Puzzle %s') % self.id
1517        else:
1518            return None
1519        page = self.download(url)
1520        if page is None:
1521            return None
1522
1523        # Logic for the games
1524        if self.url_type == TYPE_GAME:
1525            # Parser
1526            class redhotpawnparser(HTMLParser):
1527                def __init__(self):
1528                    HTMLParser.__init__(self)
1529                    self.tag_ok = False
1530                    self.pgn = None
1531
1532                def handle_starttag(self, tag, attrs):
1533                    if tag.lower() == 'textarea':
1534                        self.tag_ok = True
1535
1536                def handle_data(self, data):
1537                    if self.pgn is None and self.tag_ok:
1538                        self.pgn = data
1539
1540            # Extractor
1541            parser = redhotpawnparser()
1542            parser.feed(page)
1543            return parser.pgn.strip()
1544
1545        # Logic for the puzzles
1546        elif self.url_type == TYPE_PUZZLE:
1547            pos1 = page.find('var g_startFenStr')
1548            if pos1 != -1:
1549                pos1 = page.find("'", pos1)
1550                pos2 = page.find("'", pos1 + 1)
1551                if pos2 > pos1:
1552                    game = {}
1553                    game['_url'] = url
1554                    game['FEN'] = page[pos1 + 1:pos2]
1555                    game['SetUp'] = '1'
1556                    game['Event'] = event
1557                    game['White'] = _('White')
1558                    game['Black'] = _('Black')
1559                    pos1 = page.find('<h4>')
1560                    pos2 = page.find('</h4>', pos1)
1561                    if pos1 != -1 and pos2 > pos1:
1562                        game['_moves'] = '{%s}' % page[pos1 + 4:pos2]
1563                    return self.rebuild_pgn(game)
1564
1565        return None
1566
1567
1568# Chess-Samara.ru
1569class InternetGameChesssamara(InternetGameInterface):
1570    def get_description(self):
1571        return 'Chess-Samara.ru -- %s' % CAT_DL
1572
1573    def assign_game(self, url):
1574        rxp = re.compile(r'^https?:\/\/(\S+\.)?chess-samara\.ru\/(\d+)\-', re.IGNORECASE)
1575        m = rxp.match(url)
1576        if m is not None:
1577            gid = str(m.group(2))
1578            if gid.isdigit() and gid != '0':
1579                self.id = gid
1580                return True
1581        return False
1582
1583    def download_game(self):
1584        # Check
1585        if self.id is None:
1586            return None
1587
1588        # Download
1589        pgn = self.download('https://chess-samara.ru/view/pgn.html?gameid=%s' % self.id)
1590        if pgn is None or len(pgn) == 0:
1591            return None
1592        else:
1593            return pgn
1594
1595
1596# 2700chess.com
1597class InternetGame2700chess(InternetGameInterface):
1598    def get_description(self):
1599        return '2700chess.com -- %s' % CAT_HTML
1600
1601    def assign_game(self, url):
1602        # Verify the hostname
1603        parsed = urlparse(url)
1604        if parsed.netloc.lower() not in ['www.2700chess.com', '2700chess.com']:
1605            return False
1606
1607        # Refactor the direct link
1608        if parsed.path.lower() == '/games/download':
1609            args = parse_qs(parsed.query)
1610            if 'slug' in args:
1611                self.id = 'https://2700chess.com/games/%s' % args['slug'][0]
1612                return True
1613
1614        # Verify the path
1615        if parsed.path.startswith('/games/'):
1616            self.id = url
1617            return True
1618        else:
1619            return False
1620
1621    def download_game(self):
1622        # Download
1623        if self.id is None:
1624            return None
1625        page = self.download(self.id)
1626        if page is None:
1627            return None
1628
1629        # Extract the PGN
1630        lines = page.split(';')
1631        for line in lines:
1632            if 'analysis.setPgn(' in line:
1633                pos1 = line.find('"')
1634                if pos1 != -1:
1635                    pos2 = pos1
1636                    while pos2 < len(line):
1637                        pos2 += 1
1638                        if line[pos2] == '"' and line[pos2 - 1:pos2 + 1] != '\\"':
1639                            pgn = line[pos1 + 1:pos2]
1640                            return pgn.replace('\\"', '"').replace('\\/', '/').replace('\\n', "\n").strip()
1641        return None
1642
1643
1644# Iccf.com
1645class InternetGameIccf(InternetGameInterface):
1646    def __init__(self):
1647        InternetGameInterface.__init__(self)
1648        self.url_type = None
1649
1650    def get_description(self):
1651        return 'Iccf.com -- %s' % CAT_DL
1652
1653    def assign_game(self, url):
1654        # Verify the hostname
1655        parsed = urlparse(url)
1656        if parsed.netloc.lower() not in ['www.iccf.com', 'iccf.com']:
1657            return False
1658
1659        # Verify the path
1660        ppl = parsed.path.lower()
1661        if '/game' in ppl:
1662            ttyp = TYPE_GAME
1663        elif '/event' in ppl:
1664            ttyp = TYPE_EVENT
1665        else:
1666            return False
1667
1668        # Read the arguments
1669        args = parse_qs(parsed.query)
1670        if 'id' in args:
1671            gid = args['id'][0]
1672            if gid.isdigit() and gid != '0':
1673                self.url_type = ttyp
1674                self.id = gid
1675                return True
1676        return False
1677
1678    def download_game(self):
1679        # Check
1680        if self.url_type not in [TYPE_GAME, TYPE_EVENT] or self.id is None:
1681            return None
1682
1683        # Download
1684        if self.url_type == TYPE_GAME:
1685            url = 'https://www.iccf.com/GetPGN.aspx?id=%s'
1686        elif self.url_type == TYPE_EVENT:
1687            url = 'https://www.iccf.com/GetEventPGN.aspx?id=%s'
1688        pgn = self.download(url % self.id)
1689        if pgn in [None, ''] or 'does not exist.' in pgn or 'Invalid event' in pgn:
1690            return None
1691        else:
1692            return pgn
1693
1694
1695# SchachArena.de
1696class InternetGameSchacharena(InternetGameInterface):
1697    def __init__(self):
1698        InternetGameInterface.__init__(self)
1699
1700    def get_description(self):
1701        return 'SchachArena.de -- %s' % CAT_HTML
1702
1703    def assign_game(self, url):
1704        # Verify the URL
1705        parsed = urlparse(url)
1706        if parsed.netloc.lower() not in ['www.schacharena.de', 'schacharena.de'] or 'verlauf' not in parsed.path.lower():
1707            return False
1708
1709        # Read the arguments
1710        args = parse_qs(parsed.query)
1711        if 'brett' in args:
1712            gid = args['brett'][0]
1713            if gid.isdigit() and gid != '0':
1714                self.id = gid
1715                return True
1716        return False
1717
1718    def download_game(self):
1719        # Check
1720        if self.id is None:
1721            return None
1722
1723        # Download page
1724        page = self.download('https://www.schacharena.de/new/verlauf.php?brett=%s' % self.id)
1725        if page is None:
1726            return None
1727
1728        # Init
1729        rxp_player = re.compile(r'.*spielerstatistik.*name=(\w+).*\[([0-9]+)\].*', re.IGNORECASE)
1730        rxp_move = re.compile(r'.*<span.*onMouseOut.*fan\(([0-9]+)\).*', re.IGNORECASE)
1731        rxp_result = re.compile(r'.*>(1\-0|0\-1|1\/2\-1\/2)\s([^\<]+)<.*', re.IGNORECASE)
1732        player_count = 0
1733        board = LBoard()
1734        board.applyFen(DEFAULT_BOARD)
1735
1736        # Parse
1737        game = {}
1738        game['Result'] = '*'
1739        reason = ''
1740        game['_moves'] = ''
1741        game['_url'] = 'https://www.schacharena.de/new/verlauf_to_pgn_n.php?brett=%s' % self.id  # If one want to get the full PGN
1742        lines = page.split("\n")
1743        for line in lines:
1744            # Player
1745            m = rxp_player.match(line)
1746            if m is not None:
1747                player_count += 1
1748                if player_count == 1:
1749                    tag = 'White'
1750                elif player_count == 2:
1751                    tag = 'Black'
1752                else:
1753                    return None
1754                game[tag] = m.group(1)
1755                game[tag + 'Elo'] = m.group(2)
1756                continue
1757
1758            # Move
1759            m = rxp_move.match(line)
1760            if m is not None:
1761                move = m.group(1)
1762                move = '_abcdefgh'[int(move[0])] + move[1] + '_abcdefgh'[int(move[2])] + move[3]
1763                if self.use_an:
1764                    kmove = parseAny(board, move)
1765                    move = toSAN(board, kmove)
1766                    board.applyMove(kmove)
1767                game['_moves'] += '%s ' % move
1768                continue
1769
1770            # Result
1771            m = rxp_result.match(line)
1772            if m is not None:
1773                game['Result'] = m.group(1)
1774                reason = unescape(m.group(2))
1775                continue
1776
1777        # Final PGN
1778        if reason != '':
1779            game['_moves'] += ' {%s}' % reason
1780        return self.rebuild_pgn(game)
1781
1782
1783# ChessPuzzle.net
1784class InternetGameChesspuzzle(InternetGameInterface):
1785    def get_description(self):
1786        return 'ChessPuzzle.net -- %s' % CAT_HTML
1787
1788    def assign_game(self, url):
1789        rxp = re.compile(r'^https?:\/\/(\S+\.)?chesspuzzle\.net\/(Puzzle|Solution)\/([0-9]+)[\/\?\#]?', re.IGNORECASE)
1790        m = rxp.match(url)
1791        if m is not None:
1792            gid = str(m.group(3))
1793            if gid.isdigit() and gid != '0':
1794                self.id = gid
1795                return True
1796        return False
1797
1798    def download_game(self):
1799        # Check
1800        if self.id is None:
1801            return None
1802
1803        # Download the puzzle
1804        page = self.download('https://chesspuzzle.net/Solution/%s' % self.id, userAgent=True)  # Else 403 Forbidden
1805        if page is None:
1806            return None
1807
1808        # Definition of the parser
1809        class chesspuzzleparser(HTMLParser):
1810            def __init__(self):
1811                HTMLParser.__init__(self)
1812                self.last_tag = None
1813                self.pgn = None
1814
1815            def handle_starttag(self, tag, attrs):
1816                self.last_tag = tag.lower()
1817
1818            def handle_data(self, data):
1819                if self.pgn is None and self.last_tag == 'script':
1820                    lines = data.split("\n")
1821                    for line in lines:
1822                        pos1 = line.find('pgn_text')
1823                        if pos1 != -1:
1824                            pos1 = line.find("'", pos1 + 1)
1825                            pos2 = line.find("'", pos1 + 1)
1826                            if pos1 != -1 and pos2 > pos1:
1827                                self.pgn = line[pos1 + 1:pos2].replace(']  ', "]\n\n").replace('] ', "]\n").strip()
1828                                break
1829
1830        # Get the puzzle
1831        parser = chesspuzzleparser()
1832        parser.feed(page)
1833        return parser.pgn
1834
1835
1836# ChessKing.com
1837class InternetGameChessking(InternetGameInterface):
1838    def __init__(self):
1839        InternetGameInterface.__init__(self)
1840        self.url_type = None
1841
1842    def get_description(self):
1843        return 'ChessKing.com -- %s' % CAT_DL
1844
1845    def assign_game(self, url):
1846        rxp = re.compile(r'^https?:\/\/(\S+\.)?chessking\.com\/games\/(ff\/)?([0-9]+)[\/\?\#]?', re.IGNORECASE)
1847        m = rxp.match(url)
1848        if m is not None:
1849            gid = str(m.group(3))
1850            if gid.isdigit() and gid != '0' and len(gid) <= 9:
1851                if m.group(2) == 'ff/':
1852                    self.url_type = 'f'
1853                else:
1854                    self.url_type = 'g'
1855                self.id = gid
1856                return True
1857        return False
1858
1859    def download_game(self):
1860        # Check
1861        if None in [self.url_type, self.id]:
1862            return None
1863
1864        # Download
1865        id = self.id
1866        while len(id) < 9:
1867            id = '0%s' % id
1868        url = 'https://c1.chessking.com/pgn/%s/%s/%s/%s%s.pgn' % (self.url_type, id[:3], id[3:6], self.url_type, id)
1869        return self.download(url)
1870
1871
1872# IdeaChess.com
1873class InternetGameIdeachess(InternetGameInterface):
1874    def __init__(self):
1875        InternetGameInterface.__init__(self)
1876        self.url_type = None
1877
1878    def get_description(self):
1879        return 'IdeaChess.com -- %s' % CAT_API
1880
1881    def assign_game(self, url):
1882        # Game ID
1883        rxp = re.compile(r'^https?:\/\/(\S+\.)?ideachess\.com\/.*\/.*\/([0-9]+)[\/\?\#]?', re.IGNORECASE)
1884        m = rxp.match(url)
1885        if m is not None:
1886            gid = str(m.group(2))
1887            if gid.isdigit() and gid != '0':
1888                # Game type
1889                classification = [('/chess_tactics_puzzles/checkmate_n/', 'm'),
1890                                  ('/echecs_tactiques/mat_n/', 'm'),
1891                                  ('/scacchi_tattica/scacco_matto_n/', 'm'),
1892                                  ('/chess_tactics_puzzles/tactics_n/', 't'),
1893                                  ('/echecs_tactiques/tactiques_n/', 't'),
1894                                  ('/scacchi_tattica/tattica_n/', 't')]
1895                for path, ttyp in classification:
1896                    if path in url.lower():
1897                        self.url_type = ttyp
1898                        self.id = gid
1899                        return True
1900        return False
1901
1902    def download_game(self):
1903        # Check
1904        if self.url_type is None or self.id is None:
1905            return None
1906
1907        # Fetch the puzzle
1908        api = 'http://www.ideachess.com/com/ajax2'
1909        data = {'message': '{"action":100,"data":{"problemNumber":%s,"kind":"%s"}}' % (self.id, self.url_type)}
1910        bourne = self.send_xhr(api, data, userAgent=True)
1911        chessgame = self.json_loads(bourne)
1912        if self.json_field(chessgame, 'action') != 200:
1913            return None
1914
1915        # Build the PGN
1916        game = {}
1917        if self.url_type == 'm':
1918            game['_url'] = 'http://www.ideachess.com/chess_tactics_puzzles/checkmate_n/%s' % self.id
1919        elif self.url_type == 't':
1920            game['_url'] = 'http://www.ideachess.com/chess_tactics_puzzles/tactics_n/%s' % self.id
1921        else:
1922            assert(False)
1923        game['FEN'] = b64decode(self.json_field(chessgame, 'data/FEN')).decode().strip()
1924        game['SetUp'] = '1'
1925        game['_moves'] = self.json_field(chessgame, 'data/PGN')
1926        v = self.json_field(chessgame, 'data/requiredMoves')
1927        if v > 0:
1928            game['Site'] = _('%d moves to find') % v
1929        list = self.json_field(chessgame, 'data/extraInfo').split('|')
1930        if len(list) == 4:
1931            game['Event'] = list[0][list[0].find(' ') + 1:].strip()
1932            game['Date'] = list[1].strip()
1933            l2 = list[2].split(' - ')
1934            if len(l2) == 2:
1935                game['White'] = l2[0].strip()
1936                game['Black'] = l2[1].strip()
1937            game['Result'] = list[3].strip()
1938        else:
1939            game['Result'] = '*'
1940        return self.rebuild_pgn(game)
1941
1942
1943# Chess-DB.com
1944class InternetGameChessdb(InternetGameInterface):
1945    def get_description(self):
1946        return 'Chess-DB.com -- %s' % CAT_HTML
1947
1948    def assign_game(self, url):
1949        # Verify the URL
1950        parsed = urlparse(url)
1951        if parsed.netloc.lower() not in ['www.chess-db.com', 'chess-db.com'] or 'game.jsp' not in parsed.path.lower():
1952            return False
1953
1954        # Read the arguments
1955        args = parse_qs(parsed.query)
1956        if 'id' in args:
1957            gid = args['id'][0]
1958            rxp = re.compile(r'^[0-9\.]+$', re.IGNORECASE)
1959            if rxp.match(gid) is not None:
1960                self.id = gid
1961                return True
1962        return False
1963
1964    def download_game(self):
1965        # Download
1966        if self.id is None:
1967            return None
1968        page = self.download('https://chess-db.com/public/game.jsp?id=%s' % self.id)
1969        if page is None:
1970            return None
1971
1972        # Definition of the parser
1973        class chessdbparser(HTMLParser):
1974            def __init__(self):
1975                HTMLParser.__init__(self)
1976                self.tag_ok = False
1977                self.pgn = None
1978                self.pgn_tmp = None
1979
1980            def handle_starttag(self, tag, attrs):
1981                if tag.lower() == 'input':
1982                    for k, v in attrs:
1983                        k = k.lower()
1984                        if k == 'name' and v == 'pgn':
1985                            self.tag_ok = True
1986                        if k == 'value' and v.count('[') == v.count(']'):
1987                            self.pgn_tmp = v
1988
1989            def handle_data(self, data):
1990                if self.pgn is None and self.tag_ok:
1991                    self.pgn = self.pgn_tmp
1992
1993        # Read the PGN
1994        parser = chessdbparser()
1995        parser.feed(page)
1996        return parser.pgn
1997
1998
1999# ChessPro.ru
2000class InternetGameChesspro(InternetGameInterface):
2001    def get_description(self):
2002        return 'ChessPro.ru -- %s' % CAT_HTML
2003
2004    def assign_game(self, url):
2005        return self.reacts_to(url, 'chesspro.ru')
2006
2007    def download_game(self):
2008        # Check
2009        if self.id is None:
2010            return None
2011
2012        # Download the page
2013        page = self.download(self.id)
2014        if page is None:
2015            return None
2016
2017        # Find the chess widget
2018        rxp = re.compile(r'.*OpenGame\(\s*"g[0-9]+\"\s*,"(.*)"\s*\)\s*;.*', re.IGNORECASE)
2019        lines = page.split("\n")
2020        for line in lines:
2021            m = rxp.match(line)
2022            if m is not None:
2023                return '[Annotator "ChessPro.ru"]\n%s' % m.group(1)
2024        return None
2025
2026
2027# Ficgs.com
2028class InternetGameFicgs(InternetGameInterface):
2029    def get_description(self):
2030        return 'Ficgs.com -- %s' % CAT_DL
2031
2032    def assign_game(self, url):
2033        rxp = re.compile(r'^https?:\/\/(\S+\.)?ficgs\.com\/game_(\d+).html', re.IGNORECASE)
2034        m = rxp.match(url)
2035        if m is not None:
2036            gid = str(m.group(2))
2037            if gid.isdigit() and gid != '0':
2038                self.id = gid
2039                return True
2040        return False
2041
2042    def download_game(self):
2043        # Check
2044        if self.id is None:
2045            return None
2046
2047        # Download
2048        return self.download('http://www.ficgs.com/game_%s.pgn' % self.id)
2049
2050
2051# Generic
2052class InternetGameGeneric(InternetGameInterface):
2053    def __init__(self):
2054        InternetGameInterface.__init__(self)
2055        self.allow_octet_stream = False
2056
2057    def get_description(self):
2058        return 'Generic -- %s' % CAT_MISC
2059
2060    def assign_game(self, url):
2061        # Any page is valid
2062        self.id = url
2063        return True
2064
2065    def download_game(self):
2066        # Check
2067        if self.id is None:
2068            return None
2069
2070        # Download
2071        req = Request(self.id, headers={'User-Agent': self.userAgent})
2072        response = urlopen(req)
2073        mime = response.info().get_content_type().lower()
2074        data = self.read_data(response)
2075        if data is None:
2076            return None
2077
2078        # Chess file
2079        if (mime in ['application/x-chess-pgn', 'application/pgn']) or (self.allow_octet_stream and mime == 'application/octet-stream'):
2080            return data
2081
2082        # Web-page
2083        if mime == 'text/html':
2084            # Definition of the parser
2085            class linksParser(HTMLParser):
2086                def __init__(self):
2087                    HTMLParser.__init__(self)
2088                    self.links = []
2089
2090                def handle_starttag(self, tag, attrs):
2091                    if tag.lower() == 'a':
2092                        for k, v in attrs:
2093                            if k.lower() == 'href':
2094                                v = v.strip()
2095                                u = urlparse(v)
2096                                if u.path.lower().endswith('.pgn'):
2097                                    self.links.append(v)
2098
2099            # Read the links
2100            parser = linksParser()
2101            parser.feed(data)
2102
2103            # Rebuild a full path
2104            base = urlparse(self.id)
2105            for i, link in enumerate(parser.links):
2106                e = urlparse(link)
2107                if e.netloc == '':
2108                    url = '%s://%s/%s' % (base.scheme, base.netloc, e.path)
2109                else:
2110                    url = link
2111                parser.links[i] = url
2112
2113            # Collect the games
2114            return self.download_list(parser.links)
2115        return None
2116
2117
2118# Interface
2119chess_providers = [InternetGameLichess(),
2120                   InternetGameChessgames(),
2121                   InternetGameFicsgames(),
2122                   InternetGameChesstempo(),
2123                   InternetGameChess24(),
2124                   InternetGame365chess(),
2125                   InternetGameChesspastebin(),
2126                   InternetGameChessbomb(),
2127                   InternetGameThechessworld(),
2128                   InternetGameChessOrg(),
2129                   InternetGameEuropeechecs(),
2130                   InternetGameGameknot(),
2131                   InternetGameChessCom(),
2132                   InternetGameSchachspielen(),
2133                   InternetGameRedhotpawn(),
2134                   InternetGameChesssamara(),
2135                   InternetGame2700chess(),
2136                   InternetGameIccf(),
2137                   InternetGameSchacharena(),
2138                   InternetGameChesspuzzle(),
2139                   InternetGameChessking(),
2140                   InternetGameIdeachess(),
2141                   InternetGameChessdb(),
2142                   InternetGameChesspro(),
2143                   InternetGameFicgs(),
2144                   # TODO ChessDuo.com
2145                   InternetGameGeneric()]
2146
2147
2148# Get the list of chess providers
2149def get_internet_game_providers():
2150    list = [cp.get_description() for cp in chess_providers]
2151    list.sort()
2152    return list
2153
2154
2155# Retrieve a game from a URL
2156def get_internet_game_as_pgn(url):
2157    # Check the format
2158    if url in [None, '']:
2159        return None
2160    p = urlparse(url.strip())
2161    if '' in [p.scheme, p.netloc]:
2162        return None
2163    log.debug('URL to retrieve: %s' % url)
2164
2165    # Call the chess providers
2166    for prov in chess_providers:
2167        if not prov.is_enabled():
2168            continue
2169        if prov.assign_game(url):
2170            # Download
2171            log.debug('Responding chess provider: %s' % prov.get_description())
2172            try:
2173                pgn = prov.download_game()
2174                pgn = prov.sanitize(pgn)
2175            except Exception:
2176                pgn = None
2177
2178            # Check
2179            if pgn is None:
2180                log.debug('Download failed')
2181            else:
2182                log.debug('Successful download')
2183                return pgn
2184    return None
2185
2186
2187# print(get_internet_game_as_pgn(''))
2188