1#!/usr/bin/env python3 2# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 3from __future__ import print_function 4 5import urllib.parse 6import glob 7import os 8import sys 9 10verbose = '--verbose' in sys.argv 11dump = '--dump' in sys.argv 12internal = '--internal' in sys.argv 13plain_output = '--plain-output' in sys.argv 14if plain_output: 15 plain_file = open('plain_text_out.txt', 'w+') 16in_code = None 17 18paths = ['include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/extensions/*.hpp'] 19 20if internal: 21 paths.append('include/libtorrent/aux_/*.hpp') 22 23files = [] 24 25for p in paths: 26 files.extend(glob.glob(os.path.join('..', p))) 27 28functions = [] 29classes = [] 30enums = [] 31constants = {} 32 33# maps filename to overview description 34overviews = {} 35 36# maps names -> URL 37symbols = {} 38 39global orphaned_export 40 41# some files that need pre-processing to turn symbols into 42# links into the reference documentation 43preprocess_rst = \ 44 { 45 'manual.rst': 'manual-ref.rst', 46 'tuning.rst': 'tuning-ref.rst', 47 'upgrade_to_1.2.rst': 'upgrade_to_1.2-ref.rst', 48 'settings.rst': 'settings-ref.rst' 49 } 50 51# some pre-defined sections from the main manual 52symbols = \ 53 { 54 "queuing_": "manual-ref.html#queuing", 55 "fast-resume_": "manual-ref.html#fast-resume", 56 "storage-allocation_": "manual-ref.html#storage-allocation", 57 "alerts_": "manual-ref.html#alerts", 58 "upnp-and-nat-pmp_": "manual-ref.html#upnp-and-nat-pmp", 59 "http-seeding_": "manual-ref.html#http-seeding", 60 "metadata-from-peers_": "manual-ref.html#metadata-from-peers", 61 "magnet-links_": "manual-ref.html#magnet-links", 62 "ssl-torrents_": "manual-ref.html#ssl-torrents", 63 "dynamic-loading-of-torrent-files_": "manual-ref.html#dynamic-loading-of-torrent-files", 64 "session-statistics_": "manual-ref.html#session-statistics", 65 "peer-classes_": "manual-ref.html#peer-classes" 66 } 67 68# parse out names of settings, and add them to the symbols list, to get cross 69# references working 70with open('../src/settings_pack.cpp') as f: 71 for line in f: 72 line = line.strip() 73 if not line.startswith('SET('): 74 continue 75 76 name = line.split('(')[1].split(',')[0] 77 symbols['settings_pack::' + name] = 'reference-Settings.html#' + name 78 79static_links = \ 80 { 81 ".. _`BEP 3`: https://www.bittorrent.org/beps/bep_0003.html", 82 ".. _`BEP 17`: https://www.bittorrent.org/beps/bep_0017.html", 83 ".. _`BEP 19`: https://www.bittorrent.org/beps/bep_0019.html", 84 ".. _`BEP 42`: https://www.bittorrent.org/beps/bep_0042.html", 85 ".. _`rate based choking`: manual-ref.html#rate-based-choking", 86 } 87 88anon_index = 0 89 90category_mapping = { 91 'ed25519.hpp': 'ed25519', 92 'session.hpp': 'Session', 93 'session_handle.hpp': 'Session', 94 'add_torrent_params.hpp': 'Core', 95 'session_status.hpp': 'Session', 96 'session_stats.hpp': 'Session', 97 'session_params.hpp': 'Session', 98 'error_code.hpp': 'Error Codes', 99 'storage.hpp': 'Custom Storage', 100 'storage_defs.hpp': 'Storage', 101 'file_storage.hpp': 'Storage', 102 'file_pool.hpp': 'Custom Storage', 103 'extensions.hpp': 'Plugins', 104 'ut_metadata.hpp': 'Plugins', 105 'ut_pex.hpp': 'Plugins', 106 'ut_trackers.hpp': 'Plugins', 107 'smart_ban.hpp': 'Plugins', 108 'create_torrent.hpp': 'Create Torrents', 109 'alert.hpp': 'Alerts', 110 'alert_types.hpp': 'Alerts', 111 'bencode.hpp': 'Bencoding', 112 'lazy_entry.hpp': 'Bencoding', 113 'bdecode.hpp': 'Bdecoding', 114 'entry.hpp': 'Bencoding', 115 'time.hpp': 'Time', 116 'escape_string.hpp': 'Utility', 117 'enum_net.hpp': 'Network', 118 'broadcast_socket.hpp': 'Network', 119 'socket.hpp': 'Network', 120 'address.hpp': 'Network', 121 'socket_io.hpp': 'Network', 122 'bitfield.hpp': 'Utility', 123 'sha1_hash.hpp': 'Utility', 124 'hasher.hpp': 'Utility', 125 'hasher512.hpp': 'Utility', 126 'identify_client.hpp': 'Utility', 127 'ip_filter.hpp': 'Filter', 128 'session_settings.hpp': 'Settings', 129 'settings_pack.hpp': 'Settings', 130 'operations.hpp': 'Alerts', 131 'disk_buffer_holder.hpp': 'Custom Storage', 132 'alert_dispatcher.hpp': 'Alerts', 133} 134 135category_fun_mapping = { 136 'min_memory_usage()': 'Settings', 137 'high_performance_seed()': 'Settings', 138 'cache_status': 'Core', 139} 140 141 142def categorize_symbol(name, filename): 143 f = os.path.split(filename)[1] 144 145 if name.endswith('_category()') \ 146 or name.endswith('_error_code') \ 147 or name.endswith('error_code_enum') \ 148 or name.endswith('errors'): 149 return 'Error Codes' 150 151 if name in category_fun_mapping: 152 return category_fun_mapping[name] 153 154 if f in category_mapping: 155 return category_mapping[f] 156 157 if filename.startswith('libtorrent/kademlia/'): 158 return 'DHT' 159 160 return 'Core' 161 162 163def suppress_warning(filename, name): 164 f = os.path.split(filename)[1] 165 if f != 'alert_types.hpp': 166 return False 167 168 # if name.endswith('_alert') or name == 'message()': 169 return True 170 171 # return False 172 173 174def first_item(itr): 175 for i in itr: 176 return i 177 return None 178 179 180def is_visible(desc): 181 if desc.strip().startswith('hidden'): 182 return False 183 if internal: 184 return True 185 if desc.strip().startswith('internal'): 186 return False 187 return True 188 189 190def highlight_signature(s): 191 name = s.split('(', 1) 192 name2 = name[0].split(' ') 193 if len(name2[-1]) == 0: 194 return s 195 196 # make the name of the function bold 197 name2[-1] = '**' + name2[-1] + '** ' 198 199 # if there is a return value, make sure we preserve pointer types 200 if len(name2) > 1: 201 name2[0] = name2[0].replace('*', '\\*') 202 name[0] = ' '.join(name2) 203 204 # we have to escape asterisks, since this is rendered into 205 # a parsed literal in rst 206 name[1] = name[1].replace('*', '\\*') 207 208 # we also have to escape colons 209 name[1] = name[1].replace(':', '\\:') 210 211 # escape trailing underscores 212 name[1] = name[1].replace('_', '\\_') 213 214 # comments in signatures are italic 215 name[1] = name[1].replace('/\\*', '*/\\*') 216 name[1] = name[1].replace('\\*/', '\\*/*') 217 return '('.join(name) 218 219 220def highlight_name(s): 221 if '=' in s: 222 splitter = ' = ' 223 elif '{' in s: 224 splitter = '{' 225 else: 226 return s 227 228 name = s.split(splitter, 1) 229 name2 = name[0].split(' ') 230 if len(name2[-1]) == 0: 231 return s 232 233 name2[-1] = '**' + name2[-1] + '** ' 234 name[0] = ' '.join(name2) 235 return splitter.join(name) 236 237 238def html_sanitize(s): 239 ret = '' 240 for i in s: 241 if i == '<': 242 ret += '<' 243 elif i == '>': 244 ret += '>' 245 elif i == '&': 246 ret += '&' 247 else: 248 ret += i 249 return ret 250 251 252def looks_like_namespace(line): 253 line = line.strip() 254 if line.startswith('namespace'): 255 return True 256 return False 257 258 259def looks_like_blank(line): 260 line = line.split('//')[0] 261 line = line.replace('{', '') 262 line = line.replace('}', '') 263 line = line.replace('[', '') 264 line = line.replace(']', '') 265 line = line.replace(';', '') 266 line = line.strip() 267 return len(line) == 0 268 269 270def looks_like_variable(line): 271 line = line.split('//')[0] 272 line = line.strip() 273 if ' ' not in line and '\t' not in line: 274 return False 275 if line.startswith('friend '): 276 return False 277 if line.startswith('enum '): 278 return False 279 if line.startswith(','): 280 return False 281 if line.startswith(':'): 282 return False 283 if line.startswith('typedef'): 284 return False 285 if line.startswith('using'): 286 return False 287 if ' = ' in line: 288 return True 289 if line.endswith(';'): 290 return True 291 return False 292 293 294def looks_like_constant(line): 295 line = line.strip() 296 if line.startswith('inline'): 297 line = line.split('inline')[1] 298 line = line.strip() 299 if not line.startswith('constexpr'): 300 return False 301 line = line.split('constexpr')[1] 302 return looks_like_variable(line) 303 304 305def looks_like_forward_decl(line): 306 line = line.split('//')[0] 307 line = line.strip() 308 if not line.endswith(';'): 309 return False 310 if '{' in line: 311 return False 312 if '}' in line: 313 return False 314 if line.startswith('friend '): 315 return True 316 if line.startswith('struct '): 317 return True 318 if line.startswith('class '): 319 return True 320 return False 321 322 323def looks_like_function(line): 324 if line.startswith('friend'): 325 return False 326 if '::' in line.split('(')[0].split(' ')[-1]: 327 return False 328 if line.startswith(','): 329 return False 330 if line.startswith(':'): 331 return False 332 return '(' in line 333 334 335def parse_function(lno, lines, filename): 336 337 start_paren = 0 338 end_paren = 0 339 signature = '' 340 341 global orphaned_export 342 orphaned_export = False 343 344 while lno < len(lines): 345 line = lines[lno].strip() 346 lno += 1 347 if line.startswith('//'): 348 continue 349 350 start_paren += line.count('(') 351 end_paren += line.count(')') 352 353 sig_line = line.replace('TORRENT_EXPORT ', '') \ 354 .replace('TORRENT_EXTRA_EXPORT', '') \ 355 .replace('TORRENT_COUNTER_NOEXCEPT', '').strip() 356 if signature != '': 357 sig_line = '\n ' + sig_line 358 signature += sig_line 359 if verbose: 360 print('fun %s' % line) 361 362 if start_paren > 0 and start_paren == end_paren: 363 if signature[-1] != ';': 364 # we also need to consume the function body 365 start_paren = 0 366 end_paren = 0 367 for i in range(len(signature)): 368 if signature[i] == '(': 369 start_paren += 1 370 elif signature[i] == ')': 371 end_paren += 1 372 373 if start_paren > 0 and start_paren == end_paren: 374 for k in range(i, len(signature)): 375 if signature[k] == ':' or signature[k] == '{': 376 signature = signature[0:k].strip() 377 break 378 break 379 380 lno = consume_block(lno - 1, lines) 381 signature += ';' 382 ret = [{'file': filename[11:], 'signatures': set([signature]), 'names': set( 383 [signature.split('(')[0].split(' ')[-1].strip() + '()'])}, lno] 384 if first_item(ret[0]['names']) == '()': 385 return [None, lno] 386 return ret 387 if len(signature) > 0: 388 print('\x1b[31mFAILED TO PARSE FUNCTION\x1b[0m %s\nline: %d\nfile: %s' % (signature, lno, filename)) 389 return [None, lno] 390 391 392def add_desc(line): 393 # plain output prints just descriptions and filters out c++ code. 394 # it's used to run spell checker over 395 if plain_output: 396 for s in line.split('\n'): 397 # if the first character is a space, strip it 398 if len(s) > 0 and s[0] == ' ': 399 s = s[1:] 400 global in_code 401 if in_code is not None and not s.startswith(in_code) and len(s) > 1: 402 in_code = None 403 404 if s.strip().startswith('.. code::'): 405 in_code = s.split('.. code::')[0] + '\t' 406 407 # strip out C++ code from the plain text output since it's meant for 408 # running spell checking over 409 if not s.strip().startswith('.. ') and in_code is None: 410 plain_file.write(s + '\n') 411 412 413def parse_class(lno, lines, filename): 414 start_brace = 0 415 end_brace = 0 416 417 name = '' 418 funs = [] 419 fields = [] 420 enums = [] 421 state = 'public' 422 context = '' 423 class_type = 'struct' 424 blanks = 0 425 decl = '' 426 427 while lno < len(lines): 428 line = lines[lno].strip() 429 decl += lines[lno].replace('TORRENT_EXPORT ', '') \ 430 .replace('TORRENT_EXTRA_EXPORT', '') \ 431 .replace('TORRENT_COUNTER_NOEXCEPT', '').split('{')[0].strip() 432 if '{' in line: 433 break 434 if verbose: 435 print('class %s' % line) 436 lno += 1 437 438 if decl.startswith('class'): 439 state = 'private' 440 class_type = 'class' 441 442 name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip() 443 444 global orphaned_export 445 orphaned_export = False 446 447 while lno < len(lines): 448 line = lines[lno].strip() 449 lno += 1 450 451 if line == '': 452 blanks += 1 453 context = '' 454 continue 455 456 if line.startswith('/*'): 457 lno = consume_comment(lno - 1, lines) 458 continue 459 460 if line.startswith('#'): 461 lno = consume_ifdef(lno - 1, lines, True) 462 continue 463 464 if 'TORRENT_DEFINE_ALERT' in line: 465 if verbose: 466 print('xx %s' % line) 467 blanks += 1 468 continue 469 if 'TORRENT_DEPRECATED' in line: 470 if verbose: 471 print('xx %s' % line) 472 blanks += 1 473 continue 474 475 if line.startswith('//'): 476 if verbose: 477 print('desc %s' % line) 478 479 line = line[2:] 480 if len(line) and line[0] == ' ': 481 line = line[1:] 482 context += line + '\n' 483 continue 484 485 start_brace += line.count('{') 486 end_brace += line.count('}') 487 488 if line == 'private:': 489 state = 'private' 490 elif line == 'protected:': 491 state = 'protected' 492 elif line == 'public:': 493 state = 'public' 494 495 if start_brace > 0 and start_brace == end_brace: 496 return [{'file': filename[11:], 'enums': enums, 'fields':fields, 497 'type': class_type, 'name': name, 'decl': decl, 'fun': funs}, lno] 498 499 if state != 'public' and not internal: 500 if verbose: 501 print('private %s' % line) 502 blanks += 1 503 continue 504 505 if start_brace - end_brace > 1: 506 if verbose: 507 print('scope %s' % line) 508 blanks += 1 509 continue 510 511 if looks_like_function(line): 512 current_fun, lno = parse_function(lno - 1, lines, filename) 513 if current_fun is not None and is_visible(context): 514 if context == '' and blanks == 0 and len(funs): 515 funs[-1]['signatures'].update(current_fun['signatures']) 516 funs[-1]['names'].update(current_fun['names']) 517 else: 518 if 'TODO: ' in context: 519 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 520 sys.exit(1) 521 current_fun['desc'] = context 522 add_desc(context) 523 if context == '' and not suppress_warning(filename, first_item(current_fun['names'])): 524 print('WARNING: member function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 525 % (name + '::' + first_item(current_fun['names']), filename, lno)) 526 funs.append(current_fun) 527 context = '' 528 blanks = 0 529 continue 530 531 if looks_like_variable(line): 532 if 'constexpr static' in line: 533 print('ERROR: found "constexpr static", use "static constexpr" instead for consistency!\n%s:%d\n%s' 534 % (filename, lno, line)) 535 sys.exit(1) 536 if verbose: 537 print('var %s' % line) 538 if not is_visible(context): 539 continue 540 line = line.split('//')[0].strip() 541 # the name may look like this: 542 # std::uint8_t fails : 7; 543 # int scrape_downloaded = -1; 544 # static constexpr peer_flags_t interesting{0x1}; 545 n = line.split('=')[0].split('{')[0].strip().split(' : ')[0].split(' ')[-1].split(':')[0].split(';')[0] 546 if context == '' and blanks == 0 and len(fields): 547 fields[-1]['names'].append(n) 548 fields[-1]['signatures'].append(line) 549 else: 550 if context == '' and not suppress_warning(filename, n): 551 print('WARNING: field "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 552 % (name + '::' + n, filename, lno)) 553 add_desc(context) 554 fields.append({'signatures': [line], 'names': [n], 'desc': context}) 555 context = '' 556 blanks = 0 557 continue 558 559 if line.startswith('enum '): 560 if verbose: 561 print('enum %s' % line) 562 if not is_visible(context): 563 consume_block(lno - 1, lines) 564 else: 565 enum, lno = parse_enum(lno - 1, lines, filename) 566 if enum is not None: 567 if 'TODO: ' in context: 568 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 569 sys.exit(1) 570 enum['desc'] = context 571 add_desc(context) 572 if context == '' and not suppress_warning(filename, enum['name']): 573 print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 574 % (name + '::' + enum['name'], filename, lno)) 575 enums.append(enum) 576 context = '' 577 continue 578 579 context = '' 580 581 if verbose: 582 if looks_like_forward_decl(line) \ 583 or looks_like_blank(line) \ 584 or looks_like_namespace(line): 585 print('-- %s' % line) 586 else: 587 print('?? %s' % line) 588 589 if len(name) > 0: 590 print('\x1b[31mFAILED TO PARSE CLASS\x1b[0m %s\nfile: %s:%d' % (name, filename, lno)) 591 return [None, lno] 592 593 594def parse_constant(lno, lines, filename): 595 line = lines[lno].strip() 596 if verbose: 597 print('const %s' % line) 598 line = line.split('=')[0] 599 if 'constexpr' in line: 600 line = line.split('constexpr')[1] 601 if '{' in line and '}' in line: 602 line = line.split('{')[0] 603 t, name = line.strip().split(' ') 604 return [{'file': filename[11:], 'type': t, 'name': name}, lno + 1] 605 606 607def parse_enum(lno, lines, filename): 608 start_brace = 0 609 end_brace = 0 610 global anon_index 611 612 line = lines[lno].strip() 613 name = line.replace('enum ', '').replace('class ', '').split(':')[0].split('{')[0].strip() 614 if len(name) == 0: 615 if not internal: 616 print('WARNING: anonymous enum at: \x1b[34m%s:%d\x1b[0m' % (filename, lno)) 617 lno = consume_block(lno - 1, lines) 618 return [None, lno] 619 name = 'anonymous_enum_%d' % anon_index 620 anon_index += 1 621 622 values = [] 623 context = '' 624 if '{' not in line: 625 if verbose: 626 print('enum %s' % lines[lno]) 627 lno += 1 628 629 val = 0 630 while lno < len(lines): 631 line = lines[lno].strip() 632 lno += 1 633 634 if line.startswith('//'): 635 if verbose: 636 print('desc %s' % line) 637 line = line[2:] 638 if len(line) and line[0] == ' ': 639 line = line[1:] 640 context += line + '\n' 641 continue 642 643 if line.startswith('#'): 644 lno = consume_ifdef(lno - 1, lines) 645 continue 646 647 start_brace += line.count('{') 648 end_brace += line.count('}') 649 650 if '{' in line: 651 line = line.split('{')[1] 652 line = line.split('}')[0] 653 654 if len(line): 655 if verbose: 656 print('enumv %s' % lines[lno - 1]) 657 for v in line.split(','): 658 v = v.strip() 659 if v.startswith('//'): 660 break 661 if v == '': 662 continue 663 valstr = '' 664 try: 665 if '=' in v: 666 val = int(v.split('=')[1].strip(), 0) 667 valstr = str(val) 668 except Exception: 669 pass 670 671 if '=' in v: 672 v = v.split('=')[0].strip() 673 if is_visible(context): 674 add_desc(context) 675 values.append({'name': v.strip(), 'desc': context, 'val': valstr}) 676 if verbose: 677 print('enumv %s' % valstr) 678 context = '' 679 val += 1 680 else: 681 if verbose: 682 print('?? %s' % lines[lno - 1]) 683 684 if start_brace > 0 and start_brace == end_brace: 685 return [{'file': filename[11:], 'name': name, 'values': values}, lno] 686 687 if len(name) > 0: 688 print('\x1b[31mFAILED TO PARSE ENUM\x1b[0m %s\nline: %d\nfile: %s' % (name, lno, filename)) 689 return [None, lno] 690 691 692def consume_block(lno, lines): 693 start_brace = 0 694 end_brace = 0 695 696 while lno < len(lines): 697 line = lines[lno].strip() 698 if verbose: 699 print('xx %s' % line) 700 lno += 1 701 702 start_brace += line.count('{') 703 end_brace += line.count('}') 704 705 if start_brace > 0 and start_brace == end_brace: 706 break 707 return lno 708 709 710def consume_comment(lno, lines): 711 while lno < len(lines): 712 line = lines[lno].strip() 713 if verbose: 714 print('xx %s' % line) 715 lno += 1 716 if '*/' in line: 717 break 718 719 return lno 720 721 722def trim_define(line): 723 return line.replace('#ifndef', '').replace('#ifdef', '') \ 724 .replace('#if', '').replace('defined', '') \ 725 .replace('TORRENT_ABI_VERSION == 1', '') \ 726 .replace('||', '').replace('&&', '').replace('(', '').replace(')', '') \ 727 .replace('!', '').replace('\\', '').strip() 728 729 730def consume_ifdef(lno, lines, warn_on_ifdefs=False): 731 line = lines[lno].strip() 732 lno += 1 733 734 start_if = 1 735 end_if = 0 736 737 if verbose: 738 print('prep %s' % line) 739 740 if warn_on_ifdefs and line.strip().startswith('#if'): 741 while line.endswith('\\'): 742 lno += 1 743 line += lines[lno].strip() 744 if verbose: 745 print('prep %s' % lines[lno].trim()) 746 define = trim_define(line) 747 if 'TORRENT_' in define and 'TORRENT_ABI_VERSION' not in define: 748 print('\x1b[31mWARNING: possible ABI breakage in public struct! "%s" \x1b[34m %s:%d\x1b[0m' % 749 (define, filename, lno)) 750 # we've already warned once, no need to do it twice 751 warn_on_ifdefs = False 752 elif define != '': 753 print('\x1b[33msensitive define in public struct: "%s"\x1b[34m %s:%d\x1b[0m' % (define, filename, lno)) 754 755 if (line.startswith('#if') and ( 756 ' TORRENT_USE_ASSERTS' in line or 757 ' TORRENT_USE_INVARIANT_CHECKS' in line or 758 ' TORRENT_ASIO_DEBUGGING' in line) or 759 line == '#if TORRENT_ABI_VERSION == 1'): 760 while lno < len(lines): 761 line = lines[lno].strip() 762 lno += 1 763 if verbose: 764 print('prep %s' % line) 765 if line.startswith('#endif'): 766 end_if += 1 767 if line.startswith('#if'): 768 start_if += 1 769 if line == '#else' and start_if - end_if == 1: 770 break 771 if start_if - end_if == 0: 772 break 773 return lno 774 else: 775 while line.endswith('\\') and lno < len(lines): 776 line = lines[lno].strip() 777 lno += 1 778 if verbose: 779 print('prep %s' % line) 780 781 return lno 782 783 784for filename in files: 785 h = open(filename) 786 lines = h.read().split('\n') 787 788 if verbose: 789 print('\n=== %s ===\n' % filename) 790 791 blanks = 0 792 lno = 0 793 global orphaned_export 794 orphaned_export = False 795 796 while lno < len(lines): 797 line = lines[lno].strip() 798 799 if orphaned_export: 800 print('ERROR: TORRENT_EXPORT without function or class!\n%s:%d\n%s' % (filename, lno, line)) 801 sys.exit(1) 802 803 lno += 1 804 805 if line == '': 806 blanks += 1 807 context = '' 808 continue 809 810 if 'TORRENT_EXPORT' in line.split() \ 811 and 'ifndef TORRENT_EXPORT' not in line \ 812 and 'define TORRENT_DEPRECATED_EXPORT TORRENT_EXPORT' not in line \ 813 and 'define TORRENT_EXPORT' not in line \ 814 and 'for TORRENT_EXPORT' not in line \ 815 and 'TORRENT_EXPORT TORRENT_CFG' not in line \ 816 and 'extern TORRENT_EXPORT ' not in line \ 817 and 'struct TORRENT_EXPORT ' not in line: 818 orphaned_export = True 819 if verbose: 820 print('maybe orphaned: %s\n' % line) 821 822 if line.startswith('//') and line[2:].strip() == 'OVERVIEW': 823 # this is a section overview 824 current_overview = '' 825 while lno < len(lines): 826 line = lines[lno].strip() 827 lno += 1 828 if not line.startswith('//'): 829 # end of overview 830 overviews[filename[11:]] = current_overview 831 current_overview = '' 832 break 833 line = line[2:] 834 if line.startswith(' '): 835 line = line[1:] 836 current_overview += line + '\n' 837 838 if line.startswith('//'): 839 if verbose: 840 print('desc %s' % line) 841 line = line[2:] 842 if len(line) and line[0] == ' ': 843 line = line[1:] 844 context += line + '\n' 845 continue 846 847 if line.startswith('/*'): 848 lno = consume_comment(lno - 1, lines) 849 continue 850 851 if line.startswith('#'): 852 lno = consume_ifdef(lno - 1, lines) 853 continue 854 855 if (line == 'namespace detail {' or 856 line == 'namespace aux {' or 857 line == 'namespace libtorrent { namespace aux {') \ 858 and not internal: 859 lno = consume_block(lno - 1, lines) 860 continue 861 862 if ('namespace aux' in line or 863 'namespace detail' in line) and \ 864 '//' not in line.split('namespace')[0] and \ 865 '}' not in line.split('namespace')[1]: 866 print('ERROR: whitespace preceding namespace declaration: %s:%d' % (filename, lno)) 867 sys.exit(1) 868 869 if 'TORRENT_DEPRECATED' in line: 870 if ('class ' in line or 'struct ' in line) and ';' not in line: 871 lno = consume_block(lno - 1, lines) 872 context = '' 873 blanks += 1 874 if verbose: 875 print('xx %s' % line) 876 continue 877 878 if looks_like_constant(line): 879 if 'constexpr static' in line: 880 print('ERROR: found "constexpr static", use "static constexpr" instead for consistency!\n%s:%d\n%s' 881 % (filename, lno, line)) 882 sys.exit(1) 883 current_constant, lno = parse_constant(lno - 1, lines, filename) 884 if current_constant is not None and is_visible(context): 885 if 'TODO: ' in context: 886 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 887 sys.exit(1) 888 current_constant['desc'] = context 889 add_desc(context) 890 if context == '': 891 print('WARNING: constant "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 892 % (current_constant['name'], filename, lno)) 893 t = current_constant['type'] 894 if t in constants: 895 constants[t].append(current_constant) 896 else: 897 constants[t] = [current_constant] 898 continue 899 900 if 'TORRENT_EXPORT ' in line or line.startswith('inline ') or line.startswith('template') or internal: 901 if line.startswith('class ') or line.startswith('struct '): 902 if not line.endswith(';'): 903 current_class, lno = parse_class(lno - 1, lines, filename) 904 if current_class is not None and is_visible(context): 905 if 'TODO: ' in context: 906 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 907 sys.exit(1) 908 current_class['desc'] = context 909 add_desc(context) 910 if context == '': 911 print('WARNING: class "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 912 % (current_class['name'], filename, lno)) 913 classes.append(current_class) 914 context = '' 915 blanks += 1 916 continue 917 918 if looks_like_function(line): 919 current_fun, lno = parse_function(lno - 1, lines, filename) 920 if current_fun is not None and is_visible(context): 921 if context == '' and blanks == 0 and len(functions): 922 functions[-1]['signatures'].update(current_fun['signatures']) 923 functions[-1]['names'].update(current_fun['names']) 924 else: 925 if 'TODO: ' in context: 926 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 927 sys.exit(1) 928 current_fun['desc'] = context 929 add_desc(context) 930 if context == '': 931 print('WARNING: function "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 932 % (first_item(current_fun['names']), filename, lno)) 933 functions.append(current_fun) 934 context = '' 935 blanks = 0 936 continue 937 938 if ('enum class ' not in line and 'class ' in line or 'struct ' in line) and ';' not in line: 939 lno = consume_block(lno - 1, lines) 940 context = '' 941 blanks += 1 942 continue 943 944 if line.startswith('enum '): 945 if not is_visible(context): 946 consume_block(lno - 1, lines) 947 else: 948 current_enum, lno = parse_enum(lno - 1, lines, filename) 949 if current_enum is not None and is_visible(context): 950 if 'TODO: ' in context: 951 print('TODO comment in public documentation: %s:%d' % (filename, lno)) 952 sys.exit(1) 953 current_enum['desc'] = context 954 add_desc(context) 955 if context == '': 956 print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m' 957 % (current_enum['name'], filename, lno)) 958 enums.append(current_enum) 959 context = '' 960 blanks += 1 961 continue 962 963 blanks += 1 964 if verbose: 965 if looks_like_forward_decl(line) \ 966 or looks_like_blank(line) \ 967 or looks_like_namespace(line): 968 print('-- %s' % line) 969 else: 970 print('?? %s' % line) 971 972 context = '' 973 h.close() 974 975# ==================================================================== 976# 977# RENDER PART 978# 979# ==================================================================== 980 981 982def new_category(cat): 983 return {'classes': [], 'functions': [], 'enums': [], 984 'filename': 'reference-%s.rst' % cat.replace(' ', '_'), 985 'constants': {}} 986 987 988if dump: 989 990 if verbose: 991 print('\n===============================\n') 992 993 for c in classes: 994 print('\x1b[4m%s\x1b[0m %s\n{' % (c['type'], c['name'])) 995 for f in c['fun']: 996 for s in f['signatures']: 997 print(' %s' % s.replace('\n', '\n ')) 998 999 if len(c['fun']) > 0 and len(c['fields']) > 0: 1000 print('') 1001 1002 for f in c['fields']: 1003 for s in f['signatures']: 1004 print(' %s' % s) 1005 1006 if len(c['fields']) > 0 and len(c['enums']) > 0: 1007 print('') 1008 1009 for e in c['enums']: 1010 print(' \x1b[4menum\x1b[0m %s\n {' % e['name']) 1011 for v in e['values']: 1012 print(' %s' % v['name']) 1013 print(' };') 1014 print('};\n') 1015 1016 for f in functions: 1017 print('%s' % f['signature']) 1018 1019 for e in enums: 1020 print('\x1b[4menum\x1b[0m %s\n{' % e['name']) 1021 for v in e['values']: 1022 print(' %s' % v['name']) 1023 print('};') 1024 1025 for t, c in constants: 1026 print('\x1b[4mconstant\x1b[0m %s %s\n' % (e['type'], e['name'])) 1027 1028categories = {} 1029 1030for c in classes: 1031 cat = categorize_symbol(c['name'], c['file']) 1032 if cat not in categories: 1033 categories[cat] = new_category(cat) 1034 1035 if c['file'] in overviews: 1036 categories[cat]['overview'] = overviews[c['file']] 1037 1038 filename = categories[cat]['filename'].replace('.rst', '.html') + '#' 1039 categories[cat]['classes'].append(c) 1040 symbols[c['name']] = filename + c['name'] 1041 for f in c['fun']: 1042 for n in f['names']: 1043 symbols[n] = filename + n 1044 symbols[c['name'] + '::' + n] = filename + n 1045 1046 for f in c['fields']: 1047 for n in f['names']: 1048 symbols[c['name'] + '::' + n] = filename + n 1049 1050 for e in c['enums']: 1051 symbols[e['name']] = filename + e['name'] 1052 symbols[c['name'] + '::' + e['name']] = filename + e['name'] 1053 for v in e['values']: 1054 # symbols[v['name']] = filename + v['name'] 1055 symbols[e['name'] + '::' + v['name']] = filename + v['name'] 1056 symbols[c['name'] + '::' + v['name']] = filename + v['name'] 1057 1058for f in functions: 1059 cat = categorize_symbol(first_item(f['names']), f['file']) 1060 if cat not in categories: 1061 categories[cat] = new_category(cat) 1062 1063 if f['file'] in overviews: 1064 categories[cat]['overview'] = overviews[f['file']] 1065 1066 for n in f['names']: 1067 symbols[n] = categories[cat]['filename'].replace('.rst', '.html') + '#' + n 1068 categories[cat]['functions'].append(f) 1069 1070for e in enums: 1071 cat = categorize_symbol(e['name'], e['file']) 1072 if cat not in categories: 1073 categories[cat] = new_category(cat) 1074 categories[cat]['enums'].append(e) 1075 filename = categories[cat]['filename'].replace('.rst', '.html') + '#' 1076 symbols[e['name']] = filename + e['name'] 1077 for v in e['values']: 1078 symbols[e['name'] + '::' + v['name']] = filename + v['name'] 1079 1080for t, c in constants.items(): 1081 for const in c: 1082 cat = categorize_symbol(t, const['file']) 1083 if cat not in categories: 1084 categories[cat] = new_category(cat) 1085 if t not in categories[cat]['constants']: 1086 categories[cat]['constants'][t] = [const] 1087 else: 1088 categories[cat]['constants'][t].append(const) 1089 filename = categories[cat]['filename'].replace('.rst', '.html') + '#' 1090 symbols[t + '::' + const['name']] = filename + t + '::' + const['name'] 1091 symbols[t] = filename + t 1092 1093 1094def print_declared_in(out, o): 1095 out.write('Declared in "%s"\n\n' % print_link(o['file'], '../include/%s' % o['file'])) 1096 print(dump_link_targets(), file=out) 1097 1098# returns RST marked up string 1099 1100 1101def linkify_symbols(string): 1102 lines = string.split('\n') 1103 ret = [] 1104 in_literal = False 1105 lno = 0 1106 return_string = '' 1107 for line in lines: 1108 lno += 1 1109 # don't touch headlines, i.e. lines whose 1110 # next line entirely contains one of =, - or . 1111 if (lno < len(lines) - 1): 1112 next_line = lines[lno] 1113 else: 1114 next_line = '' 1115 1116 if '.. include:: ' in line: 1117 return_string += '\n'.join(ret) 1118 ret = [line] 1119 return_string += dump_link_targets() + '\n' 1120 continue 1121 1122 if len(next_line) > 0 and lines[lno].replace('=', ''). \ 1123 replace('-', '').replace('.', '') == '': 1124 ret.append(line) 1125 continue 1126 1127 if line.startswith('|'): 1128 ret.append(line) 1129 continue 1130 if in_literal and not line.startswith('\t') and not line == '': 1131 # print(' end literal: "%s"' % line) 1132 in_literal = False 1133 if in_literal: 1134 # print(' literal: "%s"' % line) 1135 ret.append(line) 1136 continue 1137 if line.strip() == '.. parsed-literal::' or \ 1138 line.strip().startswith('.. code::') or \ 1139 (not line.strip().startswith('..') and line.endswith('::')): 1140 # print(' start literal: "%s"' % line) 1141 in_literal = True 1142 words = line.split(' ') 1143 1144 for i in range(len(words)): 1145 # it's important to preserve leading 1146 # tabs, since that's relevant for 1147 # rst markup 1148 1149 leading = '' 1150 w = words[i] 1151 1152 if len(w) == 0: 1153 continue 1154 1155 while len(w) > 0 and \ 1156 w[0] in ['\t', ' ', '(', '[', '{']: 1157 leading += w[0] 1158 w = w[1:] 1159 1160 # preserve commas and dots at the end 1161 w = w.strip() 1162 trailing = '' 1163 1164 if len(w) == 0: 1165 continue 1166 1167 while len(w) > 1 and w[-1] in ['.', ',', ')'] and w[-2:] != '()': 1168 trailing = w[-1] + trailing 1169 w = w[:-1] 1170 1171 link_name = w 1172 1173 # print(w) 1174 1175 if len(w) == 0: 1176 continue 1177 1178 if link_name[-1] == '_': 1179 link_name = link_name[:-1] 1180 1181 if w in symbols: 1182 link_name = link_name.replace('-', ' ') 1183 # print(' found %s -> %s' % (w, link_name)) 1184 words[i] = leading + print_link(link_name, symbols[w]) + trailing 1185 ret.append(' '.join(words)) 1186 return_string += '\n'.join(ret) 1187 return return_string 1188 1189 1190link_targets = [] 1191 1192 1193def print_link(name, target): 1194 global link_targets 1195 link_targets.append(target) 1196 return "`%s`__" % name 1197 1198 1199def dump_link_targets(indent=''): 1200 global link_targets 1201 ret = '\n' 1202 for link in link_targets: 1203 ret += '%s__ %s\n' % (indent, link) 1204 link_targets = [] 1205 return ret 1206 1207 1208def heading(string, c, indent=''): 1209 string = string.strip() 1210 return '\n' + indent + string + '\n' + indent + (c * len(string)) + '\n' 1211 1212 1213def render_enums(out, enums, print_declared_reference, header_level): 1214 for e in enums: 1215 print('.. raw:: html\n', file=out) 1216 print('\t<a name="%s"></a>' % e['name'], file=out) 1217 print('', file=out) 1218 dump_report_issue('enum ' + e['name'], out) 1219 print(heading('enum %s' % e['name'], header_level), file=out) 1220 1221 print_declared_in(out, e) 1222 1223 width = [len('name'), len('value'), len('description')] 1224 1225 for i in range(len(e['values'])): 1226 e['values'][i]['desc'] = linkify_symbols(e['values'][i]['desc']) 1227 1228 for v in e['values']: 1229 width[0] = max(width[0], len(v['name'])) 1230 width[1] = max(width[1], len(v['val'])) 1231 for d in v['desc'].split('\n'): 1232 width[2] = max(width[2], len(d)) 1233 1234 print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out) 1235 print('| ' + 'name'.ljust(width[0]) + ' | ' + 'value'.ljust(width[1]) + ' | ' 1236 + 'description'.ljust(width[2]) + ' |', file=out) 1237 print('+=' + ('=' * width[0]) + '=+=' + ('=' * width[1]) + '=+=' + ('=' * width[2]) + '=+', file=out) 1238 for v in e['values']: 1239 d = v['desc'].split('\n') 1240 if len(d) == 0: 1241 d = [''] 1242 print('| ' + v['name'].ljust(width[0]) + ' | ' + v['val'].ljust(width[1]) + ' | ' 1243 + d[0].ljust(width[2]) + ' |', file=out) 1244 for s in d[1:]: 1245 print('| ' + (' ' * width[0]) + ' | ' + (' ' * width[1]) + ' | ' + s.ljust(width[2]) + ' |', file=out) 1246 print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out) 1247 print('', file=out) 1248 1249 print(dump_link_targets(), file=out) 1250 1251 1252sections = \ 1253 { 1254 'Core': 0, 1255 'DHT': 0, 1256 'Session': 0, 1257 'Settings': 0, 1258 1259 'Bencoding': 1, 1260 'Bdecoding': 1, 1261 'Filter': 1, 1262 'Error Codes': 1, 1263 'Create Torrents': 1, 1264 1265 'ed25519': 2, 1266 'Utility': 2, 1267 'Storage': 2, 1268 'Custom Storage': 2, 1269 'Plugins': 2, 1270 1271 'Alerts': 3 1272 } 1273 1274 1275def print_toc(out, categories, s): 1276 for cat in categories: 1277 if (s != 2 and cat not in sections) or \ 1278 (cat in sections and sections[cat] != s): 1279 continue 1280 1281 print('\t.. rubric:: %s\n' % cat, file=out) 1282 1283 if 'overview' in categories[cat]: 1284 print('\t| overview__', file=out) 1285 1286 for c in categories[cat]['classes']: 1287 print('\t| ' + print_link(c['name'], symbols[c['name']]), file=out) 1288 for f in categories[cat]['functions']: 1289 for n in f['names']: 1290 print('\t| ' + print_link(n, symbols[n]), file=out) 1291 for e in categories[cat]['enums']: 1292 print('\t| ' + print_link(e['name'], symbols[e['name']]), file=out) 1293 for t, c in categories[cat]['constants'].items(): 1294 print('\t| ' + print_link(t, symbols[t]), file=out) 1295 print('', file=out) 1296 1297 if 'overview' in categories[cat]: 1298 print('\t__ %s#overview' % categories[cat]['filename'].replace('.rst', '.html'), file=out) 1299 print(dump_link_targets('\t'), file=out) 1300 1301 1302def dump_report_issue(h, out): 1303 print(('.. raw:: html\n\n\t<span style="float:right;">[<a style="color:blue;" ' + 1304 'href="http://github.com/arvidn/libtorrent/issues/new?title=docs:{0}&labels=' + 1305 'documentation&body={1}">report issue</a>]</span>\n\n').format( 1306 urllib.parse.quote_plus(h), 1307 urllib.parse.quote_plus('Documentation under heading "' + h + '" could be improved')), file=out) 1308 1309 1310out = open('reference.rst', 'w+') 1311out.write('''======================= 1312reference documentation 1313======================= 1314 1315''') 1316 1317out.write('`single-page version`__\n\n__ single-page-ref.html\n\n') 1318 1319for i in range(4): 1320 1321 out.write('.. container:: main-toc\n\n') 1322 print_toc(out, categories, i) 1323 1324out.close() 1325 1326for cat in categories: 1327 out = open(categories[cat]['filename'], 'w+') 1328 1329 classes = categories[cat]['classes'] 1330 functions = categories[cat]['functions'] 1331 enums = categories[cat]['enums'] 1332 constants = categories[cat]['constants'] 1333 1334 out.write('''.. include:: header.rst 1335 1336`home`__ 1337 1338__ reference.html 1339 1340%s 1341 1342.. contents:: Table of contents 1343 :depth: 2 1344 :backlinks: none 1345 1346''' % heading(cat, '=')) 1347 1348 if 'overview' in categories[cat]: 1349 out.write('%s\n' % linkify_symbols(categories[cat]['overview'])) 1350 1351 for c in classes: 1352 1353 print('.. raw:: html\n', file=out) 1354 print('\t<a name="%s"></a>' % c['name'], file=out) 1355 print('', file=out) 1356 1357 dump_report_issue('class ' + c['name'], out) 1358 out.write('%s\n' % heading(c['name'], '-')) 1359 print_declared_in(out, c) 1360 c['desc'] = linkify_symbols(c['desc']) 1361 out.write('%s\n' % c['desc']) 1362 print(dump_link_targets(), file=out) 1363 1364 print('\n.. parsed-literal::\n\t', file=out) 1365 1366 block = '\n%s\n{\n' % c['decl'] 1367 for f in c['fun']: 1368 for s in f['signatures']: 1369 block += ' %s\n' % highlight_signature(s.replace('\n', '\n ')) 1370 1371 if len(c['fun']) > 0 and len(c['enums']) > 0: 1372 block += '\n' 1373 1374 first = True 1375 for e in c['enums']: 1376 if not first: 1377 block += '\n' 1378 first = False 1379 block += ' enum %s\n {\n' % e['name'] 1380 for v in e['values']: 1381 block += ' %s,\n' % v['name'] 1382 block += ' };\n' 1383 1384 if len(c['fun']) + len(c['enums']) > 0 and len(c['fields']): 1385 block += '\n' 1386 1387 for f in c['fields']: 1388 for s in f['signatures']: 1389 block += ' %s\n' % highlight_name(s) 1390 1391 block += '};' 1392 1393 print(block.replace('\n', '\n\t') + '\n', file=out) 1394 1395 for f in c['fun']: 1396 if f['desc'] == '': 1397 continue 1398 print('.. raw:: html\n', file=out) 1399 for n in f['names']: 1400 print('\t<a name="%s"></a>' % n, file=out) 1401 print('', file=out) 1402 h = ' '.join(f['names']) 1403 dump_report_issue('%s::[%s]' % (c['name'], h), out) 1404 print(heading(h, '.'), file=out) 1405 1406 block = '.. parsed-literal::\n\n' 1407 1408 for s in f['signatures']: 1409 block += highlight_signature(s.replace('\n', '\n ')) + '\n' 1410 print('%s\n' % block.replace('\n', '\n\t'), file=out) 1411 f['desc'] = linkify_symbols(f['desc']) 1412 print('%s' % f['desc'], file=out) 1413 1414 print(dump_link_targets(), file=out) 1415 1416 render_enums(out, c['enums'], False, '.') 1417 1418 for f in c['fields']: 1419 if f['desc'] == '': 1420 continue 1421 1422 print('.. raw:: html\n', file=out) 1423 for n in f['names']: 1424 print('\t<a name="%s"></a>' % n, file=out) 1425 print('', file=out) 1426 h = ' '.join(f['names']) 1427 dump_report_issue('%s::[%s]' % (c['name'], h), out) 1428 print(h, file=out) 1429 f['desc'] = linkify_symbols(f['desc']) 1430 print('\t%s' % f['desc'].replace('\n', '\n\t'), file=out) 1431 1432 print(dump_link_targets(), file=out) 1433 1434 for f in functions: 1435 print('.. raw:: html\n', file=out) 1436 for n in f['names']: 1437 print('\t<a name="%s"></a>' % n, file=out) 1438 print('', file=out) 1439 h = ' '.join(f['names']) 1440 dump_report_issue(h, out) 1441 print(heading(h, '-'), file=out) 1442 print_declared_in(out, f) 1443 1444 block = '.. parsed-literal::\n\n' 1445 for s in f['signatures']: 1446 block += highlight_signature(s) + '\n' 1447 1448 print('%s\n' % block.replace('\n', '\n\t'), file=out) 1449 print(linkify_symbols(f['desc']), file=out) 1450 1451 print(dump_link_targets(), file=out) 1452 1453 render_enums(out, enums, True, '-') 1454 1455 for t, c in constants.items(): 1456 print('.. raw:: html\n', file=out) 1457 print('\t<a name="%s"></a>\n' % t, file=out) 1458 dump_report_issue(t, out) 1459 print(heading(t, '-'), file=out) 1460 print_declared_in(out, c[0]) 1461 1462 for v in c: 1463 print('.. raw:: html\n', file=out) 1464 print('\t<a name="%s::%s"></a>\n' % (t, v['name']), file=out) 1465 print(v['name'], file=out) 1466 v['desc'] = linkify_symbols(v['desc']) 1467 print('\t%s' % v['desc'].replace('\n', '\n\t'), file=out) 1468 print(dump_link_targets('\t'), file=out) 1469 1470 print('', file=out) 1471 1472 print(dump_link_targets(), file=out) 1473 1474 for i in static_links: 1475 print(i, file=out) 1476 1477 out.close() 1478 1479# for s in symbols: 1480# print(s) 1481 1482for i, o in list(preprocess_rst.items()): 1483 f = open(i, 'r') 1484 out = open(o, 'w+') 1485 print('processing %s -> %s' % (i, o)) 1486 link = linkify_symbols(f.read()) 1487 print(link, end=' ', file=out) 1488 1489 print(dump_link_targets(), file=out) 1490 1491 out.close() 1492 f.close() 1493