1# -*- encoding: utf-8 *-* 2import os 3import io 4import re 5import sys 6import textwrap 7from collections import OrderedDict 8from datetime import datetime 9from glob import glob 10 11import setup_lz4 12import setup_zstd 13import setup_b2 14import setup_xxhash 15 16# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code 17prefer_system_liblz4 = True 18 19# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code 20prefer_system_libzstd = True 21 22# True: use the shared libb2 from the system, False: use the bundled blake2 code 23prefer_system_libb2 = True 24 25# True: use the shared libxxhash (>= 0.6.5 [>= 0.7.2 on ARM]) from the system, False: use the bundled xxhash code 26prefer_system_libxxhash = True 27 28# prefer_system_msgpack is another option, but you need to set it in src/borg/helpers.py. 29 30min_python = (3, 5) 31my_python = sys.version_info 32 33if my_python < min_python: 34 print("Borg requires Python %d.%d or later" % min_python) 35 sys.exit(1) 36 37# Are we building on ReadTheDocs? 38on_rtd = os.environ.get('READTHEDOCS') 39 40install_requires = [ 41 'packaging', 42] 43 44# note for package maintainers: if you package borgbackup for distribution, 45# please add llfuse as a *requirement* on all platforms that have a working 46# llfuse package. "borg mount" needs llfuse to work. 47# if you do not have llfuse, do not require it, most of borgbackup will work. 48extras_require = { 49 'fuse': [ 50 # 1.3.8 is the fixed version that works on py39 AND freebsd. 51 # if you cythonize yourself and make sure llfuse works for your 52 # OS and python version, you can use other versions than 1.3.8, too. 53 'llfuse >=1.3.4', # should nowadays pull 1.3.8 or better 54 ], 55} 56 57from setuptools import setup, find_packages, Extension, Command 58from setuptools.command.sdist import sdist 59 60compress_source = 'src/borg/compress.pyx' 61crypto_ll_source = 'src/borg/crypto/low_level.pyx' 62chunker_source = 'src/borg/chunker.pyx' 63hashindex_source = 'src/borg/hashindex.pyx' 64item_source = 'src/borg/item.pyx' 65checksums_source = 'src/borg/algorithms/checksums.pyx' 66platform_posix_source = 'src/borg/platform/posix.pyx' 67platform_linux_source = 'src/borg/platform/linux.pyx' 68platform_syncfilerange_source = 'src/borg/platform/syncfilerange.pyx' 69platform_darwin_source = 'src/borg/platform/darwin.pyx' 70platform_freebsd_source = 'src/borg/platform/freebsd.pyx' 71msgpack_packer_source = 'src/borg/algorithms/msgpack/_packer.pyx' 72msgpack_unpacker_source = 'src/borg/algorithms/msgpack/_unpacker.pyx' 73 74cython_c_sources = [ 75 # these .pyx will get compiled to .c 76 compress_source, 77 crypto_ll_source, 78 chunker_source, 79 hashindex_source, 80 item_source, 81 checksums_source, 82 platform_posix_source, 83 platform_linux_source, 84 platform_syncfilerange_source, 85 platform_freebsd_source, 86 platform_darwin_source, 87] 88 89cython_cpp_sources = [ 90 # these .pyx will get compiled to .cpp 91 msgpack_packer_source, 92 msgpack_unpacker_source, 93] 94 95try: 96 from Cython.Distutils import build_ext 97 import Cython.Compiler.Main as cython_compiler 98 99 class Sdist(sdist): 100 def __init__(self, *args, **kwargs): 101 for src in cython_c_sources: 102 cython_compiler.compile(src, cython_compiler.default_options) 103 for src in cython_cpp_sources: 104 cython_compiler.compile(src, cplus=True) 105 super().__init__(*args, **kwargs) 106 107 def make_distribution(self): 108 self.filelist.extend([ 109 'src/borg/compress.c', 110 'src/borg/crypto/low_level.c', 111 'src/borg/chunker.c', 'src/borg/_chunker.c', 112 'src/borg/hashindex.c', 'src/borg/_hashindex.c', 113 'src/borg/cache_sync/cache_sync.c', 'src/borg/cache_sync/sysdep.h', 'src/borg/cache_sync/unpack.h', 114 'src/borg/cache_sync/unpack_define.h', 'src/borg/cache_sync/unpack_template.h', 115 'src/borg/item.c', 116 'src/borg/algorithms/checksums.c', 117 'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c', 118 'src/borg/algorithms/xxh64/xxhash.h', 'src/borg/algorithms/xxh64/xxhash.c', 119 'src/borg/platform/posix.c', 120 'src/borg/platform/linux.c', 121 'src/borg/platform/syncfilerange.c', 122 'src/borg/platform/freebsd.c', 123 'src/borg/platform/darwin.c', 124 'src/borg/algorithms/msgpack/_packer.cpp', 125 'src/borg/algorithms/msgpack/_unpacker.cpp', 126 ]) 127 super().make_distribution() 128 129except ImportError: 130 class Sdist(sdist): 131 def __init__(self, *args, **kwargs): 132 raise Exception('Cython is required to run sdist') 133 134 compress_source = compress_source.replace('.pyx', '.c') 135 crypto_ll_source = crypto_ll_source.replace('.pyx', '.c') 136 chunker_source = chunker_source.replace('.pyx', '.c') 137 hashindex_source = hashindex_source.replace('.pyx', '.c') 138 item_source = item_source.replace('.pyx', '.c') 139 checksums_source = checksums_source.replace('.pyx', '.c') 140 platform_posix_source = platform_posix_source.replace('.pyx', '.c') 141 platform_linux_source = platform_linux_source.replace('.pyx', '.c') 142 platform_syncfilerange_source = platform_syncfilerange_source.replace('.pyx', '.c') 143 platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c') 144 platform_darwin_source = platform_darwin_source.replace('.pyx', '.c') 145 146 msgpack_packer_source = msgpack_packer_source.replace('.pyx', '.cpp') 147 msgpack_unpacker_source = msgpack_unpacker_source.replace('.pyx', '.cpp') 148 149 from setuptools.command.build_ext import build_ext 150 if not on_rtd and not all(os.path.exists(path) for path in [ 151 compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source, 152 platform_posix_source, platform_linux_source, platform_syncfilerange_source, platform_freebsd_source, platform_darwin_source, 153 msgpack_packer_source, msgpack_unpacker_source]): 154 raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') 155 156 157def detect_openssl(prefixes): 158 for prefix in prefixes: 159 filename = os.path.join(prefix, 'include', 'openssl', 'evp.h') 160 if os.path.exists(filename): 161 with open(filename, 'rb') as fd: 162 if b'PKCS5_PBKDF2_HMAC(' in fd.read(): 163 return prefix 164 165 166include_dirs = [] 167library_dirs = [] 168define_macros = [] 169 170possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', 171 '/usr/local/borg', '/opt/local', '/opt/pkg', '/opt/homebrew/opt/openssl@1.1', ] 172if os.environ.get('BORG_OPENSSL_PREFIX'): 173 possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) 174ssl_prefix = detect_openssl(possible_openssl_prefixes) 175if not ssl_prefix: 176 raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes))) 177include_dirs.append(os.path.join(ssl_prefix, 'include')) 178library_dirs.append(os.path.join(ssl_prefix, 'lib')) 179 180 181possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', 182 '/usr/local/borg', '/opt/local', '/opt/pkg', ] 183if os.environ.get('BORG_LIBLZ4_PREFIX'): 184 possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX')) 185liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes) 186if prefer_system_liblz4 and liblz4_prefix: 187 print('Detected and preferring liblz4 over bundled LZ4') 188 define_macros.append(('BORG_USE_LIBLZ4', 'YES')) 189 liblz4_system = True 190else: 191 liblz4_system = False 192 193possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2', 194 '/usr/local/borg', '/opt/local', '/opt/pkg', ] 195if os.environ.get('BORG_LIBB2_PREFIX'): 196 possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) 197libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes) 198if prefer_system_libb2 and libb2_prefix: 199 print('Detected and preferring libb2 over bundled BLAKE2') 200 define_macros.append(('BORG_USE_LIBB2', 'YES')) 201 libb2_system = True 202else: 203 libb2_system = False 204 205possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd', 206 '/usr/local/borg', '/opt/local', '/opt/pkg', ] 207if os.environ.get('BORG_LIBZSTD_PREFIX'): 208 possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX')) 209libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes) 210if prefer_system_libzstd and libzstd_prefix: 211 print('Detected and preferring libzstd over bundled ZSTD') 212 define_macros.append(('BORG_USE_LIBZSTD', 'YES')) 213 libzstd_system = True 214else: 215 libzstd_system = False 216 217possible_libxxhash_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libxxhash', '/usr/local/libxxhash', 218 '/usr/local/borg', '/opt/local', '/opt/pkg', ] 219if os.environ.get('BORG_LIBXXHASH_PREFIX'): 220 possible_libxxhash_prefixes.insert(0, os.environ.get('BORG_LIBXXHASH_PREFIX')) 221libxxhash_prefix = setup_xxhash.xxhash_system_prefix(possible_libxxhash_prefixes) 222if prefer_system_libxxhash and libxxhash_prefix: 223 print('Detected and preferring libxxhash over bundled XXHASH') 224 define_macros.append(('BORG_USE_LIBXXHASH', 'YES')) 225 libxxhash_system = True 226else: 227 libxxhash_system = False 228 229 230with open('README.rst', 'r') as fd: 231 long_description = fd.read() 232 # remove header, but have one \n before first headline 233 start = long_description.find('What is BorgBackup?') 234 assert start >= 0 235 long_description = '\n' + long_description[start:] 236 # remove badges 237 long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description) 238 # remove unknown directives 239 long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description) 240 241 242def format_metavar(option): 243 if option.nargs in ('*', '...'): 244 return '[%s...]' % option.metavar 245 elif option.nargs == '?': 246 return '[%s]' % option.metavar 247 elif option.nargs is None: 248 return option.metavar 249 else: 250 raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs)) 251 252 253class build_usage(Command): 254 description = "generate usage for each command" 255 256 user_options = [ 257 ('output=', 'O', 'output directory'), 258 ] 259 260 def initialize_options(self): 261 pass 262 263 def finalize_options(self): 264 pass 265 266 def run(self): 267 print('generating usage docs') 268 import borg 269 borg.doc_mode = 'build_man' 270 if not os.path.exists('docs/usage'): 271 os.mkdir('docs/usage') 272 # allows us to build docs without the C modules fully loaded during help generation 273 from borg.archiver import Archiver 274 parser = Archiver(prog='borg').build_parser() 275 # borgfs has a separate man page to satisfy debian's "every program from a package 276 # must have a man page" requirement, but it doesn't need a separate HTML docs page 277 #borgfs_parser = Archiver(prog='borgfs').build_parser() 278 279 self.generate_level("", parser, Archiver) 280 281 def generate_level(self, prefix, parser, Archiver, extra_choices=None): 282 is_subcommand = False 283 choices = {} 284 for action in parser._actions: 285 if action.choices is not None and 'SubParsersAction' in str(action.__class__): 286 is_subcommand = True 287 for cmd, parser in action.choices.items(): 288 choices[prefix + cmd] = parser 289 if extra_choices is not None: 290 choices.update(extra_choices) 291 if prefix and not choices: 292 return 293 print('found commands: %s' % list(choices.keys())) 294 295 for command, parser in sorted(choices.items()): 296 if command.startswith('debug'): 297 print('skipping', command) 298 continue 299 print('generating help for %s' % command) 300 301 if self.generate_level(command + " ", parser, Archiver): 302 continue 303 304 with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: 305 doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") 306 if command == 'help': 307 for topic in Archiver.helptext: 308 params = {"topic": topic, 309 "underline": '~' * len('borg help ' + topic)} 310 doc.write(".. _borg_{topic}:\n\n".format(**params)) 311 doc.write("borg help {topic}\n{underline}\n\n".format(**params)) 312 doc.write(Archiver.helptext[topic]) 313 else: 314 params = {"command": command, 315 "command_": command.replace(' ', '_'), 316 "underline": '-' * len('borg ' + command)} 317 doc.write(".. _borg_{command_}:\n\n".format(**params)) 318 doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params)) 319 self.write_usage(parser, doc) 320 epilog = parser.epilog 321 parser.epilog = None 322 self.write_options(parser, doc) 323 doc.write("\n\nDescription\n~~~~~~~~~~~\n") 324 doc.write(epilog) 325 326 if 'create' in choices: 327 common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] 328 with open('docs/usage/common-options.rst.inc', 'w') as doc: 329 self.write_options_group(common_options, doc, False, base_indent=0) 330 331 return is_subcommand 332 333 def write_usage(self, parser, fp): 334 if any(len(o.option_strings) for o in parser._actions): 335 fp.write(' [options]') 336 for option in parser._actions: 337 if option.option_strings: 338 continue 339 fp.write(' ' + format_metavar(option)) 340 fp.write('\n\n') 341 342 def write_options(self, parser, fp): 343 def is_positional_group(group): 344 return any(not o.option_strings for o in group._group_actions) 345 346 # HTML output: 347 # A table using some column-spans 348 349 def html_write(s): 350 for line in s.splitlines(): 351 fp.write(' ' + line + '\n') 352 353 rows = [] 354 for group in parser._action_groups: 355 if group.title == 'Common options': 356 # (no of columns used, columns, ...) 357 rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`')) 358 else: 359 if not group._group_actions: 360 continue 361 group_header = '**%s**' % group.title 362 if group.description: 363 group_header += ' — ' + group.description 364 rows.append((1, group_header)) 365 if is_positional_group(group): 366 for option in group._group_actions: 367 rows.append((3, '', '``%s``' % option.metavar, option.help or '')) 368 else: 369 for option in group._group_actions: 370 if option.metavar: 371 option_fmt = '``%s ' + option.metavar + '``' 372 else: 373 option_fmt = '``%s``' 374 option_str = ', '.join(option_fmt % s for s in option.option_strings) 375 option_desc = textwrap.dedent((option.help or '') % option.__dict__) 376 rows.append((3, '', option_str, option_desc)) 377 378 fp.write('.. only:: html\n\n') 379 table = io.StringIO() 380 table.write('.. class:: borg-options-table\n\n') 381 self.rows_to_table(rows, table.write) 382 fp.write(textwrap.indent(table.getvalue(), ' ' * 4)) 383 384 # LaTeX output: 385 # Regular rST option lists (irregular column widths) 386 latex_options = io.StringIO() 387 for group in parser._action_groups: 388 if group.title == 'Common options': 389 latex_options.write('\n\n:ref:`common_options`\n') 390 latex_options.write(' |') 391 else: 392 self.write_options_group(group, latex_options) 393 fp.write('\n.. only:: latex\n\n') 394 fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4)) 395 396 def rows_to_table(self, rows, write): 397 def write_row_separator(): 398 write('+') 399 for column_width in column_widths: 400 write('-' * (column_width + 1)) 401 write('+') 402 write('\n') 403 404 # Find column count and width 405 column_count = max(columns for columns, *_ in rows) 406 column_widths = [0] * column_count 407 for columns, *cells in rows: 408 for i in range(columns): 409 # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output 410 column_widths[i] = max(column_widths[i], len(cells[i]) + 1) 411 412 for columns, *original_cells in rows: 413 write_row_separator() 414 # If a cell contains newlines, then the row must be split up in individual rows 415 # where each cell contains no newline. 416 rowspanning_cells = [] 417 original_cells = list(original_cells) 418 while any('\n' in cell for cell in original_cells): 419 cell_bloc = [] 420 for i, cell in enumerate(original_cells): 421 pre, _, original_cells[i] = cell.partition('\n') 422 cell_bloc.append(pre) 423 rowspanning_cells.append(cell_bloc) 424 rowspanning_cells.append(original_cells) 425 for cells in rowspanning_cells: 426 for i, column_width in enumerate(column_widths): 427 if i < columns: 428 write('| ') 429 write(cells[i].ljust(column_width)) 430 else: 431 write(' ') 432 write(''.ljust(column_width)) 433 write('|\n') 434 435 write_row_separator() 436 # This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils, 437 # but does absolutely no good here. It sets bogus column widths which cannot be overridden 438 # with CSS alone. 439 # Since this is HTML-only output, it would be possible to just generate a <table> directly, 440 # but then we'd lose rST formatting. 441 write(textwrap.dedent(""" 442 .. raw:: html 443 444 <script type='text/javascript'> 445 $(document).ready(function () { 446 $('.borg-options-table colgroup').remove(); 447 }) 448 </script> 449 """)) 450 451 def write_options_group(self, group, fp, with_title=True, base_indent=4): 452 def is_positional_group(group): 453 return any(not o.option_strings for o in group._group_actions) 454 455 indent = ' ' * base_indent 456 457 if is_positional_group(group): 458 for option in group._group_actions: 459 fp.write(option.metavar + '\n') 460 fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n') 461 return 462 463 if not group._group_actions: 464 return 465 466 if with_title: 467 fp.write('\n\n') 468 fp.write(group.title + '\n') 469 470 opts = OrderedDict() 471 472 for option in group._group_actions: 473 if option.metavar: 474 option_fmt = '%s ' + option.metavar 475 else: 476 option_fmt = '%s' 477 option_str = ', '.join(option_fmt % s for s in option.option_strings) 478 option_desc = textwrap.dedent((option.help or '') % option.__dict__) 479 opts[option_str] = textwrap.indent(option_desc, ' ' * 4) 480 481 padding = len(max(opts)) + 1 482 483 for option, desc in opts.items(): 484 fp.write(indent + option.ljust(padding) + desc + '\n') 485 486 487class build_man(Command): 488 description = 'build man pages' 489 490 user_options = [] 491 492 see_also = { 493 'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'), 494 'recreate': ('patterns', 'placeholders', 'compression'), 495 'list': ('info', 'diff', 'prune', 'patterns'), 496 'info': ('list', 'diff'), 497 'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'), 498 'key-import': ('key-export', ), 499 'key-export': ('key-import', ), 500 'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page 501 'umount': ('mount', ), 502 'extract': ('mount', ), 503 } 504 505 rst_prelude = textwrap.dedent(""" 506 .. role:: ref(title) 507 508 .. |project_name| replace:: Borg 509 510 """) 511 512 usage_group = { 513 'break-lock': 'lock', 514 'with-lock': 'lock', 515 516 'change-passphrase': 'key', 517 'key_change-passphrase': 'key', 518 'key_export': 'key', 519 'key_import': 'key', 520 'key_migrate-to-repokey': 'key', 521 522 'export-tar': 'tar', 523 524 'benchmark_crud': 'benchmark', 525 526 'umount': 'mount', 527 } 528 529 def initialize_options(self): 530 pass 531 532 def finalize_options(self): 533 pass 534 535 def run(self): 536 print('building man pages (in docs/man)', file=sys.stderr) 537 import borg 538 borg.doc_mode = 'build_man' 539 os.makedirs('docs/man', exist_ok=True) 540 # allows us to build docs without the C modules fully loaded during help generation 541 from borg.archiver import Archiver 542 parser = Archiver(prog='borg').build_parser() 543 borgfs_parser = Archiver(prog='borgfs').build_parser() 544 545 self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser}) 546 self.build_topic_pages(Archiver) 547 self.build_intro_page() 548 549 def generate_level(self, prefix, parser, Archiver, extra_choices=None): 550 is_subcommand = False 551 choices = {} 552 for action in parser._actions: 553 if action.choices is not None and 'SubParsersAction' in str(action.__class__): 554 is_subcommand = True 555 for cmd, parser in action.choices.items(): 556 choices[prefix + cmd] = parser 557 if extra_choices is not None: 558 choices.update(extra_choices) 559 if prefix and not choices: 560 return 561 562 for command, parser in sorted(choices.items()): 563 if command.startswith('debug') or command == 'help': 564 continue 565 566 if command == "borgfs": 567 man_title = command 568 else: 569 man_title = 'borg-' + command.replace(' ', '-') 570 print('building man page', man_title + '(1)', file=sys.stderr) 571 572 is_intermediary = self.generate_level(command + ' ', parser, Archiver) 573 574 doc, write = self.new_doc() 575 self.write_man_header(write, man_title, parser.description) 576 577 self.write_heading(write, 'SYNOPSIS') 578 if is_intermediary: 579 subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0] 580 for subcommand in subparsers.choices: 581 write('| borg', '[common options]', command, subcommand, '...') 582 self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand)) 583 else: 584 if command == "borgfs": 585 write(command, end='') 586 else: 587 write('borg', '[common options]', command, end='') 588 self.write_usage(write, parser) 589 write('\n') 590 591 description, _, notes = parser.epilog.partition('\n.. man NOTES') 592 593 if description: 594 self.write_heading(write, 'DESCRIPTION') 595 write(description) 596 597 if not is_intermediary: 598 self.write_heading(write, 'OPTIONS') 599 write('See `borg-common(1)` for common options of Borg commands.') 600 write() 601 self.write_options(write, parser) 602 603 self.write_examples(write, command) 604 605 if notes: 606 self.write_heading(write, 'NOTES') 607 write(notes) 608 609 self.write_see_also(write, man_title) 610 611 self.gen_man_page(man_title, doc.getvalue()) 612 613 # Generate the borg-common(1) man page with the common options. 614 if 'create' in choices: 615 doc, write = self.new_doc() 616 man_title = 'borg-common' 617 self.write_man_header(write, man_title, 'Common options of Borg commands') 618 619 common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] 620 621 self.write_heading(write, 'SYNOPSIS') 622 self.write_options_group(write, common_options) 623 self.write_see_also(write, man_title) 624 self.gen_man_page(man_title, doc.getvalue()) 625 626 return is_subcommand 627 628 def build_topic_pages(self, Archiver): 629 for topic, text in Archiver.helptext.items(): 630 doc, write = self.new_doc() 631 man_title = 'borg-' + topic 632 print('building man page', man_title + '(1)', file=sys.stderr) 633 634 self.write_man_header(write, man_title, 'Details regarding ' + topic) 635 self.write_heading(write, 'DESCRIPTION') 636 write(text) 637 self.gen_man_page(man_title, doc.getvalue()) 638 639 def build_intro_page(self): 640 print('building man page borg(1)', file=sys.stderr) 641 with open('docs/man_intro.rst') as fd: 642 man_intro = fd.read() 643 self.gen_man_page('borg', self.rst_prelude + man_intro) 644 645 def new_doc(self): 646 doc = io.StringIO(self.rst_prelude) 647 doc.read() 648 write = self.printer(doc) 649 return doc, write 650 651 def printer(self, fd): 652 def write(*args, **kwargs): 653 print(*args, file=fd, **kwargs) 654 return write 655 656 def write_heading(self, write, header, char='-', double_sided=False): 657 write() 658 if double_sided: 659 write(char * len(header)) 660 write(header) 661 write(char * len(header)) 662 write() 663 664 def write_man_header(self, write, title, description): 665 self.write_heading(write, title, '=', double_sided=True) 666 self.write_heading(write, description, double_sided=True) 667 # man page metadata 668 write(':Author: The Borg Collective') 669 write(':Date:', datetime.utcnow().date().isoformat()) 670 write(':Manual section: 1') 671 write(':Manual group: borg backup tool') 672 write() 673 674 def write_examples(self, write, command): 675 command = command.replace(' ', '_') 676 with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd: 677 usage = fd.read() 678 usage_include = '.. include:: %s.rst.inc' % command 679 begin = usage.find(usage_include) 680 end = usage.find('.. include', begin + 1) 681 # If a command has a dedicated anchor, it will occur before the command's include. 682 if 0 < usage.find('.. _', begin + 1) < end: 683 end = usage.find('.. _', begin + 1) 684 examples = usage[begin:end] 685 examples = examples.replace(usage_include, '') 686 examples = examples.replace('Examples\n~~~~~~~~', '') 687 examples = examples.replace('Miscellaneous Help\n------------------', '') 688 examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.') 689 examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight 690 examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE) 691 examples = examples.strip() 692 if examples: 693 self.write_heading(write, 'EXAMPLES', '-') 694 write(examples) 695 696 def write_see_also(self, write, man_title): 697 see_also = self.see_also.get(man_title.replace('borg-', ''), ()) 698 see_also = ['`borg-%s(1)`' % s for s in see_also] 699 see_also.insert(0, '`borg-common(1)`') 700 self.write_heading(write, 'SEE ALSO') 701 write(', '.join(see_also)) 702 703 def gen_man_page(self, name, rst): 704 from docutils.writers import manpage 705 from docutils.core import publish_string 706 from docutils.nodes import inline 707 from docutils.parsers.rst import roles 708 709 def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): 710 return [inline(rawtext, '#' + text)], [] 711 712 roles.register_local_role('issue', issue) 713 # We give the source_path so that docutils can find relative includes 714 # as-if the document where located in the docs/ directory. 715 man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer()) 716 with open('docs/man/%s.1' % name, 'wb') as fd: 717 fd.write(man_page) 718 719 def write_usage(self, write, parser): 720 if any(len(o.option_strings) for o in parser._actions): 721 write(' [options] ', end='') 722 for option in parser._actions: 723 if option.option_strings: 724 continue 725 write(format_metavar(option), end=' ') 726 727 def write_options(self, write, parser): 728 for group in parser._action_groups: 729 if group.title == 'Common options' or not group._group_actions: 730 continue 731 title = 'arguments' if group.title == 'positional arguments' else group.title 732 self.write_heading(write, title, '+') 733 self.write_options_group(write, group) 734 735 def write_options_group(self, write, group): 736 def is_positional_group(group): 737 return any(not o.option_strings for o in group._group_actions) 738 739 if is_positional_group(group): 740 for option in group._group_actions: 741 write(option.metavar) 742 write(textwrap.indent(option.help or '', ' ' * 4)) 743 return 744 745 opts = OrderedDict() 746 747 for option in group._group_actions: 748 if option.metavar: 749 option_fmt = '%s ' + option.metavar 750 else: 751 option_fmt = '%s' 752 option_str = ', '.join(option_fmt % s for s in option.option_strings) 753 option_desc = textwrap.dedent((option.help or '') % option.__dict__) 754 opts[option_str] = textwrap.indent(option_desc, ' ' * 4) 755 756 padding = len(max(opts)) + 1 757 758 for option, desc in opts.items(): 759 write(option.ljust(padding), desc) 760 761 762def rm(file): 763 try: 764 os.unlink(file) 765 print('rm', file) 766 except FileNotFoundError: 767 pass 768 769 770class Clean(Command): 771 user_options = [] 772 773 def initialize_options(self): 774 pass 775 776 def finalize_options(self): 777 pass 778 779 def run(self): 780 for source in cython_c_sources: 781 genc = source.replace('.pyx', '.c') 782 rm(genc) 783 for source in cython_cpp_sources: 784 gencpp = source.replace('.pyx', '.cpp') 785 rm(gencpp) 786 for source in cython_c_sources + cython_cpp_sources: 787 compiled_glob = source.replace('.pyx', '.cpython*') 788 for compiled in sorted(glob(compiled_glob)): 789 rm(compiled) 790 791cmdclass = { 792 'build_ext': build_ext, 793 'build_usage': build_usage, 794 'build_man': build_man, 795 'sdist': Sdist, 796 'clean2': Clean, 797} 798 799ext_modules = [] 800if not on_rtd: 801 compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, 802 define_macros=define_macros) 803 compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', 804 system_prefix=liblz4_prefix, system=liblz4_system, 805 **compress_ext_kwargs) 806 compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', 807 system_prefix=libzstd_prefix, system=libzstd_system, 808 multithreaded=False, legacy=False, **compress_ext_kwargs) 809 crypto_ext_kwargs = dict(sources=[crypto_ll_source], libraries=['crypto'], 810 include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) 811 crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', 812 system_prefix=libb2_prefix, system=libb2_system, 813 **crypto_ext_kwargs) 814 815 crypto_ext_kwargs = setup_xxhash.xxhash_ext_kwargs(bundled_path='src/borg/algorithms/xxh64', 816 system_prefix=libxxhash_prefix, system=libxxhash_system, 817 **crypto_ext_kwargs) 818 819 msgpack_endian = '__BIG_ENDIAN__' if (sys.byteorder == 'big') else '__LITTLE_ENDIAN__' 820 msgpack_macros = [(msgpack_endian, '1')] 821 msgpack_packer_ext_kwargs = dict( 822 sources=[msgpack_packer_source], 823 include_dirs=include_dirs, 824 library_dirs=library_dirs, 825 define_macros=msgpack_macros, 826 language='c++', 827 ) 828 msgpack_unpacker_ext_kwargs = dict( 829 sources=[msgpack_unpacker_source], 830 include_dirs=include_dirs, 831 library_dirs=library_dirs, 832 define_macros=msgpack_macros, 833 language='c++', 834 ) 835 836 ext_modules += [ 837 Extension('borg.algorithms.msgpack._packer', **msgpack_packer_ext_kwargs), 838 Extension('borg.algorithms.msgpack._unpacker', **msgpack_unpacker_ext_kwargs), 839 Extension('borg.compress', **compress_ext_kwargs), 840 Extension('borg.crypto.low_level', **crypto_ext_kwargs), 841 Extension('borg.hashindex', [hashindex_source]), 842 Extension('borg.item', [item_source]), 843 Extension('borg.chunker', [chunker_source]), 844 Extension('borg.algorithms.checksums', [checksums_source]), 845 ] 846 if not sys.platform.startswith(('win32', )): 847 ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) 848 if sys.platform == 'linux': 849 ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])) 850 ext_modules.append(Extension('borg.platform.syncfilerange', [platform_syncfilerange_source])) 851 elif sys.platform.startswith('freebsd'): 852 ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source])) 853 elif sys.platform == 'darwin': 854 ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source])) 855 856setup( 857 name='borgbackup', 858 use_scm_version={ 859 'write_to': 'src/borg/_version.py', 860 }, 861 author='The Borg Collective (see AUTHORS file)', 862 author_email='borgbackup@python.org', 863 url='https://borgbackup.readthedocs.io/', 864 description='Deduplicated, encrypted, authenticated and compressed backups', 865 long_description=long_description, 866 license='BSD', 867 platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ], 868 classifiers=[ 869 'Development Status :: 4 - Beta', 870 'Environment :: Console', 871 'Intended Audience :: System Administrators', 872 'License :: OSI Approved :: BSD License', 873 'Operating System :: POSIX :: BSD :: FreeBSD', 874 'Operating System :: POSIX :: BSD :: OpenBSD', 875 'Operating System :: POSIX :: BSD :: NetBSD', 876 'Operating System :: MacOS :: MacOS X', 877 'Operating System :: POSIX :: Linux', 878 'Programming Language :: Python', 879 'Programming Language :: Python :: 3', 880 'Programming Language :: Python :: 3.5', 881 'Programming Language :: Python :: 3.6', 882 'Programming Language :: Python :: 3.7', 883 'Programming Language :: Python :: 3.8', 884 'Programming Language :: Python :: 3.9', 885 'Programming Language :: Python :: 3.10', 886 'Topic :: Security :: Cryptography', 887 'Topic :: System :: Archiving :: Backup', 888 ], 889 packages=find_packages('src'), 890 package_dir={'': 'src'}, 891 zip_safe=False, 892 entry_points={ 893 'console_scripts': [ 894 'borg = borg.archiver:main', 895 'borgfs = borg.archiver:main', 896 ] 897 }, 898 # See also the MANIFEST.in file. 899 # We want to install all the files in the package directories... 900 include_package_data=True, 901 # ...except the source files which have been compiled (C extensions): 902 exclude_package_data={ 903 '': ['*.c', '*.h', '*.pyx', ], 904 }, 905 cmdclass=cmdclass, 906 ext_modules=ext_modules, 907 setup_requires=['setuptools_scm>=1.7'], 908 install_requires=install_requires, 909 extras_require=extras_require, 910) 911