From 28cbc6cbd1b3d70be284a69bc8049b435e101a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 7 Oct 2015 19:16:49 -0400 Subject: [PATCH 01/20] fix build on RTFD first off, this required ticking the `Install your project inside a virtualenv using setup.py install` box in the advanced config. then, i had to disable all the C extensions build and disable some checks, based on whether we are running on RTD or not. still missing: usage builds and possibly other stuff that is in our Makefile and not in setup.py. --- setup.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 68a3db8d3..2e8ed1f18 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,9 @@ if my_python < min_python: print("Borg requires Python %d.%d or later" % min_python) sys.exit(1) +# Are we building on ReadTheDocs? +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. install_requires=['msgpack-python>=0.4.6', ] @@ -64,7 +67,7 @@ except ImportError: from distutils.command.build_ext import build_ext if not all(os.path.exists(path) for path in [ compress_source, crypto_source, chunker_source, hashindex_source, - platform_linux_source, platform_freebsd_source]): + platform_linux_source, platform_freebsd_source]) and not on_rtd: raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') @@ -103,10 +106,11 @@ possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local if os.environ.get('BORG_LZ4_PREFIX'): possible_openssl_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) lz4_prefix = detect_lz4(possible_lz4_prefixes) -if not lz4_prefix: +if lz4_prefix: + include_dirs.append(os.path.join(lz4_prefix, 'include')) + library_dirs.append(os.path.join(lz4_prefix, 'lib')) +elif not on_rtd: raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes))) -include_dirs.append(os.path.join(lz4_prefix, 'include')) -library_dirs.append(os.path.join(lz4_prefix, 'lib')) with open('README.rst', 'r') as fd: @@ -114,18 +118,20 @@ with open('README.rst', 'r') as fd: cmdclass = {'build_ext': build_ext, 'sdist': Sdist} -ext_modules = [ +ext_modules = [] +if not on_rtd: + ext_modules += [ Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs), Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs), Extension('borg.chunker', [chunker_source]), Extension('borg.hashindex', [hashindex_source]) ] -if sys.platform.startswith('linux'): - ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl'])) -elif sys.platform.startswith('freebsd'): - ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source])) -elif sys.platform == 'darwin': - ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source])) + if sys.platform.startswith('linux'): + ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl'])) + elif sys.platform.startswith('freebsd'): + ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source])) + elif sys.platform == 'darwin': + ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source])) setup( name='borgbackup', From 4787424a668d71266063f09d1fa02b1cae310bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 7 Oct 2015 19:55:56 -0400 Subject: [PATCH 02/20] move API generation to setup.py --- docs/Makefile | 17 +---------------- setup.py | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 133080cdb..1f3f7d76c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,7 +36,7 @@ help: clean: -rm -rf $(BUILDDIR)/* -html: usage api.rst +html: usage $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -153,18 +153,3 @@ usage/%.rst.inc: ../borg/archiver.py @borg help $* --usage-only | sed -e 's/^/ /' >> $@ @printf "\nDescription\n~~~~~~~~~~~\n" >> $@ @borg help $* --epilog-only >> $@ - -api.rst: Makefile - @echo "auto-generating API documentation" - @echo "Borg Backup API documentation" > $@ - @echo "=============================" >> $@ - @echo "" >> $@ - @for mod in ../borg/*.pyx ../borg/*.py; do \ - if echo "$$mod" | grep -q "/_"; then \ - continue ; \ - fi ; \ - printf ".. automodule:: "; \ - echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \ - echo " :members:"; \ - echo " :undoc-members:"; \ - done >> $@ diff --git a/setup.py b/setup.py index 2e8ed1f18..ffcbac101 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,10 @@ import os import sys from glob import glob +from distutils.command.build import build +from distutils.core import Command +from distutils.errors import DistutilsOptionError + min_python = (3, 2) my_python = sys.version_info @@ -115,8 +119,42 @@ elif not on_rtd: with open('README.rst', 'r') as fd: long_description = fd.read() +class build_api(Command): + description = "generate a basic api.rst file based on the modules available" -cmdclass = {'build_ext': build_ext, 'sdist': Sdist} + user_options = [ + ('output=', 'O', 'output directory'), + ] + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + print("auto-generating API documentation") + with open("docs/api.rst", "w") as api: + api.write(""" +Borg Backup API documentation" +============================= +""") + for mod in glob('borg/*.py') + glob('borg/*.pyx'): + print("examining module %s" % mod) + if "/_" not in mod: + api.write(""" +.. automodule:: %s + :members: + :undoc-members: +""" % mod) + +# (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands +build.sub_commands.append(('build_api', None)) + +cmdclass = { + 'build_ext': build_ext, + 'build_api': build_api, + 'sdist': Sdist +} ext_modules = [] if not on_rtd: From 13d356854805094e465a14b3d61d62b0875b9285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 7 Oct 2015 21:07:12 -0400 Subject: [PATCH 03/20] move usage generation to setup.py this is an unfortunate rewrite of the manpage creation code mentionned in #208. ideally, this would be rewritten into a class that can generate both man pages and .rst files. --- borg/archiver.py | 50 +++++++++++++++++++++++++---------------------- docs/Makefile | 16 +-------------- setup.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 38 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 57e30760e..9b18486c8 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -548,24 +548,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print(warning) return args - def run(self, args=None): - check_extension_modules() - keys_dir = get_keys_dir() - if not os.path.exists(keys_dir): - os.makedirs(keys_dir) - os.chmod(keys_dir, stat.S_IRWXU) - cache_dir = get_cache_dir() - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - os.chmod(cache_dir, stat.S_IRWXU) - with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd: - fd.write(textwrap.dedent(""" - Signature: 8a477f597d28d172789f06886806bc55 - # This file is a cache directory tag created by Borg. - # For information about cache directory tags, see: - # http://www.brynosaurus.com/cachedir/ - """).lstrip()) - common_parser = argparse.ArgumentParser(add_help=False) + def build_parser(self, args=None, prog=None): + common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='verbose output') @@ -576,11 +560,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH', help='set remote path to executable (default: "%(default)s")') - # We can't use argparse for "serve" since we don't want it to show up in "Available commands" - if args: - args = self.preprocess_args(args) - - parser = argparse.ArgumentParser(description='Borg %s - Deduplicated Backups' % __version__) + parser = argparse.ArgumentParser(prog=prog, description='Borg %s - Deduplicated Backups' % __version__) subparsers = parser.add_subparsers(title='Available commands') serve_epilog = textwrap.dedent(""" @@ -976,6 +956,30 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices)) subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') + return parser + + def run(self, args=None): + check_extension_modules() + keys_dir = get_keys_dir() + if not os.path.exists(keys_dir): + os.makedirs(keys_dir) + os.chmod(keys_dir, stat.S_IRWXU) + cache_dir = get_cache_dir() + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + os.chmod(cache_dir, stat.S_IRWXU) + with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd: + fd.write(textwrap.dedent(""" + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Borg. + # For information about cache directory tags, see: + # http://www.brynosaurus.com/cachedir/ + """).lstrip()) + + # We can't use argparse for "serve" since we don't want it to show up in "Available commands" + if args: + args = self.preprocess_args(args) + parser = self.build_parser(args) args = parser.parse_args(args or ['-h']) self.verbose = args.verbose diff --git a/docs/Makefile b/docs/Makefile index 1f3f7d76c..fb470c6a9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,7 +36,7 @@ help: clean: -rm -rf $(BUILDDIR)/* -html: usage +html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -139,17 +139,3 @@ gh-io: html inotify: html while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done - -# generate list of targets -usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//') - -# generate help file based on usage -usage/%.rst.inc: ../borg/archiver.py - @echo generating usage for $* - @printf ".. _borg_$*:\n\n" > $@ - @printf "borg $*\n" >> $@ - @echo -n borg $* | tr 'a-z- ' '-' >> $@ - @printf "\n::\n\n" >> $@ - @borg help $* --usage-only | sed -e 's/^/ /' >> $@ - @printf "\nDescription\n~~~~~~~~~~~\n" >> $@ - @borg help $* --epilog-only >> $@ diff --git a/setup.py b/setup.py index ffcbac101..2f29b4a20 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 *-* import os +import re import sys from glob import glob @@ -119,6 +120,54 @@ elif not on_rtd: with open('README.rst', 'r') as fd: long_description = fd.read() + +class build_usage(Command): + description = "generate usage for each command" + + user_options = [ + ('output=', 'O', 'output directory'), + ] + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + import pdb + print('generating usage docs') + from borg.archiver import Archiver + parser = Archiver().build_parser(prog='borg') + choices = {} + for action in parser._actions: + if action.choices is not None: + choices.update(action.choices) + print('found commands: %s' % list(choices.keys())) + if not os.path.exists('docs/usage'): + os.mkdir('docs/usage') + for command, parser in choices.items(): + if command is 'help': + continue + with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile: + print('generating help for %s' % command) + cmdfile.write(""".. _borg_{command}: + +borg {command} +{underline} +:: + +""".format(**{"command": command, + "underline": '-' * len('borg ' + command)})) + epilog = parser.epilog + parser.epilog = None + cmdfile.write(re.sub("^", " ", parser.format_help(), flags=re.M)) + cmdfile.write(""" +Description +~~~~~~~~~~~ +""") + cmdfile.write(epilog) + + class build_api(Command): description = "generate a basic api.rst file based on the modules available" @@ -149,10 +198,12 @@ Borg Backup API documentation" # (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands build.sub_commands.append(('build_api', None)) +build.sub_commands.append(('build_usage', None)) cmdclass = { 'build_ext': build_ext, 'build_api': build_api, + 'build_usage': build_usage, 'sdist': Sdist } From 8fe56f001cfeeb027162b38da14cfec9f6038754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 7 Oct 2015 21:10:09 -0400 Subject: [PATCH 04/20] main website becomes RTD, main website link is pointless --- docs/_themes/local/sidebarusefullinks.html | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html index a311fe558..a323ebe69 100644 --- a/docs/_themes/local/sidebarusefullinks.html +++ b/docs/_themes/local/sidebarusefullinks.html @@ -3,7 +3,6 @@

Useful Links

    -
  • Main Web Site
  • Releases
  • PyPI packages
  • Current ChangeLog
    • From 9cbc868764a83fe6e1d39e07c5dfdef3e8baa72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 7 Oct 2015 22:24:30 -0400 Subject: [PATCH 05/20] make tests and build work again in usage using environment this is such a crude hack it is totally embarrassing.... the proper solution would probably be to move the `build_parser()` function out of `Archiver` completely, but this is such an undertaking that i doubt it is worth doing since we're looking at switching to click anyways. the main problem in moving build_parser() out is that it references `self` all the time, so it *needs* an archiver context that it can reuse. we could make the function static and pass self in there by hand, but it seems like almost a worse hack... and besides, we would need to load the archiver in order to do that, which would break usage all over again... --- borg/archive.py | 7 ++++--- borg/archiver.py | 11 ++++++----- borg/helpers.py | 7 ++++--- borg/key.py | 5 +++-- borg/repository.py | 3 ++- setup.py | 2 ++ 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index d6eff1ba9..8350bbebc 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -12,9 +12,10 @@ import sys import time from io import BytesIO from . import xattr -from .platform import acl_get, acl_set -from .chunker import Chunker -from .hashindex import ChunkIndex +if not os.environ.get('BORG_GEN_USAGE', False): + from .platform import acl_get, acl_set + from .chunker import Chunker + from .hashindex import ChunkIndex from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int diff --git a/borg/archiver.py b/borg/archiver.py index 9b18486c8..bd49da7c9 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -15,12 +15,13 @@ import textwrap import traceback from . import __version__ +if not os.environ.get('BORG_GEN_USAGE', False): + from .compress import Compressor, COMPR_BUFFER + from .upgrader import AtticRepositoryUpgrader + from .repository import Repository + from .cache import Cache + from .key import key_creator from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS -from .compress import Compressor, COMPR_BUFFER -from .upgrader import AtticRepositoryUpgrader -from .repository import Repository -from .cache import Cache -from .key import key_creator from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ diff --git a/borg/helpers.py b/borg/helpers.py index 47d454bec..175ebf15c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -18,9 +18,10 @@ from operator import attrgetter import msgpack -from . import hashindex -from . import chunker -from . import crypto +if not os.environ.get('BORG_GEN_USAGE', False): + from . import hashindex + from . import chunker + from . import crypto class Error(Exception): diff --git a/borg/key.py b/borg/key.py index 7067a4454..877b37711 100644 --- a/borg/key.py +++ b/borg/key.py @@ -7,8 +7,9 @@ import textwrap import hmac from hashlib import sha256 -from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks -from .compress import Compressor, COMPR_BUFFER +if not os.environ.get('BORG_GEN_USAGE', False): + from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks + from .compress import Compressor, COMPR_BUFFER from .helpers import IntegrityError, get_keys_dir, Error PREFIX = b'\0' * 8 diff --git a/borg/repository.py b/borg/repository.py index 932e4fef3..30163b02c 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -8,7 +8,8 @@ import struct import sys from zlib import crc32 -from .hashindex import NSIndex +if not os.environ.get('BORG_GEN_USAGE', False): + from .hashindex import NSIndex from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify from .locking import UpgradableLock from .lrucache import LRUCache diff --git a/setup.py b/setup.py index 2f29b4a20..3a5286b70 100644 --- a/setup.py +++ b/setup.py @@ -136,6 +136,8 @@ class build_usage(Command): def run(self): import pdb print('generating usage docs') + # XXX: gross hack: allows us to skip loading C modules during help generation + os.environ['BORG_GEN_USAGE'] = "True" from borg.archiver import Archiver parser = Archiver().build_parser(prog='borg') choices = {} From c93d975d3f5aa240899dfdba4fa4a83b6f8c5049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:20:59 -0400 Subject: [PATCH 06/20] remove debugging code --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 3a5286b70..1b411c355 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,6 @@ class build_usage(Command): pass def run(self): - import pdb print('generating usage docs') # XXX: gross hack: allows us to skip loading C modules during help generation os.environ['BORG_GEN_USAGE'] = "True" From 45d9c6b3b7b6b23c0fe292798d365050842f32fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:21:52 -0400 Subject: [PATCH 07/20] fix formatting --- setup.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 1b411c355..955d5d758 100644 --- a/setup.py +++ b/setup.py @@ -151,21 +151,14 @@ class build_usage(Command): continue with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile: print('generating help for %s' % command) - cmdfile.write(""".. _borg_{command}: - -borg {command} -{underline} -:: - -""".format(**{"command": command, - "underline": '-' * len('borg ' + command)})) + params = {"command": command, + "underline": '-' * len('borg ' + command)} + cmdfile.write(".. _borg_{command}:\n\n".format(**params)) + cmdfile.write("borg {command}\n{underline}\n::\n\n".format(**params)) epilog = parser.epilog parser.epilog = None cmdfile.write(re.sub("^", " ", parser.format_help(), flags=re.M)) - cmdfile.write(""" -Description -~~~~~~~~~~~ -""") + cmdfile.write("\nDescription\n~~~~~~~~~~~\n") cmdfile.write(epilog) From 8190e33a64790a42d545e60cf9298e1c247cd99d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:22:52 -0400 Subject: [PATCH 08/20] faster and cleaner rtd check for cython --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 955d5d758..dcf486c1e 100644 --- a/setup.py +++ b/setup.py @@ -70,9 +70,9 @@ except ImportError: platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c') platform_darwin_source = platform_darwin_source.replace('.pyx', '.c') from distutils.command.build_ext import build_ext - if not all(os.path.exists(path) for path in [ + if not on_rtd and not all(os.path.exists(path) for path in [ compress_source, crypto_source, chunker_source, hashindex_source, - platform_linux_source, platform_freebsd_source]) and not on_rtd: + platform_linux_source, platform_freebsd_source]): raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') From 7ccf6b32a6c8770c497cf77a376f60b4ef6ed8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:23:37 -0400 Subject: [PATCH 09/20] simplify RTD env check --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dcf486c1e..b691ac2e6 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if my_python < min_python: sys.exit(1) # Are we building on ReadTheDocs? -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get('READTHEDOCS') # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. From da02f373c7e6f136106e652b54672b6a54e847d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:24:46 -0400 Subject: [PATCH 10/20] Revert "main website becomes RTD, main website link is pointless" Instead, we put the readthedocs.org site as main website. This reverts commit 8fe56f001cfeeb027162b38da14cfec9f6038754. --- docs/_themes/local/sidebarusefullinks.html | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html index a323ebe69..c9ae280b2 100644 --- a/docs/_themes/local/sidebarusefullinks.html +++ b/docs/_themes/local/sidebarusefullinks.html @@ -3,6 +3,7 @@

    Useful Links

      +
    • Main Web Site
    • Releases
    • PyPI packages
    • Current ChangeLog
      • From 6f9e04bc2101936ed69bbd1f8fffe94a74db0d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:29:02 -0400 Subject: [PATCH 11/20] generalise the cython check hack instead of applying this only to usage generation, use it as a generic mechanism to disable loading of Cython code. it may be incomplete: there may be other places where Cython code is loaded that is not checked, but that is sufficient to build the usage docs. the environment variable used is documented as such in the docs/usage.rst. we also move the check to a helper function and document it better. this has the unfortunate side effect of moving includes around, but I can't think of a better way. --- borg/archive.py | 6 +++--- borg/archiver.py | 12 ++++++------ borg/helpers.py | 15 ++++++++++++++- borg/key.py | 4 ++-- borg/repository.py | 4 ++-- docs/usage.rst | 6 ++++++ setup.py | 8 ++++++-- 7 files changed, 39 insertions(+), 16 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 8350bbebc..43fd6e66c 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -12,12 +12,12 @@ import sys import time from io import BytesIO from . import xattr -if not os.environ.get('BORG_GEN_USAGE', False): +from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ + Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython +if not detect_cython(): from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex -from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ - Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int ITEMS_BUFFER = 1024 * 1024 diff --git a/borg/archiver.py b/borg/archiver.py index bd49da7c9..4b831f2e0 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -15,18 +15,18 @@ import textwrap import traceback from . import __version__ -if not os.environ.get('BORG_GEN_USAGE', False): +from .helpers import Error, location_validator, format_time, format_file_size, \ + format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ + get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ + Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ + is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython +if not detect_cython(): from .compress import Compressor, COMPR_BUFFER from .upgrader import AtticRepositoryUpgrader from .repository import Repository from .cache import Cache from .key import key_creator from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS -from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ - get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ - Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec from .remote import RepositoryServer, RemoteRepository has_lchflags = hasattr(os, 'lchflags') diff --git a/borg/helpers.py b/borg/helpers.py index 175ebf15c..250f899af 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -18,7 +18,20 @@ from operator import attrgetter import msgpack -if not os.environ.get('BORG_GEN_USAGE', False): +def detect_cython(): + """allow for a way to disable Cython includes + + this is used during usage docs build, in setup.py. It is to avoid + loading the Cython libraries which are built, but sometimes not in + the search path (namely, during Tox runs). + + we simply check an environment variable (``BORG_CYTHON_DISABLE``) + which, when set (to anything) will disable includes of Cython + libraries in key places to enable usage docs to be built. + """ + return os.environ.get('BORG_CYTHON_DISABLE') + +if not detect_cython(): from . import hashindex from . import chunker from . import crypto diff --git a/borg/key.py b/borg/key.py index 877b37711..ace5c37e8 100644 --- a/borg/key.py +++ b/borg/key.py @@ -7,10 +7,10 @@ import textwrap import hmac from hashlib import sha256 -if not os.environ.get('BORG_GEN_USAGE', False): +from .helpers import IntegrityError, get_keys_dir, Error, detect_cython +if not detect_cython(): from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from .compress import Compressor, COMPR_BUFFER -from .helpers import IntegrityError, get_keys_dir, Error PREFIX = b'\0' * 8 diff --git a/borg/repository.py b/borg/repository.py index 30163b02c..80a8adbf9 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -8,9 +8,9 @@ import struct import sys from zlib import crc32 -if not os.environ.get('BORG_GEN_USAGE', False): +from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython +if not detect_cython(): from .hashindex import NSIndex -from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify from .locking import UpgradableLock from .lrucache import LRUCache diff --git a/docs/usage.rst b/docs/usage.rst index 6bd292e14..fb7768528 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -60,6 +60,12 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do For "Warning: The repository at location ... was previously located at ..." BORG_CHECK_I_KNOW_WHAT_I_AM_DOING For "Warning: 'check --repair' is an experimental feature that might result in data loss." + BORG_CYTHON_DISABLE + Disables the loading of Cython modules. This is currently + experimentaly and is used only to generate usage docs at build + time, it's unlikely to produce good results on a regular + run. The variable should be set to the calling class, and + should be unique. It is currently only used by ``build_usage``. Directories: BORG_KEYS_DIR diff --git a/setup.py b/setup.py index b691ac2e6..67fe0c738 100644 --- a/setup.py +++ b/setup.py @@ -135,10 +135,14 @@ class build_usage(Command): def run(self): print('generating usage docs') - # XXX: gross hack: allows us to skip loading C modules during help generation - os.environ['BORG_GEN_USAGE'] = "True" + # allows us to build docs without the C modules fully loaded during help generation + if 'BORG_CYTHON_DISABLE' not in os.environ: + os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__ from borg.archiver import Archiver parser = Archiver().build_parser(prog='borg') + # return to regular Cython configuration, if we changed it + if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__: + del os.environ['BORG_CYTHON_DISABLE'] choices = {} for action in parser._actions: if action.choices is not None: From 824f9c72a2c6d7e09f5ddba7747d7d963b5bebdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 08:54:49 -0400 Subject: [PATCH 12/20] cosmetic: s/cmdfile/doc/ --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 67fe0c738..c3e6ad887 100644 --- a/setup.py +++ b/setup.py @@ -153,17 +153,17 @@ class build_usage(Command): for command, parser in choices.items(): if command is 'help': continue - with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile: + with open('docs/usage/%s.rst.inc' % command, 'w') as doc: print('generating help for %s' % command) params = {"command": command, "underline": '-' * len('borg ' + command)} - cmdfile.write(".. _borg_{command}:\n\n".format(**params)) - cmdfile.write("borg {command}\n{underline}\n::\n\n".format(**params)) + doc.write(".. _borg_{command}:\n\n".format(**params)) + doc.write("borg {command}\n{underline}\n::\n\n".format(**params)) epilog = parser.epilog parser.epilog = None - cmdfile.write(re.sub("^", " ", parser.format_help(), flags=re.M)) - cmdfile.write("\nDescription\n~~~~~~~~~~~\n") - cmdfile.write(epilog) + doc.write(re.sub("^", " ", parser.format_help(), flags=re.M)) + doc.write("\nDescription\n~~~~~~~~~~~\n") + doc.write(epilog) class build_api(Command): From 86487d192a9a5ab7ff4eedb92d793485b4c30268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 10:26:02 -0400 Subject: [PATCH 13/20] use build_py to fix build on RTD it seems that our subcommands are ignored over there, for some mysterious reason. --- setup.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/setup.py b/setup.py index c3e6ad887..7376c56e2 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,8 @@ from glob import glob from distutils.command.build import build from distutils.core import Command from distutils.errors import DistutilsOptionError +from distutils import log +from setuptools.command.build_py import build_py min_python = (3, 2) my_python = sys.version_info @@ -195,13 +197,38 @@ Borg Backup API documentation" """ % mod) # (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands +# seems like this doesn't work on RTD, see below for build_py hack. build.sub_commands.append(('build_api', None)) build.sub_commands.append(('build_usage', None)) + +class build_py_custom(build_py): + """override build_py to also build our stuf + + it is unclear why this is necessary, but in some environments + (Readthedocs.org, specifically), the above + ``build.sub_commands.append()`` doesn't seem to have an effect: + our custom build commands seem to be ignored when running + ``setup.py install``. + + This class overrides the ``build_py`` target by forcing it to run + our custom steps as well. + + See also the `bug report on RTD + `_. + """ + def run(self): + super().run() + self.announce('calling custom build steps', level=log.INFO) + self.run_command('build_api') + self.run_command('build_usage') + + cmdclass = { 'build_ext': build_ext, 'build_api': build_api, 'build_usage': build_usage, + 'build_py': build_py_custom, 'sdist': Sdist } From 6c5a7733a222c0da1d0950593ba23bdc9b2579f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 10:37:19 -0400 Subject: [PATCH 14/20] force build-ext build it seems that what worked in the debug branch is not working in the main branch, even though the commit IDs are exactly the same. the RTD environment doesn't seem really reliable... besides, we want to build extensions before the rest, so should run it first, in order to have msgpack loaded. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7376c56e2..8f31a927f 100644 --- a/setup.py +++ b/setup.py @@ -220,6 +220,7 @@ class build_py_custom(build_py): def run(self): super().run() self.announce('calling custom build steps', level=log.INFO) + self.run_command('build_ext') self.run_command('build_api') self.run_command('build_usage') From 2e9bd5a8832364d067ecf11bc6d26dbcf2701113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 14:57:32 -0400 Subject: [PATCH 15/20] cosmetic: s/api/doc/ --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8f31a927f..e3338280c 100644 --- a/setup.py +++ b/setup.py @@ -182,15 +182,15 @@ class build_api(Command): def run(self): print("auto-generating API documentation") - with open("docs/api.rst", "w") as api: - api.write(""" + with open("docs/api.rst", "w") as doc: + doc.write(""" Borg Backup API documentation" ============================= """) for mod in glob('borg/*.py') + glob('borg/*.pyx'): print("examining module %s" % mod) if "/_" not in mod: - api.write(""" + doc.write(""" .. automodule:: %s :members: :undoc-members: From f98998f042200d497de75f156c434f3fea216b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 15:26:40 -0400 Subject: [PATCH 16/20] fix logical inversion in the semantics of detect_cython() --- borg/archive.py | 2 +- borg/archiver.py | 2 +- borg/helpers.py | 4 ++-- borg/key.py | 2 +- borg/repository.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 43fd6e66c..c50759bc6 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -14,7 +14,7 @@ from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython -if not detect_cython(): +if detect_cython(): from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex diff --git a/borg/archiver.py b/borg/archiver.py index 4b831f2e0..dca7e4af7 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -20,7 +20,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython -if not detect_cython(): +if detect_cython(): from .compress import Compressor, COMPR_BUFFER from .upgrader import AtticRepositoryUpgrader from .repository import Repository diff --git a/borg/helpers.py b/borg/helpers.py index 250f899af..a57feda96 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -29,9 +29,9 @@ def detect_cython(): which, when set (to anything) will disable includes of Cython libraries in key places to enable usage docs to be built. """ - return os.environ.get('BORG_CYTHON_DISABLE') + return not os.environ.get('BORG_CYTHON_DISABLE') -if not detect_cython(): +if detect_cython(): from . import hashindex from . import chunker from . import crypto diff --git a/borg/key.py b/borg/key.py index ace5c37e8..2d0ffc86e 100644 --- a/borg/key.py +++ b/borg/key.py @@ -8,7 +8,7 @@ import hmac from hashlib import sha256 from .helpers import IntegrityError, get_keys_dir, Error, detect_cython -if not detect_cython(): +if detect_cython(): from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from .compress import Compressor, COMPR_BUFFER diff --git a/borg/repository.py b/borg/repository.py index 80a8adbf9..762a40302 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -9,7 +9,7 @@ import sys from zlib import crc32 from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython -if not detect_cython(): +if detect_cython(): from .hashindex import NSIndex from .locking import UpgradableLock from .lrucache import LRUCache From ff483fe48552389a9e898ba9da2ccb2e7b3e26f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 15:26:40 -0400 Subject: [PATCH 17/20] fix logical inversion in the semantics of detect_cython() --- borg/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borg/helpers.py b/borg/helpers.py index a57feda96..bfbc45a21 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -28,6 +28,8 @@ def detect_cython(): we simply check an environment variable (``BORG_CYTHON_DISABLE``) which, when set (to anything) will disable includes of Cython libraries in key places to enable usage docs to be built. + + :returns: True if Cython is available, False otherwise. """ return not os.environ.get('BORG_CYTHON_DISABLE') From 2f803b6489278913a270449c2d1b1f90cfa953c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 15:28:53 -0400 Subject: [PATCH 18/20] fix typo, split sentence --- docs/usage.rst | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index fb7768528..b316f17d9 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -62,8 +62,8 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do For "Warning: 'check --repair' is an experimental feature that might result in data loss." BORG_CYTHON_DISABLE Disables the loading of Cython modules. This is currently - experimentaly and is used only to generate usage docs at build - time, it's unlikely to produce good results on a regular + experimental and is used only to generate usage docs at build + time. It is unlikely to produce good results on a regular run. The variable should be set to the calling class, and should be unique. It is currently only used by ``build_usage``. diff --git a/setup.py b/setup.py index e3338280c..f1bb8f68b 100644 --- a/setup.py +++ b/setup.py @@ -203,7 +203,7 @@ build.sub_commands.append(('build_usage', None)) class build_py_custom(build_py): - """override build_py to also build our stuf + """override build_py to also build our stuff it is unclear why this is necessary, but in some environments (Readthedocs.org, specifically), the above From 7ba4d47f6e849dc6a3ebd836afa29bc44bb339ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 15:29:50 -0400 Subject: [PATCH 19/20] clarify the class name part --- docs/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index b316f17d9..80f3eaa2f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -64,8 +64,8 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do Disables the loading of Cython modules. This is currently experimental and is used only to generate usage docs at build time. It is unlikely to produce good results on a regular - run. The variable should be set to the calling class, and - should be unique. It is currently only used by ``build_usage``. + run. The variable should be set to the name of the calling class, and + should be unique across all of borg. It is currently only used by ``build_usage``. Directories: BORG_KEYS_DIR From 423ff45d816e9608edd8e800db24ea8522e50802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 8 Oct 2015 15:34:44 -0400 Subject: [PATCH 20/20] rename cython detection function "have" has clearer semantics than "detect" --- borg/archive.py | 4 ++-- borg/archiver.py | 4 ++-- borg/helpers.py | 4 ++-- borg/key.py | 4 ++-- borg/repository.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index c50759bc6..27cd9d0aa 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -13,8 +13,8 @@ import time from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ - Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython -if detect_cython(): + Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython +if have_cython(): from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex diff --git a/borg/archiver.py b/borg/archiver.py index dca7e4af7..b30a47b88 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -19,8 +19,8 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython -if detect_cython(): + is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython +if have_cython(): from .compress import Compressor, COMPR_BUFFER from .upgrader import AtticRepositoryUpgrader from .repository import Repository diff --git a/borg/helpers.py b/borg/helpers.py index bfbc45a21..10ee0441a 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -18,7 +18,7 @@ from operator import attrgetter import msgpack -def detect_cython(): +def have_cython(): """allow for a way to disable Cython includes this is used during usage docs build, in setup.py. It is to avoid @@ -33,7 +33,7 @@ def detect_cython(): """ return not os.environ.get('BORG_CYTHON_DISABLE') -if detect_cython(): +if have_cython(): from . import hashindex from . import chunker from . import crypto diff --git a/borg/key.py b/borg/key.py index 2d0ffc86e..81f9185c8 100644 --- a/borg/key.py +++ b/borg/key.py @@ -7,8 +7,8 @@ import textwrap import hmac from hashlib import sha256 -from .helpers import IntegrityError, get_keys_dir, Error, detect_cython -if detect_cython(): +from .helpers import IntegrityError, get_keys_dir, Error, have_cython +if have_cython(): from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from .compress import Compressor, COMPR_BUFFER diff --git a/borg/repository.py b/borg/repository.py index 762a40302..c762d09dc 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -8,8 +8,8 @@ import struct import sys from zlib import crc32 -from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython -if detect_cython(): +from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython +if have_cython(): from .hashindex import NSIndex from .locking import UpgradableLock from .lrucache import LRUCache