diff --git a/borg/archive.py b/borg/archive.py index d6eff1ba9..27cd9d0aa 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -12,11 +12,12 @@ import sys import time from io import BytesIO from . import xattr -from .platform import acl_get, acl_set -from .chunker import Chunker -from .hashindex import ChunkIndex from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ - Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int + Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython +if have_cython(): + from .platform import acl_get, acl_set + from .chunker import Chunker + from .hashindex import ChunkIndex ITEMS_BUFFER = 1024 * 1024 diff --git a/borg/archiver.py b/borg/archiver.py index 57e30760e..b30a47b88 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -15,17 +15,18 @@ import textwrap import traceback from . import __version__ -from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS -from .compress import Compressor, COMPR_BUFFER -from .upgrader import AtticRepositoryUpgrader -from .repository import Repository -from .cache import Cache -from .key import key_creator from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec + is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython +if have_cython(): + from .compress import Compressor, COMPR_BUFFER + from .upgrader import AtticRepositoryUpgrader + from .repository import Repository + from .cache import Cache + from .key import key_creator +from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS from .remote import RepositoryServer, RemoteRepository has_lchflags = hasattr(os, 'lchflags') @@ -548,24 +549,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print(warning) return args - def run(self, args=None): - check_extension_modules() - keys_dir = get_keys_dir() - if not os.path.exists(keys_dir): - os.makedirs(keys_dir) - os.chmod(keys_dir, stat.S_IRWXU) - cache_dir = get_cache_dir() - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - os.chmod(cache_dir, stat.S_IRWXU) - with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd: - fd.write(textwrap.dedent(""" - Signature: 8a477f597d28d172789f06886806bc55 - # This file is a cache directory tag created by Borg. - # For information about cache directory tags, see: - # http://www.brynosaurus.com/cachedir/ - """).lstrip()) - common_parser = argparse.ArgumentParser(add_help=False) + def build_parser(self, args=None, prog=None): + common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='verbose output') @@ -576,11 +561,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH', help='set remote path to executable (default: "%(default)s")') - # We can't use argparse for "serve" since we don't want it to show up in "Available commands" - if args: - args = self.preprocess_args(args) - - parser = argparse.ArgumentParser(description='Borg %s - Deduplicated Backups' % __version__) + parser = argparse.ArgumentParser(prog=prog, description='Borg %s - Deduplicated Backups' % __version__) subparsers = parser.add_subparsers(title='Available commands') serve_epilog = textwrap.dedent(""" @@ -976,6 +957,30 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices)) subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') + return parser + + def run(self, args=None): + check_extension_modules() + keys_dir = get_keys_dir() + if not os.path.exists(keys_dir): + os.makedirs(keys_dir) + os.chmod(keys_dir, stat.S_IRWXU) + cache_dir = get_cache_dir() + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + os.chmod(cache_dir, stat.S_IRWXU) + with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd: + fd.write(textwrap.dedent(""" + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Borg. + # For information about cache directory tags, see: + # http://www.brynosaurus.com/cachedir/ + """).lstrip()) + + # We can't use argparse for "serve" since we don't want it to show up in "Available commands" + if args: + args = self.preprocess_args(args) + parser = self.build_parser(args) args = parser.parse_args(args or ['-h']) self.verbose = args.verbose diff --git a/borg/helpers.py b/borg/helpers.py index 47d454bec..10ee0441a 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -18,9 +18,25 @@ from operator import attrgetter import msgpack -from . import hashindex -from . import chunker -from . import crypto +def have_cython(): + """allow for a way to disable Cython includes + + this is used during usage docs build, in setup.py. It is to avoid + loading the Cython libraries which are built, but sometimes not in + the search path (namely, during Tox runs). + + we simply check an environment variable (``BORG_CYTHON_DISABLE``) + which, when set (to anything) will disable includes of Cython + libraries in key places to enable usage docs to be built. + + :returns: True if Cython is available, False otherwise. + """ + return not os.environ.get('BORG_CYTHON_DISABLE') + +if have_cython(): + from . import hashindex + from . import chunker + from . import crypto class Error(Exception): diff --git a/borg/key.py b/borg/key.py index 7067a4454..81f9185c8 100644 --- a/borg/key.py +++ b/borg/key.py @@ -7,9 +7,10 @@ import textwrap import hmac from hashlib import sha256 -from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks -from .compress import Compressor, COMPR_BUFFER -from .helpers import IntegrityError, get_keys_dir, Error +from .helpers import IntegrityError, get_keys_dir, Error, have_cython +if have_cython(): + from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks + from .compress import Compressor, COMPR_BUFFER PREFIX = b'\0' * 8 diff --git a/borg/repository.py b/borg/repository.py index 932e4fef3..c762d09dc 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -8,8 +8,9 @@ import struct import sys from zlib import crc32 -from .hashindex import NSIndex -from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify +from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython +if have_cython(): + from .hashindex import NSIndex from .locking import UpgradableLock from .lrucache import LRUCache diff --git a/docs/Makefile b/docs/Makefile index 133080cdb..fb470c6a9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,7 +36,7 @@ help: clean: -rm -rf $(BUILDDIR)/* -html: usage api.rst +html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -139,32 +139,3 @@ gh-io: html inotify: html while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done - -# generate list of targets -usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//') - -# generate help file based on usage -usage/%.rst.inc: ../borg/archiver.py - @echo generating usage for $* - @printf ".. _borg_$*:\n\n" > $@ - @printf "borg $*\n" >> $@ - @echo -n borg $* | tr 'a-z- ' '-' >> $@ - @printf "\n::\n\n" >> $@ - @borg help $* --usage-only | sed -e 's/^/ /' >> $@ - @printf "\nDescription\n~~~~~~~~~~~\n" >> $@ - @borg help $* --epilog-only >> $@ - -api.rst: Makefile - @echo "auto-generating API documentation" - @echo "Borg Backup API documentation" > $@ - @echo "=============================" >> $@ - @echo "" >> $@ - @for mod in ../borg/*.pyx ../borg/*.py; do \ - if echo "$$mod" | grep -q "/_"; then \ - continue ; \ - fi ; \ - printf ".. automodule:: "; \ - echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \ - echo " :members:"; \ - echo " :undoc-members:"; \ - done >> $@ diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html index a311fe558..c9ae280b2 100644 --- a/docs/_themes/local/sidebarusefullinks.html +++ b/docs/_themes/local/sidebarusefullinks.html @@ -3,7 +3,7 @@

Useful Links