Merge pull request #262 from anarcat/rtfd-fixup

fix build on readthedocs.org

borg/archive.py

@@ -12,11 +12,12 @@ import sys
 import time
 from io import BytesIO
 from . import xattr
-from .platform import acl_get, acl_set
-from .chunker import Chunker
-from .hashindex import ChunkIndex
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
-    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
+    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython
+if have_cython():
+    from .platform import acl_get, acl_set
+    from .chunker import Chunker
+    from .hashindex import ChunkIndex
 
 ITEMS_BUFFER = 1024 * 1024
 

borg/archiver.py

@@ -15,17 +15,18 @@ import textwrap
 import traceback
 
 from . import __version__
-from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
-from .compress import Compressor, COMPR_BUFFER
-from .upgrader import AtticRepositoryUpgrader
-from .repository import Repository
-from .cache import Cache
-from .key import key_creator
 from .helpers import Error, location_validator, format_time, format_file_size, \
     format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
     get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec
+    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython
+if have_cython():
+    from .compress import Compressor, COMPR_BUFFER
+    from .upgrader import AtticRepositoryUpgrader
+    from .repository import Repository
+    from .cache import Cache
+    from .key import key_creator
+from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
 from .remote import RepositoryServer, RemoteRepository
 
 has_lchflags = hasattr(os, 'lchflags')
@@ -548,24 +549,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                     print(warning)
         return args
 
-    def run(self, args=None):
-        check_extension_modules()
-        keys_dir = get_keys_dir()
-        if not os.path.exists(keys_dir):
-            os.makedirs(keys_dir)
-            os.chmod(keys_dir, stat.S_IRWXU)
-        cache_dir = get_cache_dir()
-        if not os.path.exists(cache_dir):
-            os.makedirs(cache_dir)
-            os.chmod(cache_dir, stat.S_IRWXU)
-            with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd:
-                fd.write(textwrap.dedent("""
-                    Signature: 8a477f597d28d172789f06886806bc55
-                    # This file is a cache directory tag created by Borg.
-                    # For information about cache directory tags, see:
-                    #       http://www.brynosaurus.com/cachedir/
-                    """).lstrip())
-        common_parser = argparse.ArgumentParser(add_help=False)
+    def build_parser(self, args=None, prog=None):
+        common_parser = argparse.ArgumentParser(add_help=False, prog=prog)
         common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                                    default=False,
                                    help='verbose output')
@@ -576,11 +561,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH',
                                    help='set remote path to executable (default: "%(default)s")')
 
-        # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
-        if args:
-            args = self.preprocess_args(args)
-
-        parser = argparse.ArgumentParser(description='Borg %s - Deduplicated Backups' % __version__)
+        parser = argparse.ArgumentParser(prog=prog, description='Borg %s - Deduplicated Backups' % __version__)
         subparsers = parser.add_subparsers(title='Available commands')
 
         serve_epilog = textwrap.dedent("""
@@ -976,6 +957,30 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices))
         subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?',
                                help='additional help on TOPIC')
+        return parser
+
+    def run(self, args=None):
+        check_extension_modules()
+        keys_dir = get_keys_dir()
+        if not os.path.exists(keys_dir):
+            os.makedirs(keys_dir)
+            os.chmod(keys_dir, stat.S_IRWXU)
+        cache_dir = get_cache_dir()
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+            os.chmod(cache_dir, stat.S_IRWXU)
+            with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd:
+                fd.write(textwrap.dedent("""
+                    Signature: 8a477f597d28d172789f06886806bc55
+                    # This file is a cache directory tag created by Borg.
+                    # For information about cache directory tags, see:
+                    #       http://www.brynosaurus.com/cachedir/
+                    """).lstrip())
+
+        # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
+        if args:
+            args = self.preprocess_args(args)
+        parser = self.build_parser(args)
 
         args = parser.parse_args(args or ['-h'])
         self.verbose = args.verbose

borg/helpers.py

@@ -18,9 +18,25 @@ from operator import attrgetter
 
 import msgpack
 
-from . import hashindex
-from . import chunker
-from . import crypto
+def have_cython():
+    """allow for a way to disable Cython includes
+
+    this is used during usage docs build, in setup.py. It is to avoid
+    loading the Cython libraries which are built, but sometimes not in
+    the search path (namely, during Tox runs).
+
+    we simply check an environment variable (``BORG_CYTHON_DISABLE``)
+    which, when set (to anything) will disable includes of Cython
+    libraries in key places to enable usage docs to be built.
+
+    :returns: True if Cython is available, False otherwise.
+    """
+    return not os.environ.get('BORG_CYTHON_DISABLE')
+
+if have_cython():
+    from . import hashindex
+    from . import chunker
+    from . import crypto
 
 
 class Error(Exception):

borg/key.py

@@ -7,9 +7,10 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
-from .compress import Compressor, COMPR_BUFFER
-from .helpers import IntegrityError, get_keys_dir, Error
+from .helpers import IntegrityError, get_keys_dir, Error, have_cython
+if have_cython():
+    from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
+    from .compress import Compressor, COMPR_BUFFER
 
 PREFIX = b'\0' * 8
 

borg/repository.py

@@ -8,8 +8,9 @@ import struct
 import sys
 from zlib import crc32
 
-from .hashindex import NSIndex
-from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify
+from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython
+if have_cython():
+    from .hashindex import NSIndex
 from .locking import UpgradableLock
 from .lrucache import LRUCache
 

docs/Makefile

@@ -36,7 +36,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html: usage api.rst
+html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -139,32 +139,3 @@ gh-io: html
 
 inotify: html
 	while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done
-
-# generate list of targets
-usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//')
-
-# generate help file based on usage
-usage/%.rst.inc: ../borg/archiver.py
-	@echo generating usage for $*
-	@printf ".. _borg_$*:\n\n" > $@
-	@printf "borg $*\n" >> $@
-	@echo -n borg $* | tr 'a-z- ' '-' >> $@
-	@printf "\n::\n\n" >> $@
-	@borg help $* --usage-only | sed -e 's/^/    /' >> $@
-	@printf "\nDescription\n~~~~~~~~~~~\n" >> $@
-	@borg help $* --epilog-only >> $@
-
-api.rst: Makefile
-	@echo "auto-generating API documentation"
-	@echo "Borg Backup API documentation" > $@
-	@echo "=============================" >> $@
-	@echo "" >> $@
-	@for mod in ../borg/*.pyx ../borg/*.py; do \
-		if echo "$$mod" | grep -q "/_"; then \
-			continue ; \
-		fi ; \
-		printf ".. automodule:: "; \
-		echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \
-		echo "    :members:"; \
-		echo "    :undoc-members:"; \
-	done >> $@

docs/_themes/local/sidebarusefullinks.html

@@ -3,7 +3,7 @@
 
 <h3>Useful Links</h3>
 <ul>
-  <li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
+  <li><a href="https://borgbackup.readthedocs.org/">Main Web Site</a></li>
   <li><a href="https://github.com/borgbackup/borg/releases">Releases</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>

docs/usage.rst

@@ -60,6 +60,12 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         For "Warning: The repository at location ... was previously located at ..."
     BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
         For "Warning: 'check --repair' is an experimental feature that might result in data loss."
+    BORG_CYTHON_DISABLE
+        Disables the loading of Cython modules. This is currently
+        experimental and is used only to generate usage docs at build
+        time. It is unlikely to produce good results on a regular
+        run. The variable should be set to the name of the  calling class, and
+        should be unique across all of borg. It is currently only used by ``build_usage``.
 
 Directories:
     BORG_KEYS_DIR

setup.py

@@ -1,8 +1,15 @@
 # -*- encoding: utf-8 *-*
 import os
+import re
 import sys
 from glob import glob
 
+from distutils.command.build import build
+from distutils.core import Command
+from distutils.errors import DistutilsOptionError
+from distutils import log
+from setuptools.command.build_py import build_py
+
 min_python = (3, 2)
 my_python = sys.version_info
 
@@ -10,6 +17,9 @@ if my_python < min_python:
     print("Borg requires Python %d.%d or later" % min_python)
     sys.exit(1)
 
+# Are we building on ReadTheDocs?
+on_rtd = os.environ.get('READTHEDOCS')
+
 # msgpack pure python data corruption was fixed in 0.4.6.
 # Also, we might use some rather recent API features.
 install_requires=['msgpack-python>=0.4.6', ]
@@ -62,7 +72,7 @@ except ImportError:
     platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
     from distutils.command.build_ext import build_ext
-    if not all(os.path.exists(path) for path in [
+    if not on_rtd and not all(os.path.exists(path) for path in [
         compress_source, crypto_source, chunker_source, hashindex_source,
         platform_linux_source, platform_freebsd_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
@@ -103,29 +113,140 @@ possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local
 if os.environ.get('BORG_LZ4_PREFIX'):
     possible_openssl_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
 lz4_prefix = detect_lz4(possible_lz4_prefixes)
-if not lz4_prefix:
+if lz4_prefix:
+    include_dirs.append(os.path.join(lz4_prefix, 'include'))
+    library_dirs.append(os.path.join(lz4_prefix, 'lib'))
+elif not on_rtd:
     raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
-include_dirs.append(os.path.join(lz4_prefix, 'include'))
-library_dirs.append(os.path.join(lz4_prefix, 'lib'))
 
 
 with open('README.rst', 'r') as fd:
     long_description = fd.read()
 
-cmdclass = {'build_ext': build_ext, 'sdist': Sdist}
+class build_usage(Command):
+    description = "generate usage for each command"
 
-ext_modules = [
+    user_options = [
+        ('output=', 'O', 'output directory'),
+    ]
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print('generating usage docs')
+        # allows us to build docs without the C modules fully loaded during help generation
+        if 'BORG_CYTHON_DISABLE' not in os.environ:
+            os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
+        from borg.archiver import Archiver
+        parser = Archiver().build_parser(prog='borg')
+        # return to regular Cython configuration, if we changed it
+        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
+            del os.environ['BORG_CYTHON_DISABLE']
+        choices = {}
+        for action in parser._actions:
+            if action.choices is not None:
+                choices.update(action.choices)
+        print('found commands: %s' % list(choices.keys()))
+        if not os.path.exists('docs/usage'):
+            os.mkdir('docs/usage')
+        for command, parser in choices.items():
+            if command is 'help':
+                continue
+            with open('docs/usage/%s.rst.inc' % command, 'w') as doc:
+                print('generating help for %s' % command)
+                params = {"command": command,
+                          "underline": '-' * len('borg ' + command)}
+                doc.write(".. _borg_{command}:\n\n".format(**params))
+                doc.write("borg {command}\n{underline}\n::\n\n".format(**params))
+                epilog = parser.epilog
+                parser.epilog = None
+                doc.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
+                doc.write("\nDescription\n~~~~~~~~~~~\n")
+                doc.write(epilog)
+
+
+class build_api(Command):
+    description = "generate a basic api.rst file based on the modules available"
+
+    user_options = [
+        ('output=', 'O', 'output directory'),
+    ]
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print("auto-generating API documentation")
+        with open("docs/api.rst", "w") as doc:
+            doc.write("""
+Borg Backup API documentation"
+=============================
+""")
+            for mod in glob('borg/*.py') + glob('borg/*.pyx'):
+                print("examining module %s" % mod)
+                if "/_" not in mod:
+                    doc.write("""
+.. automodule:: %s
+    :members:
+    :undoc-members:
+""" % mod)
+
+# (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands
+# seems like this doesn't work on RTD, see below for build_py hack.
+build.sub_commands.append(('build_api', None))
+build.sub_commands.append(('build_usage', None))
+
+
+class build_py_custom(build_py):
+    """override build_py to also build our stuff
+
+    it is unclear why this is necessary, but in some environments
+    (Readthedocs.org, specifically), the above
+    ``build.sub_commands.append()`` doesn't seem to have an effect:
+    our custom build commands seem to be ignored when running
+    ``setup.py install``.
+
+    This class overrides the ``build_py`` target by forcing it to run
+    our custom steps as well.
+
+    See also the `bug report on RTD
+    <https://github.com/rtfd/readthedocs.org/issues/1740>`_.
+    """
+    def run(self):
+        super().run()
+        self.announce('calling custom build steps', level=log.INFO)
+        self.run_command('build_ext')
+        self.run_command('build_api')
+        self.run_command('build_usage')
+
+
+cmdclass = {
+    'build_ext': build_ext,
+    'build_api': build_api,
+    'build_usage': build_usage,
+    'build_py': build_py_custom,
+    'sdist': Sdist
+}
+
+ext_modules = []
+if not on_rtd:
+    ext_modules += [
     Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.chunker', [chunker_source]),
     Extension('borg.hashindex', [hashindex_source])
 ]
-if sys.platform.startswith('linux'):
-    ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
-elif sys.platform.startswith('freebsd'):
-    ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
-elif sys.platform == 'darwin':
-    ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
+    if sys.platform.startswith('linux'):
+        ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
+    elif sys.platform.startswith('freebsd'):
+        ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
+    elif sys.platform == 'darwin':
+        ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
 
 setup(
     name='borgbackup',