generalise the cython check hack

instead of applying this only to usage generation, use it as a generic
mechanism to disable loading of Cython code.

it may be incomplete: there may be other places where Cython code is
loaded that is not checked, but that is sufficient to build the usage
docs. the environment variable used is documented as such in the
docs/usage.rst.

we also move the check to a helper function and document it
better. this has the unfortunate side effect of moving includes
around, but I can't think of a better way.

borg/archive.py

@@ -12,12 +12,12 @@ import sys
 import time
 from io import BytesIO
 from . import xattr
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
+    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython
+if not detect_cython():
     from .platform import acl_get, acl_set
     from .chunker import Chunker
     from .hashindex import ChunkIndex
-from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
-    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
 
 ITEMS_BUFFER = 1024 * 1024
 

borg/archiver.py

@@ -15,18 +15,18 @@ import textwrap
 import traceback
 
 from . import __version__
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import Error, location_validator, format_time, format_file_size, \
+    format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
+    get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
+    Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
+    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython
+if not detect_cython():
     from .compress import Compressor, COMPR_BUFFER
     from .upgrader import AtticRepositoryUpgrader
     from .repository import Repository
     from .cache import Cache
     from .key import key_creator
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
-from .helpers import Error, location_validator, format_time, format_file_size, \
-    format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
-    get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
-    Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec
 from .remote import RepositoryServer, RemoteRepository
 
 has_lchflags = hasattr(os, 'lchflags')

borg/helpers.py

@@ -18,7 +18,20 @@ from operator import attrgetter
 
 import msgpack
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+def detect_cython():
+    """allow for a way to disable Cython includes
+
+    this is used during usage docs build, in setup.py. It is to avoid
+    loading the Cython libraries which are built, but sometimes not in
+    the search path (namely, during Tox runs).
+
+    we simply check an environment variable (``BORG_CYTHON_DISABLE``)
+    which, when set (to anything) will disable includes of Cython
+    libraries in key places to enable usage docs to be built.
+    """
+    return os.environ.get('BORG_CYTHON_DISABLE')
+
+if not detect_cython():
     from . import hashindex
     from . import chunker
     from . import crypto

borg/key.py

@@ -7,10 +7,10 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import IntegrityError, get_keys_dir, Error, detect_cython
+if not detect_cython():
     from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
     from .compress import Compressor, COMPR_BUFFER
-from .helpers import IntegrityError, get_keys_dir, Error
 
 PREFIX = b'\0' * 8
 

borg/repository.py

@@ -8,9 +8,9 @@ import struct
 import sys
 from zlib import crc32
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython
+if not detect_cython():
     from .hashindex import NSIndex
-from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify
 from .locking import UpgradableLock
 from .lrucache import LRUCache
 

docs/usage.rst

@@ -60,6 +60,12 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         For "Warning: The repository at location ... was previously located at ..."
     BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
         For "Warning: 'check --repair' is an experimental feature that might result in data loss."
+    BORG_CYTHON_DISABLE
+        Disables the loading of Cython modules. This is currently
+        experimentaly and is used only to generate usage docs at build
+        time, it's unlikely to produce good results on a regular
+        run. The variable should be set to the calling class, and
+        should be unique. It is currently only used by ``build_usage``.
 
 Directories:
     BORG_KEYS_DIR

setup.py

@@ -135,10 +135,14 @@ class build_usage(Command):
 
     def run(self):
         print('generating usage docs')
-        # XXX: gross hack: allows us to skip loading C modules during help generation
-        os.environ['BORG_GEN_USAGE'] = "True"
+        # allows us to build docs without the C modules fully loaded during help generation
+        if 'BORG_CYTHON_DISABLE' not in os.environ:
+            os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
+        # return to regular Cython configuration, if we changed it
+        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
+            del os.environ['BORG_CYTHON_DISABLE']
         choices = {}
         for action in parser._actions:
             if action.choices is not None: