diff --git a/setup.py b/setup.py index dc3cc396f..63aa64170 100644 --- a/setup.py +++ b/setup.py @@ -46,11 +46,24 @@ compress_source = 'src/borg/compress.pyx' crypto_source = 'src/borg/crypto.pyx' chunker_source = 'src/borg/chunker.pyx' hashindex_source = 'src/borg/hashindex.pyx' -platform_posix_source = 'src/borg/platform_posix.pyx' -platform_linux_source = 'src/borg/platform_linux.pyx' -platform_darwin_source = 'src/borg/platform_darwin.pyx' -platform_freebsd_source = 'src/borg/platform_freebsd.pyx' -platform_windows_source = 'src/borg/platform_windows.pyx' +platform_posix_source = 'src/borg/platform/posix.pyx' +platform_linux_source = 'src/borg/platform/linux.pyx' +platform_darwin_source = 'src/borg/platform/darwin.pyx' +platform_freebsd_source = 'src/borg/platform/freebsd.pyx' +platform_windows_source = 'src/borg/platform/windows.pyx' + +cython_sources = [ + compress_source, + crypto_source, + chunker_source, + hashindex_source, + + platform_posix_source, + platform_linux_source, + platform_freebsd_source, + platform_darwin_source, + platform_windows_source, +] try: from Cython.Distutils import build_ext @@ -58,7 +71,7 @@ try: class Sdist(sdist): def __init__(self, *args, **kwargs): - for src in glob('src/borg/*.pyx'): + for src in cython_sources: cython_compiler.compile(src, cython_compiler.default_options) super().__init__(*args, **kwargs) @@ -68,11 +81,12 @@ try: 'src/borg/crypto.c', 'src/borg/chunker.c', 'src/borg/_chunker.c', 'src/borg/hashindex.c', 'src/borg/_hashindex.c', - 'src/borg/platform_posix.c', - 'src/borg/platform_linux.c', - 'src/borg/platform_freebsd.c', - 'src/borg/platform_darwin.c', - 'src/borg/platform_windows.c', + + 'src/borg/platform/posix.c', + 'src/borg/platform/linux.c', + 'src/borg/platform/freebsd.c', + 'src/borg/platform/darwin.c', + 'src/borg/platform/windows.c', ]) super().make_distribution() @@ -318,16 +332,16 @@ if not on_rtd: Extension('borg.hashindex', [hashindex_source]) ] if sys.platform.startswith(('linux', 'freebsd', 'darwin')): - ext_modules.append(Extension('borg.platform_posix', [platform_posix_source])) + ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) if sys.platform == 'linux': - ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl'])) + ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])) elif sys.platform.startswith('freebsd'): - ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source])) + ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source])) elif sys.platform == 'darwin': - ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source])) + ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source])) elif sys.platform == 'win32': - ext_modules.append(Extension('borg.platform_windows', [platform_windows_source])) + ext_modules.append(Extension('borg.platform.windows', [platform_windows_source])) def parse(root, describe_command=None): diff --git a/src/borg/archive.py b/src/borg/archive.py index 86574b369..d461f140b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1,38 +1,45 @@ -from datetime import datetime, timezone -from getpass import getuser -from itertools import groupby import errno - -from .logger import create_logger -logger = create_logger() -from .key import key_factory -from .remote import cache_if_remote - import os -from shutil import get_terminal_size import socket import stat import sys import time +from datetime import datetime, timezone +from getpass import getuser from io import BytesIO +from itertools import groupby +from shutil import get_terminal_size + +import msgpack + +from .logger import create_logger +logger = create_logger() + from . import xattr -from .compress import COMPR_BUFFER +from .cache import ChunkListEntry +from .chunker import Chunker from .constants import * # NOQA -from .helpers import Chunk, Error, uid2user, user2uid, gid2group, group2gid, \ - parse_timestamp, to_localtime, format_time, format_timedelta, safe_encode, safe_decode, \ - Manifest, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, bin_to_hex, \ - ProgressIndicatorPercent, ChunkIteratorFileWrapper, remove_surrogates, log_multi, \ - PathPrefixPattern, FnmatchPattern, open_item, file_status, format_file_size, consume, \ - CompressionDecider1, CompressionDecider2, CompressionSpec, \ - IntegrityError +from .hashindex import ChunkIndex, ChunkIndexEntry +from .helpers import Manifest +from .helpers import Chunk, ChunkIteratorFileWrapper, open_item +from .helpers import Error, IntegrityError +from .helpers import uid2user, user2uid, gid2group, group2gid +from .helpers import parse_timestamp, to_localtime +from .helpers import format_time, format_timedelta, format_file_size, file_status +from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates +from .helpers import decode_dict, StableDict +from .helpers import int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import ProgressIndicatorPercent, log_multi +from .helpers import PathPrefixPattern, FnmatchPattern +from .helpers import consume +from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec +from .key import key_factory +from .platform import acl_get, acl_set, set_flags, get_flags, swidth +from .remote import cache_if_remote from .repository import Repository + if sys.platform == 'win32': from .platform import get_owner, set_owner -from .platform import acl_get, acl_set, set_flags, get_flags, swidth -from .chunker import Chunker -from .hashindex import ChunkIndex, ChunkIndexEntry -from .cache import ChunkListEntry -import msgpack has_lchmod = hasattr(os, 'lchmod') diff --git a/src/borg/archiver.py b/src/borg/archiver.py index dfebc65a8..0a967af26 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1,7 +1,3 @@ -from binascii import unhexlify -from datetime import datetime -from itertools import zip_longest -from operator import attrgetter import argparse import collections import functools @@ -18,28 +14,36 @@ import subprocess import sys import textwrap import traceback +from binascii import unhexlify +from datetime import datetime +from itertools import zip_longest -from . import __version__ -from .helpers import Error, location_validator, archivename_validator, format_time, format_file_size, \ - parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ - get_cache_dir, prune_within, prune_split, bin_to_hex, safe_encode, \ - Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, \ - dir_is_tagged, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ - log_multi, PatternMatcher, ItemFormatter from .logger import create_logger, setup_logging logger = create_logger() + +from . import __version__ from . import helpers -from .compress import Compressor, COMPR_BUFFER -from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader -from .repository import Repository +from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics from .cache import Cache from .constants import * # NOQA +from .helpers import Error +from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec +from .helpers import ItemFormatter, format_time, format_file_size, format_archive +from .helpers import safe_encode, remove_surrogates, bin_to_hex +from .helpers import prune_within, prune_split +from .helpers import to_localtime, timestamp +from .helpers import get_cache_dir +from .helpers import Manifest +from .helpers import update_excludes, check_extension_modules +from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo +from .helpers import log_multi +from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern from .key import key_creator, RepoKey, PassphraseKey -from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics -from .remote import RepositoryServer, RemoteRepository, cache_if_remote -from .selftest import selftest -from .hashindex import ChunkIndexEntry from .platform import get_flags +from .remote import RepositoryServer, RemoteRepository, cache_if_remote +from .repository import Repository +from .selftest import selftest +from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader if sys.platform == 'win32': import posixpath diff --git a/src/borg/cache.py b/src/borg/cache.py index fff2dc33f..9b7ad07d7 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -1,21 +1,25 @@ import configparser -from .remote import cache_if_remote -from collections import namedtuple import os import stat -from binascii import unhexlify import shutil - -from .key import PlaintextKey -from .logger import create_logger -logger = create_logger() -from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \ - bigint_to_int, bin_to_hex, format_file_size, yes -from .locking import UpgradableLock -from .hashindex import ChunkIndex, ChunkIndexEntry +from binascii import unhexlify +from collections import namedtuple import msgpack +from .logger import create_logger +logger = create_logger() + +from .hashindex import ChunkIndex, ChunkIndexEntry +from .helpers import Error +from .helpers import get_cache_dir +from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import format_file_size +from .helpers import yes +from .key import PlaintextKey +from .locking import UpgradableLock +from .remote import cache_if_remote + ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') diff --git a/src/borg/fuse.py b/src/borg/fuse.py index d92c24ed4..c43ba9399 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -1,20 +1,23 @@ -from collections import defaultdict import errno import io -import llfuse import os import stat import tempfile import time -from .archive import Archive -from .helpers import daemonize, bigint_to_int -from .logger import create_logger -from .lrucache import LRUCache +from collections import defaultdict from distutils.version import LooseVersion + +import llfuse import msgpack +from .logger import create_logger logger = create_logger() +from .archive import Archive +from .helpers import daemonize +from .helpers import bigint_to_int +from .lrucache import LRUCache + # Does this version of llfuse support ns precision? have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f8f52fe47..8cd6cb339 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1,46 +1,46 @@ -import argparse -from binascii import hexlify -from collections import namedtuple, deque -from functools import wraps, partial -import sys + if sys.platform != 'win32': import grp import pwd else: import posixpath import encodings.idna + +import argparse import hashlib -from itertools import islice +import logging import os import os.path +import platform +import re +import socket +import sys import stat import textwrap - -import re -from string import Formatter -import platform import time import unicodedata -import logging +from binascii import hexlify +from collections import namedtuple, deque +from datetime import datetime, timezone, timedelta +from fnmatch import translate +from functools import wraps, partial +from itertools import islice +from operator import attrgetter +from string import Formatter + +import msgpack +import msgpack.fallback from .logger import create_logger logger = create_logger() -from datetime import datetime, timezone, timedelta -from fnmatch import translate -from operator import attrgetter - from . import __version__ as borg_version -from . import hashindex from . import chunker -from .constants import * # NOQA from . import crypto -from .compress import COMPR_BUFFER, get_compressor +from . import hashindex from . import shellpattern -import msgpack -import msgpack.fallback - -import socket +from .constants import * # NOQA +from .compress import COMPR_BUFFER, get_compressor # meta dict, data bytes _Chunk = namedtuple('_Chunk', 'meta data') diff --git a/src/borg/item.py b/src/borg/item.py index 05ac47a2c..1bccade36 100644 --- a/src/borg/item.py +++ b/src/borg/item.py @@ -1,5 +1,7 @@ from .constants import ITEM_KEYS -from .helpers import safe_encode, safe_decode, bigint_to_int, int_to_bigint, StableDict +from .helpers import safe_encode, safe_decode +from .helpers import bigint_to_int, int_to_bigint +from .helpers import StableDict class PropDict: diff --git a/src/borg/key.py b/src/borg/key.py index 7b4c5c6ce..be5338b13 100644 --- a/src/borg/key.py +++ b/src/borg/key.py @@ -1,20 +1,27 @@ -from binascii import a2b_base64, b2a_base64, hexlify import configparser import getpass import os import sys import textwrap -from hmac import compare_digest +from binascii import a2b_base64, b2a_base64, hexlify from hashlib import sha256, pbkdf2_hmac +from hmac import compare_digest + +import msgpack -from .helpers import Chunk, IntegrityError, get_keys_dir, Error, yes, bin_to_hex, CompressionDecider2, CompressionSpec from .logger import create_logger logger = create_logger() from .constants import * # NOQA -from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256 from .compress import Compressor, COMPR_BUFFER, get_compressor -import msgpack +from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256 +from .helpers import Chunk +from .helpers import Error, IntegrityError +from .helpers import yes +from .helpers import get_keys_dir +from .helpers import bin_to_hex +from .helpers import CompressionDecider2, CompressionSpec + PREFIX = b'\0' * 8 diff --git a/src/borg/locking.py b/src/borg/locking.py index bc3660d61..f07c87247 100644 --- a/src/borg/locking.py +++ b/src/borg/locking.py @@ -3,7 +3,7 @@ import os import socket import time -from borg.helpers import Error, ErrorWithTraceback +from .helpers import Error, ErrorWithTraceback ADD, REMOVE = 'add', 'remove' SHARED, EXCLUSIVE = 'shared', 'exclusive' diff --git a/src/borg/platform.py b/src/borg/platform.py deleted file mode 100644 index b9b91aa70..000000000 --- a/src/borg/platform.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys - -from .platform_base import acl_get, acl_set, SyncFile, sync_dir, set_flags, get_flags, swidth, API_VERSION - -if sys.platform.startswith('linux'): # pragma: linux only - from .platform_linux import acl_get, acl_set, SyncFile, set_flags, get_flags, swidth, API_VERSION -elif sys.platform.startswith('freebsd'): # pragma: freebsd only - from .platform_freebsd import acl_get, acl_set, swidth, API_VERSION -elif sys.platform == 'darwin': # pragma: darwin only - from .platform_darwin import acl_get, acl_set, swidth, API_VERSION -elif sys.platform == 'win32': # pragma: windows only - from .platform_windows import acl_get, acl_set, API_VERSION, get_owner, set_owner, sync_dir, SyncFile diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py new file mode 100644 index 000000000..1d5d467d3 --- /dev/null +++ b/src/borg/platform/__init__.py @@ -0,0 +1,29 @@ +import sys + +""" +Platform-specific APIs. + +Public APIs are documented in platform.base. +""" + +from .base import acl_get, acl_set +from .base import set_flags, get_flags +from .base import SyncFile, sync_dir, fdatasync +from .base import swidth, API_VERSION + +if sys.platform.startswith('linux'): # pragma: linux only + from .linux import acl_get, acl_set + from .linux import set_flags, get_flags + from .linux import SyncFile + from .linux import swidth, API_VERSION +elif sys.platform.startswith('freebsd'): # pragma: freebsd only + from .freebsd import acl_get, acl_set + from .freebsd import swidth, API_VERSION +elif sys.platform == 'darwin': # pragma: darwin only + from .darwin import acl_get, acl_set + from .darwin import swidth, API_VERSION +elif sys.platform == 'win32': # pragma: windows only + from .windows import acl_get, acl_set + from .windows import API_VERSION + from .windows import sync_dir, SyncFile + from .windows import get_owner, set_owner diff --git a/src/borg/platform_base.py b/src/borg/platform/base.py similarity index 83% rename from src/borg/platform_base.py rename to src/borg/platform/base.py index da62260b6..e8d28b52d 100644 --- a/src/borg/platform_base.py +++ b/src/borg/platform/base.py @@ -1,5 +1,17 @@ import os +""" +platform base module +==================== + +Contains platform API implementations based on what Python itself provides. More specific +APIs are stubs in this module. + +When functions in this module use platform APIs themselves they access the public +platform API: that way platform APIs provided by the platform-specific support module +are correctly composed into the base functionality. +""" + API_VERSION = 3 fdatasync = getattr(os, 'fdatasync', os.fsync) @@ -80,16 +92,18 @@ class SyncFile: Synchronize file contents. Everything written prior to sync() must become durable before anything written after sync(). """ + from .. import platform self.fd.flush() - fdatasync(self.fileno) + platform.fdatasync(self.fileno) if hasattr(os, 'posix_fadvise'): os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED) def close(self): """sync() and close.""" + from .. import platform self.sync() self.fd.close() - sync_dir(os.path.dirname(self.fd.name)) + platform.sync_dir(os.path.dirname(self.fd.name)) def swidth(s): diff --git a/src/borg/platform_darwin.pyx b/src/borg/platform/darwin.pyx similarity index 95% rename from src/borg/platform_darwin.pyx rename to src/borg/platform/darwin.pyx index fb94d9c84..016ad5aab 100644 --- a/src/borg/platform_darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -1,6 +1,8 @@ import os -from .helpers import user2uid, group2gid, safe_decode, safe_encode -from .platform_posix import swidth + +from ..helpers import user2uid, group2gid +from ..helpers import safe_decode, safe_encode +from .posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_freebsd.pyx b/src/borg/platform/freebsd.pyx similarity index 95% rename from src/borg/platform_freebsd.pyx rename to src/borg/platform/freebsd.pyx index eae730f49..7553a26cc 100644 --- a/src/borg/platform_freebsd.pyx +++ b/src/borg/platform/freebsd.pyx @@ -1,6 +1,8 @@ import os -from .helpers import posix_acl_use_stored_uid_gid, safe_encode, safe_decode -from .platform_posix import swidth + +from ..helpers import posix_acl_use_stored_uid_gid +from ..helpers import safe_encode, safe_decode +from .posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_linux.pyx b/src/borg/platform/linux.pyx similarity index 97% rename from src/borg/platform_linux.pyx rename to src/borg/platform/linux.pyx index 2d958e281..76d76521f 100644 --- a/src/borg/platform_linux.pyx +++ b/src/borg/platform/linux.pyx @@ -3,9 +3,11 @@ import re import resource import stat -from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode -from .platform_base import SyncFile as BaseSyncFile -from .platform_posix import swidth +from ..helpers import posix_acl_use_stored_uid_gid +from ..helpers import user2uid, group2gid +from ..helpers import safe_decode, safe_encode +from .base import SyncFile as BaseSyncFile +from .posix import swidth from libc cimport errno from libc.stdint cimport int64_t diff --git a/src/borg/platform_posix.pyx b/src/borg/platform/posix.pyx similarity index 100% rename from src/borg/platform_posix.pyx rename to src/borg/platform/posix.pyx diff --git a/src/borg/platform_windows.pyx b/src/borg/platform/windows.pyx similarity index 100% rename from src/borg/platform_windows.pyx rename to src/borg/platform/windows.pyx diff --git a/src/borg/remote.py b/src/borg/remote.py index 4c743b9de..343bfadfc 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -6,16 +6,19 @@ import logging import os import select import shlex -from subprocess import Popen, PIPE +import sys import tempfile - -from . import __version__ - -from .helpers import Error, IntegrityError, get_home_dir, sysinfo, bin_to_hex -from .repository import Repository +from subprocess import Popen, PIPE import msgpack +from . import __version__ +from .helpers import Error, IntegrityError +from .helpers import get_home_dir +from .helpers import sysinfo +from .helpers import bin_to_hex +from .repository import Repository + RPC_PROTOCOL_VERSION = 2 BUFSIZE = 10 * 1024 * 1024 diff --git a/src/borg/repository.py b/src/borg/repository.py index eab6e1343..704131839 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,23 +1,26 @@ -from configparser import ConfigParser -from binascii import unhexlify -from datetime import datetime -from itertools import islice import errno -import logging -logger = logging.getLogger(__name__) - import os import shutil import struct +from binascii import unhexlify from collections import defaultdict +from configparser import ConfigParser +from datetime import datetime from functools import partial +from itertools import islice from zlib import crc32 import msgpack + +import logging +logger = logging.getLogger(__name__) + from .constants import * # NOQA -from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError, Location, ProgressIndicatorPercent, \ - bin_to_hex from .hashindex import NSIndex +from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError +from .helpers import Location +from .helpers import ProgressIndicatorPercent +from .helpers import bin_to_hex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache from .platform import SyncFile, sync_dir @@ -52,7 +55,7 @@ class Repository: established by a COMMIT. Note that the COMMIT can't establish consistency by itself, but only manages to do so with proper support from - the platform (including the hardware). See platform_base.SyncFile for details. + the platform (including the hardware). See platform.base.SyncFile for details. A PUT inserts a key-value pair. The value is stored in the log entry, hence the repository implements full data logging, meaning that all data is consistent, not just metadata (which is common in file systems). diff --git a/src/borg/shellpattern.py b/src/borg/shellpattern.py index 7cb8f2114..e669b0ff4 100644 --- a/src/borg/shellpattern.py +++ b/src/borg/shellpattern.py @@ -1,5 +1,5 @@ -import re import os +import re def translate(pat): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 02afc6279..a6ce4ce7a 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -17,6 +17,10 @@ from unittest.mock import patch from hashlib import sha256 import pytest +try: + import llfuse +except ImportError: + pass from .. import xattr, helpers, platform from ..archive import Archive, ChunkBuffer, ArchiveRecreater @@ -24,18 +28,15 @@ from ..archiver import Archiver from ..cache import Cache from ..constants import * # NOQA from ..crypto import bytes_to_long, num_aes_blocks -from ..helpers import Chunk, Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex +from ..helpers import Chunk, Manifest +from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR +from ..helpers import bin_to_hex from ..key import KeyfileKeyBase from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import has_lchflags, has_llfuse from . import BaseTestCase, changedir, environment_variable -try: - import llfuse -except ImportError: - pass - src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index ae1f886fd..ede20e16f 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -1,23 +1,27 @@ import hashlib -import io import logging -from time import mktime, strptime -from datetime import datetime, timezone, timedelta import os +import sys +from datetime import datetime, timezone, timedelta +from time import mktime, strptime, sleep import pytest -import sys import msgpack import msgpack.fallback -import time -from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, clean_lines, \ - prune_within, prune_split, get_cache_dir, get_keys_dir, is_slow_msgpack, \ - yes, TRUISH, FALSISH, DEFAULTISH, \ - StableDict, int_to_bigint, bigint_to_int, bin_to_hex, parse_timestamp, ChunkerParams, Chunk, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \ - CompressionSpec, CompressionDecider1, CompressionDecider2, \ - PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, partial_format, ChunkIteratorFileWrapper +from ..helpers import Location +from ..helpers import partial_format, format_file_size, format_timedelta +from ..helpers import make_path_safe, clean_lines +from ..helpers import prune_within, prune_split +from ..helpers import get_cache_dir, get_keys_dir +from ..helpers import is_slow_msgpack +from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH +from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex +from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk +from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless +from ..helpers import load_excludes +from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2 +from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern from . import BaseTestCase, environment_variable, FakeInputs @@ -115,7 +119,7 @@ class TestLocationWithoutEnv: assert repr(Location('/some/path::archive{pid}')) == \ "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive{}')".format(test_pid) location_time1 = Location('/some/path::archive{now:%s}') - time.sleep(1.1) + sleep(1.1) location_time2 = Location('/some/path::archive{now:%s}') assert location_time1.archive != location_time2.archive diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 8a1642a1d..c62f5ffef 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -1,15 +1,15 @@ import getpass -import os import re -import shutil import tempfile from binascii import hexlify, unhexlify import pytest from ..crypto import bytes_to_long, num_aes_blocks +from ..helpers import Location +from ..helpers import Chunk +from ..helpers import IntegrityError from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex -from ..helpers import Location, Chunk, IntegrityError @pytest.fixture(autouse=True) diff --git a/src/borg/testsuite/lrucache.py b/src/borg/testsuite/lrucache.py index 2ed2ef9b3..9fb4f92b6 100644 --- a/src/borg/testsuite/lrucache.py +++ b/src/borg/testsuite/lrucache.py @@ -1,7 +1,9 @@ -from ..lrucache import LRUCache -import pytest from tempfile import TemporaryFile +import pytest + +from ..lrucache import LRUCache + class TestLRUCache: diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 9eaede988..857920079 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -102,7 +102,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_in(group_entry_numeric, acl_access_numeric) def test_utils(self): - from ..platform_linux import acl_use_local_uid_gid + from ..platform.linux import acl_use_local_uid_gid self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-') self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-') self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-') diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index cf8c6bb32..88eb5389a 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -7,7 +7,8 @@ import tempfile from unittest.mock import patch from ..hashindex import NSIndex -from ..helpers import Location, IntegrityError, InternalOSError +from ..helpers import Location +from ..helpers import IntegrityError, InternalOSError from ..locking import UpgradableLock, LockFailed from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line from ..repository import Repository, LoggedIO, MAGIC diff --git a/src/borg/upgrader.py b/src/borg/upgrader.py index d0ea9680d..af692bb60 100644 --- a/src/borg/upgrader.py +++ b/src/borg/upgrader.py @@ -1,14 +1,16 @@ import datetime -import logging -logger = logging.getLogger(__name__) import os import shutil import time -from .helpers import get_home_dir, get_keys_dir, get_cache_dir, ProgressIndicatorPercent +import logging +logger = logging.getLogger(__name__) + +from .helpers import get_home_dir, get_keys_dir, get_cache_dir +from .helpers import ProgressIndicatorPercent +from .key import KeyfileKey, KeyfileNotFoundError from .locking import UpgradableLock from .repository import Repository, MAGIC -from .key import KeyfileKey, KeyfileNotFoundError ATTIC_MAGIC = b'ATTICSEG'