From 49484d6d6758ba2698c15fb29f05290bff0779b6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 31 May 2016 00:33:13 +0200 Subject: [PATCH 1/5] split helpers import into multiple ones --- src/borg/archive.py | 23 ++++++++++++++--------- src/borg/archiver.py | 18 ++++++++++++------ src/borg/cache.py | 7 +++++-- src/borg/fuse.py | 3 ++- src/borg/item.py | 4 +++- src/borg/key.py | 7 ++++++- src/borg/locking.py | 2 +- src/borg/platform_darwin.pyx | 5 ++++- src/borg/platform_freebsd.pyx | 5 ++++- src/borg/platform_linux.pyx | 4 +++- src/borg/remote.py | 5 ++++- src/borg/repository.py | 6 ++++-- src/borg/testsuite/archiver.py | 4 +++- src/borg/testsuite/helpers.py | 20 +++++++++++++------- src/borg/testsuite/key.py | 4 +++- src/borg/testsuite/repository.py | 3 ++- src/borg/upgrader.py | 3 ++- 17 files changed, 85 insertions(+), 38 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 905cbff22..89a1469ac 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1,7 +1,7 @@ +import errno from datetime import datetime, timezone from getpass import getuser from itertools import groupby -import errno from .logger import create_logger logger = create_logger() @@ -16,15 +16,20 @@ import sys import time from io import BytesIO from . import xattr -from .compress import COMPR_BUFFER from .constants import * # NOQA -from .helpers import Chunk, Error, uid2user, user2uid, gid2group, group2gid, \ - parse_timestamp, to_localtime, format_time, format_timedelta, safe_encode, safe_decode, \ - Manifest, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, bin_to_hex, \ - ProgressIndicatorPercent, ChunkIteratorFileWrapper, remove_surrogates, log_multi, \ - PathPrefixPattern, FnmatchPattern, open_item, file_status, format_file_size, consume, \ - CompressionDecider1, CompressionDecider2, CompressionSpec, \ - IntegrityError +from .helpers import Manifest +from .helpers import Chunk, ChunkIteratorFileWrapper, open_item +from .helpers import Error, IntegrityError +from .helpers import uid2user, user2uid, gid2group, group2gid +from .helpers import parse_timestamp, to_localtime +from .helpers import format_time, format_timedelta, format_file_size, file_status +from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates +from .helpers import decode_dict, StableDict +from .helpers import int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import ProgressIndicatorPercent, log_multi +from .helpers import PathPrefixPattern, FnmatchPattern +from .helpers import consume +from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec from .repository import Repository from .platform import acl_get, acl_set, set_flags, get_flags, swidth from .chunker import Chunker diff --git a/src/borg/archiver.py b/src/borg/archiver.py index d9eebeebb..9b5a38521 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -20,12 +20,18 @@ import textwrap import traceback from . import __version__ -from .helpers import Error, location_validator, archivename_validator, format_time, format_file_size, \ - parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ - get_cache_dir, prune_within, prune_split, bin_to_hex, safe_encode, \ - Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, \ - dir_is_tagged, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ - log_multi, PatternMatcher, ItemFormatter +from .helpers import Error +from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec +from .helpers import ItemFormatter, format_time, format_file_size, format_archive +from .helpers import safe_encode, remove_surrogates, bin_to_hex +from .helpers import prune_within, prune_split +from .helpers import to_localtime, timestamp +from .helpers import get_cache_dir +from .helpers import Manifest +from .helpers import update_excludes, check_extension_modules +from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo +from .helpers import log_multi +from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern from .logger import create_logger, setup_logging logger = create_logger() from . import helpers diff --git a/src/borg/cache.py b/src/borg/cache.py index fff2dc33f..f9af7d06d 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -9,8 +9,11 @@ import shutil from .key import PlaintextKey from .logger import create_logger logger = create_logger() -from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \ - bigint_to_int, bin_to_hex, format_file_size, yes +from .helpers import Error +from .helpers import get_cache_dir +from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import format_file_size +from .helpers import yes from .locking import UpgradableLock from .hashindex import ChunkIndex, ChunkIndexEntry diff --git a/src/borg/fuse.py b/src/borg/fuse.py index d92c24ed4..36ca3c563 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -7,7 +7,8 @@ import stat import tempfile import time from .archive import Archive -from .helpers import daemonize, bigint_to_int +from .helpers import daemonize +from .helpers import bigint_to_int from .logger import create_logger from .lrucache import LRUCache from distutils.version import LooseVersion diff --git a/src/borg/item.py b/src/borg/item.py index 05ac47a2c..1bccade36 100644 --- a/src/borg/item.py +++ b/src/borg/item.py @@ -1,5 +1,7 @@ from .constants import ITEM_KEYS -from .helpers import safe_encode, safe_decode, bigint_to_int, int_to_bigint, StableDict +from .helpers import safe_encode, safe_decode +from .helpers import bigint_to_int, int_to_bigint +from .helpers import StableDict class PropDict: diff --git a/src/borg/key.py b/src/borg/key.py index 7b4c5c6ce..6f98401e3 100644 --- a/src/borg/key.py +++ b/src/borg/key.py @@ -7,7 +7,12 @@ import textwrap from hmac import compare_digest from hashlib import sha256, pbkdf2_hmac -from .helpers import Chunk, IntegrityError, get_keys_dir, Error, yes, bin_to_hex, CompressionDecider2, CompressionSpec +from .helpers import Chunk +from .helpers import Error, IntegrityError +from .helpers import yes +from .helpers import get_keys_dir +from .helpers import bin_to_hex +from .helpers import CompressionDecider2, CompressionSpec from .logger import create_logger logger = create_logger() diff --git a/src/borg/locking.py b/src/borg/locking.py index bc3660d61..f07c87247 100644 --- a/src/borg/locking.py +++ b/src/borg/locking.py @@ -3,7 +3,7 @@ import os import socket import time -from borg.helpers import Error, ErrorWithTraceback +from .helpers import Error, ErrorWithTraceback ADD, REMOVE = 'add', 'remove' SHARED, EXCLUSIVE = 'shared', 'exclusive' diff --git a/src/borg/platform_darwin.pyx b/src/borg/platform_darwin.pyx index fb94d9c84..6b552c3aa 100644 --- a/src/borg/platform_darwin.pyx +++ b/src/borg/platform_darwin.pyx @@ -1,5 +1,8 @@ import os -from .helpers import user2uid, group2gid, safe_decode, safe_encode + +from .helpers import user2uid, group2gid +from .helpers import safe_decode, safe_encode + from .platform_posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_freebsd.pyx b/src/borg/platform_freebsd.pyx index eae730f49..4b9dd49eb 100644 --- a/src/borg/platform_freebsd.pyx +++ b/src/borg/platform_freebsd.pyx @@ -1,5 +1,8 @@ import os -from .helpers import posix_acl_use_stored_uid_gid, safe_encode, safe_decode + +from .helpers import posix_acl_use_stored_uid_gid +from .helpers import safe_encode, safe_decode + from .platform_posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_linux.pyx b/src/borg/platform_linux.pyx index 2d958e281..7a0c83d39 100644 --- a/src/borg/platform_linux.pyx +++ b/src/borg/platform_linux.pyx @@ -3,7 +3,9 @@ import re import resource import stat -from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode +from .helpers import posix_acl_use_stored_uid_gid +from .helpers import user2uid, group2gid +from .helpers import safe_decode, safe_encode from .platform_base import SyncFile as BaseSyncFile from .platform_posix import swidth diff --git a/src/borg/remote.py b/src/borg/remote.py index 47492242d..12fbb6024 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -10,7 +10,10 @@ import tempfile from . import __version__ -from .helpers import Error, IntegrityError, get_home_dir, sysinfo, bin_to_hex +from .helpers import Error, IntegrityError +from .helpers import get_home_dir +from .helpers import sysinfo +from .helpers import bin_to_hex from .repository import Repository import msgpack diff --git a/src/borg/repository.py b/src/borg/repository.py index eab6e1343..3d8c5ae01 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -15,8 +15,10 @@ from zlib import crc32 import msgpack from .constants import * # NOQA -from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError, Location, ProgressIndicatorPercent, \ - bin_to_hex +from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError +from .helpers import Location +from .helpers import ProgressIndicatorPercent +from .helpers import bin_to_hex from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 02afc6279..ca52f1472 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -24,7 +24,9 @@ from ..archiver import Archiver from ..cache import Cache from ..constants import * # NOQA from ..crypto import bytes_to_long, num_aes_blocks -from ..helpers import Chunk, Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex +from ..helpers import Chunk, Manifest +from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR +from ..helpers import bin_to_hex from ..key import KeyfileKeyBase from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index ae1f886fd..62d1b74d2 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -11,13 +11,19 @@ import msgpack import msgpack.fallback import time -from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, clean_lines, \ - prune_within, prune_split, get_cache_dir, get_keys_dir, is_slow_msgpack, \ - yes, TRUISH, FALSISH, DEFAULTISH, \ - StableDict, int_to_bigint, bigint_to_int, bin_to_hex, parse_timestamp, ChunkerParams, Chunk, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \ - CompressionSpec, CompressionDecider1, CompressionDecider2, \ - PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, partial_format, ChunkIteratorFileWrapper +from ..helpers import Location +from ..helpers import partial_format, format_file_size, format_timedelta +from ..helpers import make_path_safe, clean_lines +from ..helpers import prune_within, prune_split +from ..helpers import get_cache_dir, get_keys_dir +from ..helpers import is_slow_msgpack +from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH +from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex +from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk +from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless +from ..helpers import load_excludes +from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2 +from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern from . import BaseTestCase, environment_variable, FakeInputs diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 8a1642a1d..258ec929b 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -9,7 +9,9 @@ import pytest from ..crypto import bytes_to_long, num_aes_blocks from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex -from ..helpers import Location, Chunk, IntegrityError +from ..helpers import Location +from ..helpers import Chunk +from ..helpers import IntegrityError @pytest.fixture(autouse=True) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index cf8c6bb32..88eb5389a 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -7,7 +7,8 @@ import tempfile from unittest.mock import patch from ..hashindex import NSIndex -from ..helpers import Location, IntegrityError, InternalOSError +from ..helpers import Location +from ..helpers import IntegrityError, InternalOSError from ..locking import UpgradableLock, LockFailed from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line from ..repository import Repository, LoggedIO, MAGIC diff --git a/src/borg/upgrader.py b/src/borg/upgrader.py index d0ea9680d..181c60f94 100644 --- a/src/borg/upgrader.py +++ b/src/borg/upgrader.py @@ -5,7 +5,8 @@ import os import shutil import time -from .helpers import get_home_dir, get_keys_dir, get_cache_dir, ProgressIndicatorPercent +from .helpers import get_home_dir, get_keys_dir, get_cache_dir +from .helpers import ProgressIndicatorPercent from .locking import UpgradableLock from .repository import Repository, MAGIC from .key import KeyfileKey, KeyfileNotFoundError From d3deec9fc90554165116d85d9c27231a0aef5dfc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 31 May 2016 01:18:03 +0200 Subject: [PATCH 2/5] cleanup / reorder imports import stdlib from stdlib import x import 3rd party import borg stuff --- src/borg/archive.py | 31 +++++++++++++------------- src/borg/archiver.py | 30 ++++++++++++------------- src/borg/cache.py | 15 +++++++------ src/borg/fuse.py | 16 ++++++++------ src/borg/helpers.py | 40 ++++++++++++++++------------------ src/borg/key.py | 18 ++++++++------- src/borg/platform_darwin.pyx | 1 - src/borg/platform_freebsd.pyx | 1 - src/borg/remote.py | 7 +++--- src/borg/repository.py | 17 ++++++++------- src/borg/shellpattern.py | 2 +- src/borg/testsuite/archiver.py | 9 ++++---- src/borg/testsuite/helpers.py | 10 ++++----- src/borg/testsuite/key.py | 4 +--- src/borg/testsuite/lrucache.py | 6 +++-- src/borg/upgrader.py | 7 +++--- 16 files changed, 106 insertions(+), 108 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 89a1469ac..c4d8c290a 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1,22 +1,25 @@ import errno -from datetime import datetime, timezone -from getpass import getuser -from itertools import groupby - -from .logger import create_logger -logger = create_logger() -from .key import key_factory -from .remote import cache_if_remote - import os -from shutil import get_terminal_size import socket import stat import sys import time +from datetime import datetime, timezone +from getpass import getuser from io import BytesIO +from itertools import groupby +from shutil import get_terminal_size + +import msgpack + +from .logger import create_logger +logger = create_logger() + from . import xattr +from .cache import ChunkListEntry +from .chunker import Chunker from .constants import * # NOQA +from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Manifest from .helpers import Chunk, ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError @@ -30,12 +33,10 @@ from .helpers import ProgressIndicatorPercent, log_multi from .helpers import PathPrefixPattern, FnmatchPattern from .helpers import consume from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec -from .repository import Repository +from .key import key_factory from .platform import acl_get, acl_set, set_flags, get_flags, swidth -from .chunker import Chunker -from .hashindex import ChunkIndex, ChunkIndexEntry -from .cache import ChunkListEntry -import msgpack +from .remote import cache_if_remote +from .repository import Repository has_lchmod = hasattr(os, 'lchmod') diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 9b5a38521..102e91e26 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1,7 +1,3 @@ -from binascii import unhexlify -from datetime import datetime -from itertools import zip_longest -from operator import attrgetter import argparse import collections import functools @@ -18,8 +14,18 @@ import subprocess import sys import textwrap import traceback +from binascii import unhexlify +from datetime import datetime +from itertools import zip_longest + +from .logger import create_logger, setup_logging +logger = create_logger() from . import __version__ +from . import helpers +from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics +from .cache import Cache +from .constants import * # NOQA from .helpers import Error from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec from .helpers import ItemFormatter, format_time, format_file_size, format_archive @@ -32,20 +38,12 @@ from .helpers import update_excludes, check_extension_modules from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo from .helpers import log_multi from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern -from .logger import create_logger, setup_logging -logger = create_logger() -from . import helpers -from .compress import Compressor, COMPR_BUFFER -from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader -from .repository import Repository -from .cache import Cache -from .constants import * # NOQA from .key import key_creator, RepoKey, PassphraseKey -from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics -from .remote import RepositoryServer, RemoteRepository, cache_if_remote -from .selftest import selftest -from .hashindex import ChunkIndexEntry from .platform import get_flags +from .remote import RepositoryServer, RemoteRepository, cache_if_remote +from .repository import Repository +from .selftest import selftest +from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader def argument(args, str_or_bool): diff --git a/src/borg/cache.py b/src/borg/cache.py index f9af7d06d..9b7ad07d7 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -1,23 +1,24 @@ import configparser -from .remote import cache_if_remote -from collections import namedtuple import os import stat -from binascii import unhexlify import shutil +from binascii import unhexlify +from collections import namedtuple + +import msgpack -from .key import PlaintextKey from .logger import create_logger logger = create_logger() + +from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Error from .helpers import get_cache_dir from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex from .helpers import format_file_size from .helpers import yes +from .key import PlaintextKey from .locking import UpgradableLock -from .hashindex import ChunkIndex, ChunkIndexEntry - -import msgpack +from .remote import cache_if_remote ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 36ca3c563..c43ba9399 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -1,20 +1,22 @@ -from collections import defaultdict import errno import io -import llfuse import os import stat import tempfile import time +from collections import defaultdict +from distutils.version import LooseVersion + +import llfuse +import msgpack + +from .logger import create_logger +logger = create_logger() + from .archive import Archive from .helpers import daemonize from .helpers import bigint_to_int -from .logger import create_logger from .lrucache import LRUCache -from distutils.version import LooseVersion -import msgpack - -logger = create_logger() # Does this version of llfuse support ns precision? have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index d93a1c3e7..bd29ecd22 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1,42 +1,40 @@ import argparse -from binascii import hexlify -from collections import namedtuple, deque -from functools import wraps, partial import grp import hashlib -from itertools import islice +import logging import os import os.path -import stat -import textwrap +import platform import pwd import re +import socket import sys -from string import Formatter -import platform +import stat +import textwrap import time import unicodedata -import logging +from binascii import hexlify +from collections import namedtuple, deque +from datetime import datetime, timezone, timedelta +from fnmatch import translate +from functools import wraps, partial +from itertools import islice +from operator import attrgetter +from string import Formatter + +import msgpack +import msgpack.fallback from .logger import create_logger logger = create_logger() -from datetime import datetime, timezone, timedelta -from fnmatch import translate -from operator import attrgetter - from . import __version__ as borg_version -from . import hashindex from . import chunker -from .constants import * # NOQA from . import crypto -from .compress import COMPR_BUFFER, get_compressor +from . import hashindex from . import shellpattern -import msgpack -import msgpack.fallback - -import socket - +from .constants import * # NOQA +from .compress import COMPR_BUFFER, get_compressor # meta dict, data bytes _Chunk = namedtuple('_Chunk', 'meta data') diff --git a/src/borg/key.py b/src/borg/key.py index 6f98401e3..be5338b13 100644 --- a/src/borg/key.py +++ b/src/borg/key.py @@ -1,25 +1,27 @@ -from binascii import a2b_base64, b2a_base64, hexlify import configparser import getpass import os import sys import textwrap -from hmac import compare_digest +from binascii import a2b_base64, b2a_base64, hexlify from hashlib import sha256, pbkdf2_hmac +from hmac import compare_digest +import msgpack + +from .logger import create_logger +logger = create_logger() + +from .constants import * # NOQA +from .compress import Compressor, COMPR_BUFFER, get_compressor +from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256 from .helpers import Chunk from .helpers import Error, IntegrityError from .helpers import yes from .helpers import get_keys_dir from .helpers import bin_to_hex from .helpers import CompressionDecider2, CompressionSpec -from .logger import create_logger -logger = create_logger() -from .constants import * # NOQA -from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256 -from .compress import Compressor, COMPR_BUFFER, get_compressor -import msgpack PREFIX = b'\0' * 8 diff --git a/src/borg/platform_darwin.pyx b/src/borg/platform_darwin.pyx index 6b552c3aa..7e88376be 100644 --- a/src/borg/platform_darwin.pyx +++ b/src/borg/platform_darwin.pyx @@ -2,7 +2,6 @@ import os from .helpers import user2uid, group2gid from .helpers import safe_decode, safe_encode - from .platform_posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_freebsd.pyx b/src/borg/platform_freebsd.pyx index 4b9dd49eb..ecf51beda 100644 --- a/src/borg/platform_freebsd.pyx +++ b/src/borg/platform_freebsd.pyx @@ -2,7 +2,6 @@ import os from .helpers import posix_acl_use_stored_uid_gid from .helpers import safe_encode, safe_decode - from .platform_posix import swidth API_VERSION = 3 diff --git a/src/borg/remote.py b/src/borg/remote.py index 12fbb6024..4fb9463fc 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -4,20 +4,19 @@ import logging import os import select import shlex -from subprocess import Popen, PIPE import sys import tempfile +from subprocess import Popen, PIPE + +import msgpack from . import __version__ - from .helpers import Error, IntegrityError from .helpers import get_home_dir from .helpers import sysinfo from .helpers import bin_to_hex from .repository import Repository -import msgpack - RPC_PROTOCOL_VERSION = 2 BUFSIZE = 10 * 1024 * 1024 diff --git a/src/borg/repository.py b/src/borg/repository.py index 3d8c5ae01..0714f7262 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,25 +1,26 @@ -from configparser import ConfigParser -from binascii import unhexlify -from datetime import datetime -from itertools import islice import errno -import logging -logger = logging.getLogger(__name__) - import os import shutil import struct +from binascii import unhexlify from collections import defaultdict +from configparser import ConfigParser +from datetime import datetime from functools import partial +from itertools import islice from zlib import crc32 import msgpack + +import logging +logger = logging.getLogger(__name__) + from .constants import * # NOQA +from .hashindex import NSIndex from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError from .helpers import Location from .helpers import ProgressIndicatorPercent from .helpers import bin_to_hex -from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache from .platform import SyncFile, sync_dir diff --git a/src/borg/shellpattern.py b/src/borg/shellpattern.py index 7cb8f2114..e669b0ff4 100644 --- a/src/borg/shellpattern.py +++ b/src/borg/shellpattern.py @@ -1,5 +1,5 @@ -import re import os +import re def translate(pat): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index ca52f1472..a6ce4ce7a 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -17,6 +17,10 @@ from unittest.mock import patch from hashlib import sha256 import pytest +try: + import llfuse +except ImportError: + pass from .. import xattr, helpers, platform from ..archive import Archive, ChunkBuffer, ArchiveRecreater @@ -33,11 +37,6 @@ from ..repository import Repository from . import has_lchflags, has_llfuse from . import BaseTestCase, changedir, environment_variable -try: - import llfuse -except ImportError: - pass - src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 62d1b74d2..ede20e16f 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -1,15 +1,13 @@ import hashlib -import io import logging -from time import mktime, strptime -from datetime import datetime, timezone, timedelta import os +import sys +from datetime import datetime, timezone, timedelta +from time import mktime, strptime, sleep import pytest -import sys import msgpack import msgpack.fallback -import time from ..helpers import Location from ..helpers import partial_format, format_file_size, format_timedelta @@ -121,7 +119,7 @@ class TestLocationWithoutEnv: assert repr(Location('/some/path::archive{pid}')) == \ "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive{}')".format(test_pid) location_time1 = Location('/some/path::archive{now:%s}') - time.sleep(1.1) + sleep(1.1) location_time2 = Location('/some/path::archive{now:%s}') assert location_time1.archive != location_time2.archive diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 258ec929b..c62f5ffef 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -1,17 +1,15 @@ import getpass -import os import re -import shutil import tempfile from binascii import hexlify, unhexlify import pytest from ..crypto import bytes_to_long, num_aes_blocks -from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex from ..helpers import Location from ..helpers import Chunk from ..helpers import IntegrityError +from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex @pytest.fixture(autouse=True) diff --git a/src/borg/testsuite/lrucache.py b/src/borg/testsuite/lrucache.py index 2ed2ef9b3..9fb4f92b6 100644 --- a/src/borg/testsuite/lrucache.py +++ b/src/borg/testsuite/lrucache.py @@ -1,7 +1,9 @@ -from ..lrucache import LRUCache -import pytest from tempfile import TemporaryFile +import pytest + +from ..lrucache import LRUCache + class TestLRUCache: diff --git a/src/borg/upgrader.py b/src/borg/upgrader.py index 181c60f94..af692bb60 100644 --- a/src/borg/upgrader.py +++ b/src/borg/upgrader.py @@ -1,15 +1,16 @@ import datetime -import logging -logger = logging.getLogger(__name__) import os import shutil import time +import logging +logger = logging.getLogger(__name__) + from .helpers import get_home_dir, get_keys_dir, get_cache_dir from .helpers import ProgressIndicatorPercent +from .key import KeyfileKey, KeyfileNotFoundError from .locking import UpgradableLock from .repository import Repository, MAGIC -from .key import KeyfileKey, KeyfileNotFoundError ATTIC_MAGIC = b'ATTICSEG' From 2d90bb2c5514c050a5cdcc14cd3eae30f7299db8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 31 May 2016 02:22:27 +0200 Subject: [PATCH 3/5] Compose platform_base from the platform-specific functionality Noticed while working on the windows branch. --- src/borg/platform_base.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/borg/platform_base.py b/src/borg/platform_base.py index da62260b6..e8d28b52d 100644 --- a/src/borg/platform_base.py +++ b/src/borg/platform_base.py @@ -1,5 +1,17 @@ import os +""" +platform base module +==================== + +Contains platform API implementations based on what Python itself provides. More specific +APIs are stubs in this module. + +When functions in this module use platform APIs themselves they access the public +platform API: that way platform APIs provided by the platform-specific support module +are correctly composed into the base functionality. +""" + API_VERSION = 3 fdatasync = getattr(os, 'fdatasync', os.fsync) @@ -80,16 +92,18 @@ class SyncFile: Synchronize file contents. Everything written prior to sync() must become durable before anything written after sync(). """ + from .. import platform self.fd.flush() - fdatasync(self.fileno) + platform.fdatasync(self.fileno) if hasattr(os, 'posix_fadvise'): os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED) def close(self): """sync() and close.""" + from .. import platform self.sync() self.fd.close() - sync_dir(os.path.dirname(self.fd.name)) + platform.sync_dir(os.path.dirname(self.fd.name)) def swidth(s): From 0333f91eb2f4f70dcca0cc50cf59a7a8065120e3 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 31 May 2016 02:22:38 +0200 Subject: [PATCH 4/5] platform: rearrange imports --- src/borg/platform.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/borg/platform.py b/src/borg/platform.py index d05b60fb9..88f32ca6b 100644 --- a/src/borg/platform.py +++ b/src/borg/platform.py @@ -1,10 +1,24 @@ import sys -from .platform_base import acl_get, acl_set, SyncFile, sync_dir, set_flags, get_flags, swidth, API_VERSION +""" +Platform-specific APIs. + +Public APIs are documented in platform_base. +""" + +from .platform_base import acl_get, acl_set +from .platform_base import set_flags, get_flags +from .platform_base import SyncFile, sync_dir, fdatasync +from .platform_base import swidth, API_VERSION if sys.platform.startswith('linux'): # pragma: linux only - from .platform_linux import acl_get, acl_set, SyncFile, set_flags, get_flags, swidth, API_VERSION + from .platform_linux import acl_get, acl_set + from .platform_linux import set_flags, get_flags + from .platform_linux import SyncFile + from .platform_linux import swidth, API_VERSION elif sys.platform.startswith('freebsd'): # pragma: freebsd only - from .platform_freebsd import acl_get, acl_set, swidth, API_VERSION + from .platform_freebsd import acl_get, acl_set + from .platform_freebsd import swidth, API_VERSION elif sys.platform == 'darwin': # pragma: darwin only - from .platform_darwin import acl_get, acl_set, swidth, API_VERSION + from .platform_darwin import acl_get, acl_set + from .platform_darwin import swidth, API_VERSION From 0ca1a18b33e1ff6ce68d40f64a3f235b69c5aeb8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 31 May 2016 02:35:54 +0200 Subject: [PATCH 5/5] Make borg.platform a package --- setup.py | 38 ++++++++++++------- src/borg/platform.py | 24 ------------ src/borg/platform/__init__.py | 24 ++++++++++++ .../{platform_base.py => platform/base.py} | 0 .../darwin.pyx} | 6 +-- .../freebsd.pyx} | 6 +-- .../linux.pyx} | 10 ++--- .../posix.pyx} | 0 src/borg/repository.py | 2 +- src/borg/testsuite/platform.py | 2 +- 10 files changed, 62 insertions(+), 50 deletions(-) delete mode 100644 src/borg/platform.py create mode 100644 src/borg/platform/__init__.py rename src/borg/{platform_base.py => platform/base.py} (100%) rename src/borg/{platform_darwin.pyx => platform/darwin.pyx} (95%) rename src/borg/{platform_freebsd.pyx => platform/freebsd.pyx} (95%) rename src/borg/{platform_linux.pyx => platform/linux.pyx} (97%) rename src/borg/{platform_posix.pyx => platform/posix.pyx} (100%) diff --git a/setup.py b/setup.py index 90fe402ed..23f2f97ea 100644 --- a/setup.py +++ b/setup.py @@ -45,10 +45,22 @@ compress_source = 'src/borg/compress.pyx' crypto_source = 'src/borg/crypto.pyx' chunker_source = 'src/borg/chunker.pyx' hashindex_source = 'src/borg/hashindex.pyx' -platform_posix_source = 'src/borg/platform_posix.pyx' -platform_linux_source = 'src/borg/platform_linux.pyx' -platform_darwin_source = 'src/borg/platform_darwin.pyx' -platform_freebsd_source = 'src/borg/platform_freebsd.pyx' +platform_posix_source = 'src/borg/platform/posix.pyx' +platform_linux_source = 'src/borg/platform/linux.pyx' +platform_darwin_source = 'src/borg/platform/darwin.pyx' +platform_freebsd_source = 'src/borg/platform/freebsd.pyx' + +cython_sources = [ + compress_source, + crypto_source, + chunker_source, + hashindex_source, + + platform_posix_source, + platform_linux_source, + platform_freebsd_source, + platform_darwin_source, +] try: from Cython.Distutils import build_ext @@ -56,7 +68,7 @@ try: class Sdist(sdist): def __init__(self, *args, **kwargs): - for src in glob('src/borg/*.pyx'): + for src in cython_sources: cython_compiler.compile(src, cython_compiler.default_options) super().__init__(*args, **kwargs) @@ -66,10 +78,10 @@ try: 'src/borg/crypto.c', 'src/borg/chunker.c', 'src/borg/_chunker.c', 'src/borg/hashindex.c', 'src/borg/_hashindex.c', - 'src/borg/platform_posix.c', - 'src/borg/platform_linux.c', - 'src/borg/platform_freebsd.c', - 'src/borg/platform_darwin.c', + 'src/borg/platform/posix.c', + 'src/borg/platform/linux.c', + 'src/borg/platform/freebsd.c', + 'src/borg/platform/darwin.c', ]) super().make_distribution() @@ -295,14 +307,14 @@ if not on_rtd: Extension('borg.hashindex', [hashindex_source]) ] if sys.platform.startswith(('linux', 'freebsd', 'darwin')): - ext_modules.append(Extension('borg.platform_posix', [platform_posix_source])) + ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) if sys.platform == 'linux': - ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl'])) + ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])) elif sys.platform.startswith('freebsd'): - ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source])) + ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source])) elif sys.platform == 'darwin': - ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source])) + ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source])) setup( name='borgbackup', diff --git a/src/borg/platform.py b/src/borg/platform.py deleted file mode 100644 index 88f32ca6b..000000000 --- a/src/borg/platform.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys - -""" -Platform-specific APIs. - -Public APIs are documented in platform_base. -""" - -from .platform_base import acl_get, acl_set -from .platform_base import set_flags, get_flags -from .platform_base import SyncFile, sync_dir, fdatasync -from .platform_base import swidth, API_VERSION - -if sys.platform.startswith('linux'): # pragma: linux only - from .platform_linux import acl_get, acl_set - from .platform_linux import set_flags, get_flags - from .platform_linux import SyncFile - from .platform_linux import swidth, API_VERSION -elif sys.platform.startswith('freebsd'): # pragma: freebsd only - from .platform_freebsd import acl_get, acl_set - from .platform_freebsd import swidth, API_VERSION -elif sys.platform == 'darwin': # pragma: darwin only - from .platform_darwin import acl_get, acl_set - from .platform_darwin import swidth, API_VERSION diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py new file mode 100644 index 000000000..29a97fc43 --- /dev/null +++ b/src/borg/platform/__init__.py @@ -0,0 +1,24 @@ +import sys + +""" +Platform-specific APIs. + +Public APIs are documented in platform.base. +""" + +from .base import acl_get, acl_set +from .base import set_flags, get_flags +from .base import SyncFile, sync_dir, fdatasync +from .base import swidth, API_VERSION + +if sys.platform.startswith('linux'): # pragma: linux only + from .linux import acl_get, acl_set + from .linux import set_flags, get_flags + from .linux import SyncFile + from .linux import swidth, API_VERSION +elif sys.platform.startswith('freebsd'): # pragma: freebsd only + from .freebsd import acl_get, acl_set + from .freebsd import swidth, API_VERSION +elif sys.platform == 'darwin': # pragma: darwin only + from .darwin import acl_get, acl_set + from .darwin import swidth, API_VERSION diff --git a/src/borg/platform_base.py b/src/borg/platform/base.py similarity index 100% rename from src/borg/platform_base.py rename to src/borg/platform/base.py diff --git a/src/borg/platform_darwin.pyx b/src/borg/platform/darwin.pyx similarity index 95% rename from src/borg/platform_darwin.pyx rename to src/borg/platform/darwin.pyx index 7e88376be..016ad5aab 100644 --- a/src/borg/platform_darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -1,8 +1,8 @@ import os -from .helpers import user2uid, group2gid -from .helpers import safe_decode, safe_encode -from .platform_posix import swidth +from ..helpers import user2uid, group2gid +from ..helpers import safe_decode, safe_encode +from .posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_freebsd.pyx b/src/borg/platform/freebsd.pyx similarity index 95% rename from src/borg/platform_freebsd.pyx rename to src/borg/platform/freebsd.pyx index ecf51beda..7553a26cc 100644 --- a/src/borg/platform_freebsd.pyx +++ b/src/borg/platform/freebsd.pyx @@ -1,8 +1,8 @@ import os -from .helpers import posix_acl_use_stored_uid_gid -from .helpers import safe_encode, safe_decode -from .platform_posix import swidth +from ..helpers import posix_acl_use_stored_uid_gid +from ..helpers import safe_encode, safe_decode +from .posix import swidth API_VERSION = 3 diff --git a/src/borg/platform_linux.pyx b/src/borg/platform/linux.pyx similarity index 97% rename from src/borg/platform_linux.pyx rename to src/borg/platform/linux.pyx index 7a0c83d39..76d76521f 100644 --- a/src/borg/platform_linux.pyx +++ b/src/borg/platform/linux.pyx @@ -3,11 +3,11 @@ import re import resource import stat -from .helpers import posix_acl_use_stored_uid_gid -from .helpers import user2uid, group2gid -from .helpers import safe_decode, safe_encode -from .platform_base import SyncFile as BaseSyncFile -from .platform_posix import swidth +from ..helpers import posix_acl_use_stored_uid_gid +from ..helpers import user2uid, group2gid +from ..helpers import safe_decode, safe_encode +from .base import SyncFile as BaseSyncFile +from .posix import swidth from libc cimport errno from libc.stdint cimport int64_t diff --git a/src/borg/platform_posix.pyx b/src/borg/platform/posix.pyx similarity index 100% rename from src/borg/platform_posix.pyx rename to src/borg/platform/posix.pyx diff --git a/src/borg/repository.py b/src/borg/repository.py index 0714f7262..704131839 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -55,7 +55,7 @@ class Repository: established by a COMMIT. Note that the COMMIT can't establish consistency by itself, but only manages to do so with proper support from - the platform (including the hardware). See platform_base.SyncFile for details. + the platform (including the hardware). See platform.base.SyncFile for details. A PUT inserts a key-value pair. The value is stored in the log entry, hence the repository implements full data logging, meaning that all data is consistent, not just metadata (which is common in file systems). diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 9eaede988..857920079 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -102,7 +102,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_in(group_entry_numeric, acl_access_numeric) def test_utils(self): - from ..platform_linux import acl_use_local_uid_gid + from ..platform.linux import acl_use_local_uid_gid self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-') self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-') self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-')