From bbc3f8d6fdc5970d0dd8ce3ae598a483de83610c Mon Sep 17 00:00:00 2001 From: Sitaram Chamarty Date: Sun, 17 Mar 2019 19:35:04 +0530 Subject: [PATCH 1/5] add a sample logging.conf in docs/misc (#4380) add a sample logging.conf in docs/misc (and point to it in the man page) --- docs/misc/logging.conf | 23 +++++++++++++++++++++++ docs/usage_general.rst.inc | 1 + 2 files changed, 24 insertions(+) create mode 100644 docs/misc/logging.conf diff --git a/docs/misc/logging.conf b/docs/misc/logging.conf new file mode 100644 index 000000000..be8fb9cdc --- /dev/null +++ b/docs/misc/logging.conf @@ -0,0 +1,23 @@ +[loggers] +keys=root + +[handlers] +keys=logfile + +[formatters] +keys=logfile + +[logger_root] +level=NOTSET +handlers=logfile + +[handler_logfile] +class=FileHandler +level=INFO +formatter=logfile +args=('borg.log', 'w') + +[formatter_logfile] +format=%(asctime)s %(levelname)s %(message)s +datefmt= +class=logging.Formatter diff --git a/docs/usage_general.rst.inc b/docs/usage_general.rst.inc index d2d7d5c27..2d4c05333 100644 --- a/docs/usage_general.rst.inc +++ b/docs/usage_general.rst.inc @@ -202,6 +202,7 @@ General: use fqdn@uniqueid. BORG_LOGGING_CONF When set, use the given filename as INI_-style logging configuration. + A basic example conf can be found at ``docs/misc/logging.conf``. BORG_RSH When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options. Using From 8ad90c1242de7f384fe4d463ae6131c7faa2e648 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Feb 2019 10:09:40 +0100 Subject: [PATCH 2/5] remove ARCHIVE_KEYS duplication also: get key set in sync, obviously we have "recreate_partial_chunks" in ArchiveItem still. --- src/borg/constants.py | 2 +- src/borg/item.pyx | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/borg/constants.py b/src/borg/constants.py index 5a67da54a..1d12d02ce 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -10,7 +10,7 @@ REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ]) # this set must be kept complete, otherwise rebuild_manifest might malfunction: ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'hostname', 'username', 'time', 'time_end', 'comment', 'chunker_params', - 'recreate_cmdline', 'recreate_source_id', 'recreate_args']) + 'recreate_cmdline', 'recreate_source_id', 'recreate_args', 'recreate_partial_chunks',]) # this is the set of keys that are always present in archives: REQUIRED_ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'time', ]) diff --git a/src/borg/item.pyx b/src/borg/item.pyx index 1dc0b1f61..140ac03fd 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -3,7 +3,7 @@ import stat from collections import namedtuple -from .constants import ITEM_KEYS +from .constants import ITEM_KEYS, ARCHIVE_KEYS from .helpers import safe_encode, safe_decode from .helpers import bigint_to_int, int_to_bigint from .helpers import StableDict @@ -292,10 +292,7 @@ class ArchiveItem(PropDict): If a ArchiveItem shall be serialized, give as_dict() method output to msgpack packer. """ - VALID_KEYS = {'version', 'name', 'items', 'cmdline', 'hostname', 'username', 'time', 'time_end', - 'comment', 'chunker_params', - 'recreate_cmdline', 'recreate_source_id', 'recreate_args', 'recreate_partial_chunks', - } # str-typed keys + VALID_KEYS = ARCHIVE_KEYS # str-typed keys __slots__ = ("_dict", ) # avoid setting attributes not supported by properties From 1efec3ced145e5849775751c2f8c979c4f361304 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Feb 2019 10:49:24 +0100 Subject: [PATCH 3/5] add comment about unused recreate_* members in ArchiveItem --- src/borg/constants.py | 4 +++- src/borg/helpers.py | 2 +- src/borg/item.pyx | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/borg/constants.py b/src/borg/constants.py index 1d12d02ce..85c26fb2b 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -10,7 +10,9 @@ REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ]) # this set must be kept complete, otherwise rebuild_manifest might malfunction: ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'hostname', 'username', 'time', 'time_end', 'comment', 'chunker_params', - 'recreate_cmdline', 'recreate_source_id', 'recreate_args', 'recreate_partial_chunks',]) + 'recreate_cmdline', + 'recreate_source_id', 'recreate_args', 'recreate_partial_chunks', # used in 1.1.0b1 .. b2 + ]) # this is the set of keys that are always present in archives: REQUIRED_ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'time', ]) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index ed6b3afeb..f7a02e2d9 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -141,7 +141,7 @@ def check_extension_modules(): raise ExtensionModuleError if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_04': raise ExtensionModuleError - if item.API_VERSION != '1.1_02': + if item.API_VERSION != '1.1_03': raise ExtensionModuleError diff --git a/src/borg/item.pyx b/src/borg/item.pyx index 140ac03fd..6b3cc70ce 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -8,7 +8,7 @@ from .helpers import safe_encode, safe_decode from .helpers import bigint_to_int, int_to_bigint from .helpers import StableDict -API_VERSION = '1.1_02' +API_VERSION = '1.1_03' class PropDict: @@ -306,8 +306,9 @@ class ArchiveItem(PropDict): time_end = PropDict._make_property('time_end', str, 'surrogate-escaped str', encode=safe_encode, decode=safe_decode) comment = PropDict._make_property('comment', str, 'surrogate-escaped str', encode=safe_encode, decode=safe_decode) chunker_params = PropDict._make_property('chunker_params', tuple) - recreate_source_id = PropDict._make_property('recreate_source_id', bytes) recreate_cmdline = PropDict._make_property('recreate_cmdline', list) # list of s-e-str + # recreate_source_id, recreate_args, recreate_partial_chunks were used in 1.1.0b1 .. b2 + recreate_source_id = PropDict._make_property('recreate_source_id', bytes) recreate_args = PropDict._make_property('recreate_args', list) # list of s-e-str recreate_partial_chunks = PropDict._make_property('recreate_partial_chunks', list) # list of tuples From 6ee42216bfb4e0c13009bde4f58b61ab78e0c407 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Jun 2018 02:43:34 +0200 Subject: [PATCH 4/5] avoid stale filehandle issues, fixes #3265 --- src/borg/constants.py | 2 ++ src/borg/lrucache.py | 6 ++++++ src/borg/repository.py | 33 ++++++++++++++++++++++++++------- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/borg/constants.py b/src/borg/constants.py index 85c26fb2b..0de72b044 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -54,6 +54,8 @@ LIST_SCAN_LIMIT = 100000 DEFAULT_SEGMENTS_PER_DIR = 1000 +FD_MAX_AGE = 4 * 60 # 4 minutes + CHUNK_MIN_EXP = 19 # 2**19 == 512kiB CHUNK_MAX_EXP = 23 # 2**23 == 8MiB HASH_WINDOW_SIZE = 0xfff # 4095B diff --git a/src/borg/lrucache.py b/src/borg/lrucache.py index 492e18b62..d362bc92b 100644 --- a/src/borg/lrucache.py +++ b/src/borg/lrucache.py @@ -39,6 +39,12 @@ class LRUCache: self._lru.append(key) return value + def upd(self, key, value): + # special use only: update the value for an existing key without having to dispose it first + # this method complements __setitem__ which should be used for the normal use case. + assert key in self._cache, "Unexpected attempt to update a non-existing item." + self._cache[key] = value + def clear(self): for value in self._cache.values(): self._dispose(value) diff --git a/src/borg/repository.py b/src/borg/repository.py index afa6a3f91..3137d0809 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -3,6 +3,7 @@ import mmap import os import shutil import struct +import time from binascii import hexlify, unhexlify from collections import defaultdict from configparser import ConfigParser @@ -1164,8 +1165,7 @@ class LoggedIO: def __init__(self, path, limit, segments_per_dir, capacity=90): self.path = path - self.fds = LRUCache(capacity, - dispose=self.close_fd) + self.fds = LRUCache(capacity, dispose=self._close_fd) self.segment = 0 self.limit = limit self.segments_per_dir = segments_per_dir @@ -1177,7 +1177,8 @@ class LoggedIO: self.fds.clear() self.fds = None # Just to make sure we're disabled - def close_fd(self, fd): + def _close_fd(self, ts_fd): + ts, fd = ts_fd safe_fadvise(fd.fileno(), 0, 0, 'DONTNEED') fd.close() @@ -1291,13 +1292,31 @@ class LoggedIO: return self._write_fd def get_fd(self, segment): - try: - return self.fds[segment] - except KeyError: + # note: get_fd() returns a fd with undefined file pointer position, + # so callers must always seek() to desired position afterwards. + now = time.monotonic() + + def open_fd(): fd = open(self.segment_filename(segment), 'rb') - self.fds[segment] = fd + self.fds[segment] = (now, fd) return fd + try: + ts, fd = self.fds[segment] + except KeyError: + fd = open_fd() + else: + if now - ts > FD_MAX_AGE: + # we do not want to touch long-unused file handles to + # avoid ESTALE issues (e.g. on network filesystems). + del self.fds[segment] + fd = open_fd() + else: + # fd is fresh enough, so we use it. + # also, we update the timestamp of the lru cache entry. + self.fds.upd(segment, (now, fd)) + return fd + def close_segment(self): # set self._write_fd to None early to guard against reentry from error handling code paths: fd, self._write_fd = self._write_fd, None From 0f72a8a0cf5d2a4950f119f2434f73169f112e31 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Mar 2019 02:27:41 +0100 Subject: [PATCH 5/5] lrucache: regularly remove old FDs, fixes #4427 --- src/borg/lrucache.py | 1 - src/borg/repository.py | 25 ++++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/borg/lrucache.py b/src/borg/lrucache.py index d362bc92b..4f7f1f829 100644 --- a/src/borg/lrucache.py +++ b/src/borg/lrucache.py @@ -50,7 +50,6 @@ class LRUCache: self._dispose(value) self._cache.clear() - # useful for testing def items(self): return self._cache.items() diff --git a/src/borg/repository.py b/src/borg/repository.py index 3137d0809..ef70ccd69 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1171,6 +1171,7 @@ class LoggedIO: self.segments_per_dir = segments_per_dir self.offset = 0 self._write_fd = None + self._fds_cleaned = 0 def close(self): self.close_segment() @@ -1301,20 +1302,26 @@ class LoggedIO: self.fds[segment] = (now, fd) return fd + def clean_old(): + # we regularly get rid of all old FDs here: + if now - self._fds_cleaned > FD_MAX_AGE // 8: + self._fds_cleaned = now + for k, ts_fd in list(self.fds.items()): + ts, fd = ts_fd + if now - ts > FD_MAX_AGE: + # we do not want to touch long-unused file handles to + # avoid ESTALE issues (e.g. on network filesystems). + del self.fds[k] + + clean_old() try: ts, fd = self.fds[segment] except KeyError: fd = open_fd() else: - if now - ts > FD_MAX_AGE: - # we do not want to touch long-unused file handles to - # avoid ESTALE issues (e.g. on network filesystems). - del self.fds[segment] - fd = open_fd() - else: - # fd is fresh enough, so we use it. - # also, we update the timestamp of the lru cache entry. - self.fds.upd(segment, (now, fd)) + # we only have fresh enough stuff here. + # update the timestamp of the lru cache entry. + self.fds.upd(segment, (now, fd)) return fd def close_segment(self):