diff --git a/src/borg/archiver.py b/src/borg/archiver.py index ea70f412a..eac059bb7 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -360,6 +360,10 @@ class Archiver: if chunks_healthy is not None: item._dict['chunks_healthy'] = chunks item._dict.pop('source') # not used for hardlinks any more, replaced by hlid + for attr in 'atime', 'ctime', 'mtime', 'birthtime': + if attr in item: + ns = getattr(item, attr) # decode (bigint or Timestamp) --> int ns + setattr(item, attr, ns) # encode int ns --> msgpack.Timestamp only, no bigint any more item._dict.pop('hardlink_master', None) # not used for hardlinks any more, replaced by hlid item._dict.pop('acl', None) # remove remnants of bug in attic <= 0.13 item.get_size(memorize=True) # if not already present: compute+remember size for items with chunks diff --git a/src/borg/cache.py b/src/borg/cache.py index 6fa74e692..6cd612359 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -19,7 +19,7 @@ from .helpers import Location from .helpers import Error from .helpers import Manifest from .helpers import get_cache_dir, get_security_dir -from .helpers import int_to_bigint, bigint_to_int, bin_to_hex, parse_stringified_list +from .helpers import bin_to_hex, parse_stringified_list from .helpers import format_file_size from .helpers import safe_ns from .helpers import yes @@ -28,6 +28,7 @@ from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage from .helpers import set_ec, EXIT_WARNING from .helpers import safe_unlink from .helpers import msgpack +from .helpers.msgpack import int_to_timestamp, timestamp_to_int from .item import ArchiveItem, ChunkListEntry from .crypto.key import PlaintextKey from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError @@ -623,7 +624,7 @@ class LocalCache(CacheStatsMixin): # this is to avoid issues with filesystem snapshots and cmtime granularity. # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet. entry = FileCacheEntry(*msgpack.unpackb(item)) - if entry.age == 0 and bigint_to_int(entry.cmtime) < self._newest_cmtime or \ + if entry.age == 0 and timestamp_to_int(entry.cmtime) < self._newest_cmtime or \ entry.age > 0 and entry.age < ttl: msgpack.pack((path_hash, entry), fd) entry_count += 1 @@ -1018,10 +1019,10 @@ class LocalCache(CacheStatsMixin): if 'i' in cache_mode and entry.inode != st.st_ino: files_cache_logger.debug('KNOWN-CHANGED: file inode number has changed: %r', hashed_path) return True, None - if 'c' in cache_mode and bigint_to_int(entry.cmtime) != st.st_ctime_ns: + if 'c' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns: files_cache_logger.debug('KNOWN-CHANGED: file ctime has changed: %r', hashed_path) return True, None - elif 'm' in cache_mode and bigint_to_int(entry.cmtime) != st.st_mtime_ns: + elif 'm' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns: files_cache_logger.debug('KNOWN-CHANGED: file mtime has changed: %r', hashed_path) return True, None # we ignored the inode number in the comparison above or it is still same. @@ -1049,7 +1050,7 @@ class LocalCache(CacheStatsMixin): elif 'm' in cache_mode: cmtime_type = 'mtime' cmtime_ns = safe_ns(st.st_mtime_ns) - entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_bigint(cmtime_ns), chunk_ids=ids) + entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids) self.files[path_hash] = msgpack.packb(entry) self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns) files_cache_logger.debug('FILES-CACHE-UPDATE: put %r [has %s] <- %r', diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py index 2ace88fee..411f00fec 100644 --- a/src/borg/helpers/msgpack.py +++ b/src/borg/helpers/msgpack.py @@ -24,7 +24,7 @@ from msgpack import unpackb as mp_unpackb from msgpack import unpack as mp_unpack from msgpack import version as mp_version -from msgpack import ExtType +from msgpack import ExtType, Timestamp from msgpack import OutOfData @@ -164,7 +164,7 @@ def get_limited_unpacker(kind): return Unpacker(**args) -def bigint_to_int(mtime): +def bigint_to_int(mtime): # legacy """Convert bytearray to int """ if isinstance(mtime, bytes): @@ -172,7 +172,7 @@ def bigint_to_int(mtime): return mtime -def int_to_bigint(value): +def int_to_bigint(value): # legacy """Convert integers larger than 64 bits to bytearray Smaller integers are left alone @@ -180,3 +180,14 @@ def int_to_bigint(value): if value.bit_length() > 63: return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True) return value + + +def int_to_timestamp(ns): + return Timestamp.from_unix_nano(ns) + + +def timestamp_to_int(ts): + if isinstance(ts, Timestamp): + return ts.to_unix_nano() + # legacy support note: we need to keep the bigint conversion for compatibility with borg < 1.3 archives. + return bigint_to_int(ts) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 3e145ada2..9bcbce22c 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -19,6 +19,7 @@ logger = create_logger() from .errors import Error from .fs import get_keys_dir +from .msgpack import Timestamp from .time import OutputTimestamp, format_time, to_localtime, safe_timestamp, safe_s from .. import __version__ as borg_version from .. import __version_tuple__ as borg_version_tuple @@ -1043,6 +1044,8 @@ def prepare_dump_dict(d): value = decode_tuple(value) elif isinstance(value, bytes): value = decode_bytes(value) + elif isinstance(value, Timestamp): + value = value.to_unix_nano() if isinstance(key, bytes): key = key.decode() res[key] = value diff --git a/src/borg/item.pyx b/src/borg/item.pyx index 0b2598ffe..764279db0 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -3,9 +3,9 @@ from collections import namedtuple from .constants import ITEM_KEYS, ARCHIVE_KEYS from .helpers import safe_encode, safe_decode -from .helpers import bigint_to_int, int_to_bigint from .helpers import StableDict from .helpers import format_file_size +from .helpers.msgpack import timestamp_to_int, int_to_timestamp cdef extern from "_item.c": @@ -171,11 +171,10 @@ class Item(PropDict): rdev = PropDict._make_property('rdev', int) bsdflags = PropDict._make_property('bsdflags', int) - # note: we need to keep the bigint conversion for compatibility with borg 1.0 archives. - atime = PropDict._make_property('atime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) - ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) - mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) - birthtime = PropDict._make_property('birthtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) + atime = PropDict._make_property('atime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int) + ctime = PropDict._make_property('ctime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int) + mtime = PropDict._make_property('mtime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int) + birthtime = PropDict._make_property('birthtime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int) # size is only present for items with a chunk list and then it is sum(chunk_sizes) # compatibility note: this is a new feature, in old archives size will be missing. diff --git a/src/borg/testsuite/item.py b/src/borg/testsuite/item.py index aa40cc066..80b38edce 100644 --- a/src/borg/testsuite/item.py +++ b/src/borg/testsuite/item.py @@ -3,6 +3,7 @@ import pytest from ..cache import ChunkListEntry from ..item import Item from ..helpers import StableDict +from ..helpers.msgpack import Timestamp def test_item_empty(): @@ -77,15 +78,15 @@ def test_item_int_property(): item.mode = "invalid" -def test_item_bigint_property(): +def test_item_mptimestamp_property(): item = Item() small, big = 42, 2 ** 65 item.atime = small assert item.atime == small - assert item.as_dict() == {'atime': small} + assert item.as_dict() == {'atime': Timestamp.from_unix_nano(small)} item.atime = big assert item.atime == big - assert item.as_dict() == {'atime': b'\0' * 8 + b'\x02'} + assert item.as_dict() == {'atime': Timestamp.from_unix_nano(big)} def test_item_user_group_none():