From c492011a929d1745e6842f7ac41cee38fdc05811 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Oct 2015 02:07:55 +0100 Subject: [PATCH 1/4] backup atime and ctime additionally to mtime, fixes #317 restore: mtime and atime FUSE: support ctime and atime additionally to mtime --- borg/archive.py | 18 +++++++++++++----- borg/fuse.py | 13 +++++++------ borg/helpers.py | 14 +++++++++++++- borg/testsuite/archiver.py | 13 ++++++++++++- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 9b060ffe3..2333a1029 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -18,7 +18,8 @@ import time from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \ - Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython + Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython, \ + st_atime_ns, st_ctime_ns, st_mtime_ns if have_cython(): from .platform import acl_get, acl_set from .chunker import Chunker @@ -384,12 +385,17 @@ Number of files: {0.stats.nfiles}'''.format(self) elif has_lchmod: # Not available on Linux os.lchmod(path, item[b'mode']) mtime = bigint_to_int(item[b'mtime']) + if b'atime' in item: + atime = bigint_to_int(item[b'atime']) + else: + # old archives only had mtime in item metadata + atime = mtime if fd and utime_supports_fd: # Python >= 3.3 - os.utime(fd, None, ns=(mtime, mtime)) + os.utime(fd, None, ns=(atime, mtime)) elif utime_supports_follow_symlinks: # Python >= 3.3 - os.utime(path, None, ns=(mtime, mtime), follow_symlinks=False) + os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) elif not symlink: - os.utime(path, (mtime / 1e9, mtime / 1e9)) + os.utime(path, (atime / 1e9, mtime / 1e9)) acl_set(path, item, self.numeric_owner) # Only available on OS X and FreeBSD if has_lchflags and b'bsdflags' in item: @@ -428,7 +434,9 @@ Number of files: {0.stats.nfiles}'''.format(self) b'mode': st.st_mode, b'uid': st.st_uid, b'user': uid2user(st.st_uid), b'gid': st.st_gid, b'group': gid2group(st.st_gid), - b'mtime': int_to_bigint(st_mtime_ns(st)) + b'atime': int_to_bigint(st_atime_ns(st)), + b'ctime': int_to_bigint(st_ctime_ns(st)), + b'mtime': int_to_bigint(st_mtime_ns(st)), } if self.numeric_owner: item[b'user'] = item[b'group'] = None diff --git a/borg/fuse.py b/borg/fuse.py index 6f98f0a01..417811feb 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -14,7 +14,7 @@ if have_cython(): import msgpack # Does this version of llfuse support ns precision? -have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') +have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') class ItemCache: @@ -155,14 +155,15 @@ class FuseOperations(llfuse.Operations): entry.st_size = size entry.st_blksize = 512 entry.st_blocks = dsize / 512 - if have_fuse_mtime_ns: - entry.st_atime_ns = item[b'mtime'] + # note: older archives only have mtime (not atime nor ctime) + if have_fuse_xtime_ns: + entry.st_atime_ns = item.get(b'atime') or item[b'mtime'] entry.st_mtime_ns = item[b'mtime'] - entry.st_ctime_ns = item[b'mtime'] + entry.st_ctime_ns = item.get(b'ctime') or item[b'mtime'] else: - entry.st_atime = item[b'mtime'] / 1e9 + entry.st_atime = (item.get(b'atime') or item[b'mtime']) / 1e9 entry.st_mtime = item[b'mtime'] / 1e9 - entry.st_ctime = item[b'mtime'] / 1e9 + entry.st_ctime = (item.get(b'ctime') or item[b'mtime']) / 1e9 return entry def listxattr(self, inode): diff --git a/borg/helpers.py b/borg/helpers.py index 1a36a90d0..c9ffc00b7 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -747,7 +747,13 @@ class StableDict(dict): if sys.version < '3.3': - # st_mtime_ns attribute only available in 3.3+ + # st_xtime_ns attributes only available in 3.3+ + def st_atime_ns(st): + return int(st.st_atime * 1e9) + + def st_ctime_ns(st): + return int(st.st_ctime * 1e9) + def st_mtime_ns(st): return int(st.st_mtime * 1e9) @@ -757,6 +763,12 @@ if sys.version < '3.3': data = data.encode('ascii') return binascii.unhexlify(data) else: + def st_atime_ns(st): + return st.st_atime_ns + + def st_ctime_ns(st): + return st.st_ctime_ns + def st_mtime_ns(st): return st.st_mtime_ns diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index f0adf2cba..d8a91c220 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -20,7 +20,7 @@ from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP from ..archiver import Archiver from ..cache import Cache from ..crypto import bytes_to_long, num_aes_blocks -from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR +from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, st_atime_ns, st_mtime_ns from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import BaseTestCase @@ -286,6 +286,17 @@ class ArchiverTestCase(ArchiverTestCaseBase): # end the same way as info_output assert info_output2.endswith(info_output) + def test_atime(self): + have_root = self.create_test_files() + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + sti = os.stat('input/file1') + sto = os.stat('output/input/file1') + assert st_mtime_ns(sti) == st_mtime_ns(sto) + assert st_atime_ns(sti) == st_atime_ns(sto) + def _extract_repository_id(self, path): return Repository(self.repository_path).id From 765999195d95d7f86db1ed519aa8ac8b9d09240f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Oct 2015 22:34:08 +0100 Subject: [PATCH 2/4] atime unit test: hardcode "round" timestamps just to avoid rounding / precision issues with floating point computations on py < 3.3 I used 2 hardcoded "full second" values on the input file and check if they get restored correctly. --- borg/testsuite/archiver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d8a91c220..b476c77d3 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -288,14 +288,16 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_atime(self): have_root = self.create_test_files() + atime, mtime = 123456780, 234567890 + os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') with changedir('output'): self.cmd('extract', self.repository_location + '::test') sti = os.stat('input/file1') sto = os.stat('output/input/file1') - assert st_mtime_ns(sti) == st_mtime_ns(sto) - assert st_atime_ns(sti) == st_atime_ns(sto) + assert st_mtime_ns(sti) == st_mtime_ns(sto) == mtime * 1e9 + assert st_atime_ns(sti) == st_atime_ns(sto) == atime * 1e9 def _extract_repository_id(self, path): return Repository(self.repository_path).id From 3e73998710e3073b2280af74da44004d34b78254 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Oct 2015 23:08:37 +0100 Subject: [PATCH 3/4] atime unit test: do not compare input file's current atime and explain why (missing O_NOATIME open mode modified the atime at backup time). --- borg/testsuite/archiver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index b476c77d3..4d5f96146 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -297,7 +297,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): sti = os.stat('input/file1') sto = os.stat('output/input/file1') assert st_mtime_ns(sti) == st_mtime_ns(sto) == mtime * 1e9 - assert st_atime_ns(sti) == st_atime_ns(sto) == atime * 1e9 + # to support testing on platforms without O_NOATIME open mode, we just + # compare the restored file with the original atime (not with the + # atime it has now). + assert st_atime_ns(sto) == atime * 1e9 def _extract_repository_id(self, path): return Repository(self.repository_path).id From 92d31be087a654b81a6a74416b8887398882648f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Oct 2015 23:16:41 +0100 Subject: [PATCH 4/4] atime unit test: test adapts to O_NOATIME support on platform --- borg/testsuite/archiver.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 4d5f96146..a8a1c8df5 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -297,10 +297,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): sti = os.stat('input/file1') sto = os.stat('output/input/file1') assert st_mtime_ns(sti) == st_mtime_ns(sto) == mtime * 1e9 - # to support testing on platforms without O_NOATIME open mode, we just - # compare the restored file with the original atime (not with the - # atime it has now). - assert st_atime_ns(sto) == atime * 1e9 + if hasattr(os, 'O_NOATIME'): + assert st_atime_ns(sti) == st_atime_ns(sto) == atime * 1e9 + else: + # it touched the input file's atime while backing it up + assert st_atime_ns(sto) == atime * 1e9 def _extract_repository_id(self, path): return Repository(self.repository_path).id