diff --git a/docs/borg_theme/css/borg.css b/docs/borg_theme/css/borg.css index fd80000db..3fa45332e 100644 --- a/docs/borg_theme/css/borg.css +++ b/docs/borg_theme/css/borg.css @@ -13,7 +13,6 @@ border-color: #000000; } - .wy-side-nav-search > a { color: rgba(255, 255, 255, 0.5); } diff --git a/src/borg/remote.py b/src/borg/remote.py index cd382ecf2..604506cd0 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -42,6 +42,14 @@ class InvalidRPCMethod(Error): """RPC method {} is not valid""" +class UnexpectedRPCDataFormatFromClient(Error): + """Borg {}: Got unexpected RPC data format from client.""" + + +class UnexpectedRPCDataFormatFromServer(Error): + """Got unexpected RPC data format from server.""" + + class RepositoryServer: # pragma: no cover rpc_methods = ( '__len__', @@ -84,13 +92,18 @@ class RepositoryServer: # pragma: no cover if r: data = os.read(stdin_fd, BUFSIZE) if not data: - self.repository.close() + if self.repository is not None: + self.repository.close() + else: + os.write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n" + .format(__version__).encode()) return unpacker.feed(data) for unpacked in unpacker: if not (isinstance(unpacked, tuple) and len(unpacked) == 4): - self.repository.close() - raise Exception("Unexpected RPC data format.") + if self.repository is not None: + self.repository.close() + raise UnexpectedRPCDataFormatFromClient(__version__) type, msgid, method, args = unpacked method = method.decode('ascii') try: @@ -354,7 +367,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. self.unpacker.feed(data) for unpacked in self.unpacker: if not (isinstance(unpacked, tuple) and len(unpacked) == 4): - raise Exception("Unexpected RPC data format.") + raise UnexpectedRPCDataFormatFromServer() type, msgid, error, res = unpacked if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index fd7eb5fc8..dfa540e01 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -285,8 +285,14 @@ class ArchiverTestCaseBase(BaseTestCase): # Symlink if are_symlinks_supported(): os.symlink('somewhere', os.path.join(self.input_path, 'link1')) - if xattr.is_enabled(self.input_path): - xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar') + self.create_regular_file('fusexattr', size=1) + if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): + # ironically, due to the way how fakeroot works, comparing fuse file xattrs to orig file xattrs + # will FAIL if fakeroot supports xattrs, thus we only set the xattr if XATTR_FAKEROOT is False. + # This is because fakeroot with xattr-support does not propagate xattrs of the underlying file + # into "fakeroot space". Because the xattrs exposed by borgfs are these of an underlying file + # (from fakeroots point of view) they are invisible to the test process inside the fakeroot. + xattr.setxattr(os.path.join(self.input_path, 'fusexattr'), 'user.foo', b'bar') # XXX this always fails for me # ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot # same for newer ubuntu and centos. @@ -362,7 +368,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in(name, list_output) self.assert_dirs_equal('input', 'output/input') info_output = self.cmd('info', self.repository_location + '::test') - item_count = 3 if has_lchflags else 4 # one file is UF_NODUMP + item_count = 4 if has_lchflags else 5 # one file is UF_NODUMP self.assert_in('Number of files: %d' % item_count, info_output) shutil.rmtree(self.cache_path) with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): @@ -1399,7 +1405,9 @@ class ArchiverTestCase(ArchiverTestCaseBase): with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f: assert in_f.read() == out_f.read() # list/read xattrs - if xattr.is_enabled(self.input_path): + in_fn = 'input/fusexattr' + out_fn = os.path.join(mountpoint, 'input', 'fusexattr') + if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): assert xattr.listxattr(out_fn) == ['user.foo', ] assert xattr.getxattr(out_fn, 'user.foo') == b'bar' else: diff --git a/src/borg/testsuite/hashindex.py b/src/borg/testsuite/hashindex.py index a7d757145..5ddb85171 100644 --- a/src/borg/testsuite/hashindex.py +++ b/src/borg/testsuite/hashindex.py @@ -24,19 +24,19 @@ class HashIndexTestCase(BaseTestCase): self.assert_equal(len(idx), 0) # Test set for x in range(100): - idx[bytes('%-32d' % x, 'ascii')] = make_value(x) + idx[H(x)] = make_value(x) self.assert_equal(len(idx), 100) for x in range(100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x)) + self.assert_equal(idx[H(x)], make_value(x)) # Test update for x in range(100): - idx[bytes('%-32d' % x, 'ascii')] = make_value(x * 2) + idx[H(x)] = make_value(x * 2) self.assert_equal(len(idx), 100) for x in range(100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) + self.assert_equal(idx[H(x)], make_value(x * 2)) # Test delete for x in range(50): - del idx[bytes('%-32d' % x, 'ascii')] + del idx[H(x)] self.assert_equal(len(idx), 50) idx_name = tempfile.NamedTemporaryFile() idx.write(idx_name.name) @@ -48,7 +48,7 @@ class HashIndexTestCase(BaseTestCase): idx = cls.read(idx_name.name) self.assert_equal(len(idx), 50) for x in range(50, 100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) + self.assert_equal(idx[H(x)], make_value(x * 2)) idx.clear() self.assert_equal(len(idx), 0) idx.write(idx_name.name) @@ -57,11 +57,11 @@ class HashIndexTestCase(BaseTestCase): def test_nsindex(self): self._generic_test(NSIndex, lambda x: (x, x), - '80fba5b40f8cf12f1486f1ba33c9d852fb2b41a5b5961d3b9d1228cf2aa9c4c9') + 'b96ec1ddabb4278cc92261ee171f7efc979dc19397cc5e89b778f05fa25bf93f') def test_chunkindex(self): self._generic_test(ChunkIndex, lambda x: (x, x, x), - '1d71865e72e3c3af18d3c7216b6fa7b014695eaa3ed7f14cf9cd02fba75d1c95') + '9d437a1e145beccc790c69e66ba94fc17bd982d83a401c9c6e524609405529d8') def test_resize(self): n = 2000 # Must be >= MIN_BUCKETS @@ -71,11 +71,11 @@ class HashIndexTestCase(BaseTestCase): initial_size = os.path.getsize(idx_name.name) self.assert_equal(len(idx), 0) for x in range(n): - idx[bytes('%-32d' % x, 'ascii')] = x, x + idx[H(x)] = x, x idx.write(idx_name.name) self.assert_true(initial_size < os.path.getsize(idx_name.name)) for x in range(n): - del idx[bytes('%-32d' % x, 'ascii')] + del idx[H(x)] self.assert_equal(len(idx), 0) idx.write(idx_name.name) self.assert_equal(initial_size, os.path.getsize(idx_name.name)) @@ -83,7 +83,7 @@ class HashIndexTestCase(BaseTestCase): def test_iteritems(self): idx = NSIndex() for x in range(100): - idx[bytes('%-0.32d' % x, 'ascii')] = x, x + idx[H(x)] = x, x all = list(idx.iteritems()) self.assert_equal(len(all), 100) second_half = list(idx.iteritems(marker=all[49][0])) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 67c250e8d..620bcaf3e 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -45,21 +45,21 @@ class RepositoryTestCaseBase(BaseTestCase): self.repository = self.open(exclusive=exclusive) def add_keys(self): - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'bar') - self.repository.put(b'00000000000000000000000000000003', b'bar') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'bar') + self.repository.put(H(3), b'bar') self.repository.commit() - self.repository.put(b'00000000000000000000000000000001', b'bar2') - self.repository.put(b'00000000000000000000000000000002', b'boo') - self.repository.delete(b'00000000000000000000000000000003') + self.repository.put(H(1), b'bar2') + self.repository.put(H(2), b'boo') + self.repository.delete(H(3)) class RepositoryTestCase(RepositoryTestCaseBase): def test1(self): for x in range(100): - self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') - key50 = ('%-32d' % 50).encode('ascii') + self.repository.put(H(x), b'SOMEDATA') + key50 = H(50) self.assert_equal(self.repository.get(key50), b'SOMEDATA') self.repository.delete(key50) self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50)) @@ -70,69 +70,69 @@ class RepositoryTestCase(RepositoryTestCaseBase): for x in range(100): if x == 50: continue - self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA') + self.assert_equal(repository2.get(H(x)), b'SOMEDATA') def test2(self): """Test multiple sequential transactions """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'foo') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'foo') self.repository.commit() - self.repository.delete(b'00000000000000000000000000000000') - self.repository.put(b'00000000000000000000000000000001', b'bar') + self.repository.delete(H(0)) + self.repository.put(H(1), b'bar') self.repository.commit() - self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar') + self.assert_equal(self.repository.get(H(1)), b'bar') def test_consistency(self): """Test cache consistency """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') - self.repository.put(b'00000000000000000000000000000000', b'foo2') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') - self.repository.put(b'00000000000000000000000000000000', b'bar') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar') - self.repository.delete(b'00000000000000000000000000000000') - self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(b'00000000000000000000000000000000')) + self.repository.put(H(0), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') + self.repository.put(H(0), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') + self.repository.put(H(0), b'bar') + self.assert_equal(self.repository.get(H(0)), b'bar') + self.repository.delete(H(0)) + self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0))) def test_consistency2(self): """Test cache consistency2 """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') + self.repository.put(H(0), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') self.repository.commit() - self.repository.put(b'00000000000000000000000000000000', b'foo2') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') + self.repository.put(H(0), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') self.repository.rollback() - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') def test_overwrite_in_same_transaction(self): """Test cache consistency2 """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000000', b'foo2') + self.repository.put(H(0), b'foo') + self.repository.put(H(0), b'foo2') self.repository.commit() - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') def test_single_kind_transactions(self): # put - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() self.repository.close() # replace self.repository = self.open() with self.repository: - self.repository.put(b'00000000000000000000000000000000', b'bar') + self.repository.put(H(0), b'bar') self.repository.commit() # delete self.repository = self.open() with self.repository: - self.repository.delete(b'00000000000000000000000000000000') + self.repository.delete(H(0)) self.repository.commit() def test_list(self): for x in range(100): - self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') + self.repository.put(H(x), b'SOMEDATA') all = self.repository.list() self.assert_equal(len(all), 100) first_half = self.repository.list(limit=50) @@ -145,10 +145,10 @@ class RepositoryTestCase(RepositoryTestCaseBase): def test_max_data_size(self): max_data = b'x' * MAX_DATA_SIZE - self.repository.put(b'00000000000000000000000000000000', max_data) - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), max_data) + self.repository.put(H(0), max_data) + self.assert_equal(self.repository.get(H(0)), max_data) self.assert_raises(IntegrityError, - lambda: self.repository.put(b'00000000000000000000000000000001', max_data + b'x')) + lambda: self.repository.put(H(1), max_data + b'x')) class LocalRepositoryTestCase(RepositoryTestCaseBase): @@ -163,22 +163,22 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase): assert self.repository.compact[0] == 41 + 9 def test_sparse1(self): - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'123456789') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'123456789') self.repository.commit() - self.repository.put(b'00000000000000000000000000000001', b'bar') + self.repository.put(H(1), b'bar') self._assert_sparse() def test_sparse2(self): - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'123456789') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'123456789') self.repository.commit() - self.repository.delete(b'00000000000000000000000000000001') + self.repository.delete(H(1)) self._assert_sparse() def test_sparse_delete(self): - self.repository.put(b'00000000000000000000000000000000', b'1245') - self.repository.delete(b'00000000000000000000000000000000') + self.repository.put(H(0), b'1245') + self.repository.delete(H(0)) self.repository.io._write_fd.sync() # The on-line tracking works on a per-object basis... @@ -267,7 +267,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): self.assert_equal(len(self.repository), 3) def test_ignores_commit_tag_in_data(self): - self.repository.put(b'0' * 32, LoggedIO.COMMIT) + self.repository.put(H(0), LoggedIO.COMMIT) self.reopen() with self.repository: io = self.repository.io @@ -359,19 +359,19 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase): def test_append_only(self): def segments_in_repository(): return len(list(self.repository.io.segment_iterator())) - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() self.repository.append_only = False assert segments_in_repository() == 2 - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() # normal: compact squashes the data together, only one segment assert segments_in_repository() == 4 self.repository.append_only = True assert segments_in_repository() == 4 - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() # append only: does not compact, only new segments written assert segments_in_repository() == 6 @@ -385,7 +385,7 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase): self.reopen() with self.repository: - self.repository.put(b'00000000000000000000000000000000', b'foobar') + self.repository.put(H(0), b'foobar') with pytest.raises(Repository.InsufficientFreeSpaceError): self.repository.commit() @@ -393,13 +393,13 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase): class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase): def setUp(self): super().setUp() - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() self.repository.close() def do_commit(self): with self.repository: - self.repository.put(b'00000000000000000000000000000000', b'fox') + self.repository.put(H(0), b'fox') self.repository.commit() def test_corrupted_hints(self): @@ -453,12 +453,12 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): def get_objects(self, *ids): for id_ in ids: - self.repository.get(('%032d' % id_).encode('ascii')) + self.repository.get(H(id_)) def add_objects(self, segments): for ids in segments: for id_ in ids: - self.repository.put(('%032d' % id_).encode('ascii'), b'data') + self.repository.put(H(id_), b'data') self.repository.commit() def get_head(self): @@ -469,7 +469,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): def corrupt_object(self, id_): idx = self.open_index() - segment, offset = idx[('%032d' % id_).encode('ascii')] + segment, offset = idx[H(id_)] with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd: fd.seek(offset) fd.write(b'BOOM') @@ -560,8 +560,8 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects()) def test_crash_before_compact(self): - self.repository.put(bytes(32), b'data') - self.repository.put(bytes(32), b'data2') + self.repository.put(H(0), b'data') + self.repository.put(H(0), b'data2') # Simulate a crash before compact with patch.object(Repository, 'compact_segments') as compact: self.repository.commit() @@ -569,7 +569,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.reopen() with self.repository: self.check(repair=True) - self.assert_equal(self.repository.get(bytes(32)), b'data2') + self.assert_equal(self.repository.get(H(0)), b'data2') class RemoteRepositoryTestCase(RepositoryTestCase):