Fix ChunkerFixed sparse handling and update tests

- Update fixed_test.py expectations for non-sparse chunking.
- Enable `sparse=True` in interaction_test.py and reader_test.py where zero detection is required.
- Catch `ValueError` in _build_fmap to support `BytesIO` seeking.
This commit is contained in:
Thomas Waldmann 2025-11-22 03:58:35 +01:00
parent ef014a6b98
commit fa376b0d9d
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
4 changed files with 41 additions and 5 deletions

View file

@ -137,7 +137,7 @@ class FileFMAPReader:
if self.try_sparse:
try:
fmap = list(sparsemap(self.fd, self.fh))
except OSError as err:
except (OSError, ValueError) as err:
# seeking did not work
pass
@ -170,6 +170,9 @@ class FileFMAPReader:
# read block from the range
data = dread(offset, wanted, self.fd, self.fh)
got = len(data)
# Detect zero-filled blocks regardless of sparse mode.
# Zero detection is important to avoid reading/storing allocated zeros
# even when we are not using sparse file handling based on SEEK_HOLE/SEEK_DATA.
if zeros.startswith(data):
data = None
allocation = CH_ALLOC

View file

@ -40,7 +40,40 @@ def test_chunkify_sparse(tmpdir, fname, sparse_map, header_size, sparse):
fn = str(tmpdir / fname)
make_sparsefile(fn, sparse_map, header_size=header_size)
get_chunks(fn, sparse=sparse, header_size=header_size) == make_content(sparse_map, header_size=header_size)
expected_content = make_content(sparse_map, header_size=header_size)
# ChunkerFixed splits everything into fixed-size chunks (except maybe the header)
# We need to split the expected content similarly.
expected = []
# Handle header if present (it's the first item if header_size > 0)
if header_size > 0:
header = expected_content.pop(0)
expected.append(header)
# Flatten the rest and split into 4096 chunks
current_chunk_size = 4096
for item in expected_content:
if isinstance(item, int):
# Hole
count = item
while count > 0:
size = min(count, current_chunk_size)
expected.append(size)
count -= size
else:
# Data
data = item
while len(data) > 0:
size = min(len(data), current_chunk_size)
expected.append(data[:size])
data = data[size:]
if not sparse:
# if the chunker is not sparse-aware, it will read holes as zeros
expected = [b"\0" * x if isinstance(x, int) else x for x in expected]
assert get_chunks(fn, sparse=sparse, header_size=header_size) == expected
@pytest.mark.skipif("BORG_TESTS_SLOW" not in os.environ, reason="slow tests not enabled, use BORG_TESTS_SLOW=1")

View file

@ -29,7 +29,7 @@ def test_reader_chunker_interaction(chunker_params):
random_data = os.urandom(data_size // 3) + b"\0" * (data_size // 3) + os.urandom(data_size // 3)
# Chunk the data
chunker = get_chunker(*chunker_params)
chunker = get_chunker(*chunker_params, sparse=True)
data_file = BytesIO(random_data)
chunks = list(chunker.chunkify(data_file))

View file

@ -170,7 +170,7 @@ def test_filereader_read_with_mock(mock_chunks, read_size, expected_data, expect
)
def test_filefmapreader_basic(file_content, read_size, expected_chunks):
"""Test basic functionality of FileFMAPReader with different file contents."""
reader = FileFMAPReader(fd=BytesIO(file_content), fh=-1, read_size=read_size, sparse=False, fmap=None)
reader = FileFMAPReader(fd=BytesIO(file_content), fh=-1, read_size=read_size, sparse=True, fmap=None)
# Collect all chunks from blockify
chunks = list(reader.blockify())
@ -252,7 +252,7 @@ def test_filefmapreader_allocation_types(zeros_length, read_size, expected_alloc
# Create a file with all zeros
file_content = b"\0" * zeros_length
reader = FileFMAPReader(fd=BytesIO(file_content), fh=-1, read_size=read_size, sparse=False, fmap=None)
reader = FileFMAPReader(fd=BytesIO(file_content), fh=-1, read_size=read_size, sparse=True, fmap=None)
# Collect all chunks from blockify
chunks = list(reader.blockify())