upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-03-25 03:45:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #1449 from enkore/f/extractprogress

Browse source 
extract --progress
This commit is contained in:
TW 2016-08-21 20:24:44 +02:00 committed by GitHub
commit 2f65f7bdec
9 changed files with 74 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/borg/archive.py
View file

@ -422,7 +422,7 @@ Number of files: {0.stats.nfiles}'''.format(
return stats
def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sparse=False,
hardlink_masters=None, original_path=None):
hardlink_masters=None, original_path=None, pi=None):
"""
Extract archive item.
@ -433,11 +433,14 @@ Number of files: {0.stats.nfiles}'''.format(
:param sparse: write sparse files (chunk-granularity, independent of the original being sparse)
:param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly
:param original_path: 'path' key as stored in archive
:param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes)
"""
has_damaged_chunks = 'chunks_healthy' in item
if dry_run or stdout:
if 'chunks' in item:
for _, data in self.pipeline.fetch_many([c.id for c in item.chunks], is_preloaded=True):
if pi:
pi.show(increase=len(data))
if stdout:
sys.stdout.buffer.write(data)
if stdout:
@ -489,6 +492,8 @@ Number of files: {0.stats.nfiles}'''.format(
with fd:
ids = [c.id for c in item.chunks]
for _, data in self.pipeline.fetch_many(ids, is_preloaded=True):
if pi:
pi.show(increase=len(data))
with backup_io():
if sparse and self.zeros.startswith(data):
# all-zero chunk: create a hole in a sparse file
@ -635,7 +640,7 @@ Number of files: {0.stats.nfiles}'''.format(
try:
unpacker = msgpack.Unpacker(use_list=False)
items_ids = self.metadata.items
pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True)
pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%")
for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))):
if progress:
pi.show(i)
@ -1028,7 +1033,7 @@ class ArchiveChecker:
logger.info('Starting cryptographic data integrity verification...')
count = len(self.chunks)
errors = 0
pi = ProgressIndicatorPercent(total=count, msg="Verifying data %6.2f%%", step=0.01, same_line=True)
pi = ProgressIndicatorPercent(total=count, msg="Verifying data %6.2f%%", step=0.01)
for chunk_id, (refcount, *_) in self.chunks.iteritems():
pi.show()
try:

19
src/borg/archiver.py
View file

@ -41,6 +41,7 @@ from .helpers import log_multi
from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
from .helpers import signal_handler
from .helpers import ErrorIgnoringTextIOWrapper
from .helpers import ProgressIndicatorPercent
from .item import Item
from .key import key_creator, RepoKey, PassphraseKey
from .platform import get_flags
@ -439,6 +440,7 @@ class Archiver:
matcher, include_patterns = self.build_matcher(args.excludes, args.paths)
progress = args.progress
output_list = args.output_list
dry_run = args.dry_run
stdout = args.stdout
@ -453,6 +455,14 @@ class Archiver:
item.get('hardlink_master', True) and 'source' not in item)
filter = self.build_filter(matcher, item_is_hardlink_master, strip_components)
if progress:
progress_logger = logging.getLogger(ProgressIndicatorPercent.LOGGER)
progress_logger.info('Calculating size')
extracted_size = sum(item.file_size() for item in archive.iter_items(filter))
pi = ProgressIndicatorPercent(total=extracted_size, msg='Extracting files %5.1f%%', step=0.1)
else:
pi = None
for item in archive.iter_items(filter, preload=True):
orig_path = item.path
if item_is_hardlink_master(item):
@ -472,19 +482,21 @@ class Archiver:
logging.getLogger('borg.output.list').info(remove_surrogates(orig_path))
try:
if dry_run:
archive.extract_item(item, dry_run=True)
archive.extract_item(item, dry_run=True, pi=pi)
else:
if stat.S_ISDIR(item.mode):
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
else:
archive.extract_item(item, stdout=stdout, sparse=sparse, hardlink_masters=hardlink_masters,
original_path=orig_path)
original_path=orig_path, pi=pi)
except BackupOSError as e:
self.print_warning('%s: %s', remove_surrogates(orig_path), e)
if not args.dry_run:
pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%')
while dirs:
pi.show()
dir_item = dirs.pop(-1)
try:
archive.extract_item(dir_item)
@ -1641,6 +1653,9 @@ class Archiver:
formatter_class=argparse.RawDescriptionHelpFormatter,
help='extract archive contents')
subparser.set_defaults(func=self.do_extract)
subparser.add_argument('-p', '--progress', dest='progress',
action='store_true', default=False,
help='show progress while extracting (may be slower)')
subparser.add_argument('--list', dest='output_list',
action='store_true', default=False,
help='output verbose list of items (files, dirs, ...)')

21
src/borg/helpers.py
View file

@ -1105,14 +1105,15 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
class ProgressIndicatorPercent:
def __init__(self, total, step=5, start=0, same_line=False, msg="%3.0f%%"):
LOGGER = 'borg.output.progress'
def __init__(self, total, step=5, start=0, msg="%3.0f%%"):
"""
Percentage-based progress indicator
:param total: total amount of items
:param step: step size in percent
:param start: at which percent value to start
:param same_line: if True, emit output always on same line
:param msg: output message, must contain one %f placeholder for the percentage
"""
self.counter = 0 # 0 .. (total-1)
@ -1120,9 +1121,8 @@ class ProgressIndicatorPercent:
self.trigger_at = start # output next percentage value when reaching (at least) this
self.step = step
self.msg = msg
self.same_line = same_line
self.handler = None
self.logger = logging.getLogger('borg.output.progress')
self.logger = logging.getLogger(self.LOGGER)
# If there are no handlers, set one up explicitly because the
# terminator and propagation needs to be set. If there are,
@ -1130,7 +1130,7 @@ class ProgressIndicatorPercent:
if not self.logger.handlers:
self.handler = logging.StreamHandler(stream=sys.stderr)
self.handler.setLevel(logging.INFO)
self.handler.terminator = '\r' if self.same_line else '\n'
self.handler.terminator = '\r'
self.logger.addHandler(self.handler)
if self.logger.level == logging.NOTSET:
@ -1142,17 +1142,17 @@ class ProgressIndicatorPercent:
self.logger.removeHandler(self.handler)
self.handler.close()
def progress(self, current=None):
def progress(self, current=None, increase=1):
if current is not None:
self.counter = current
pct = self.counter * 100 / self.total
self.counter += 1
self.counter += increase
if pct >= self.trigger_at:
self.trigger_at += self.step
return pct
def show(self, current=None):
pct = self.progress(current)
def show(self, current=None, increase=1):
pct = self.progress(current, increase)
if pct is not None:
return self.output(pct)
@ -1160,8 +1160,7 @@ class ProgressIndicatorPercent:
self.logger.info(self.msg % percent)
def finish(self):
if self.same_line:
self.logger.info(" " * len(self.msg % 100.0))
self.logger.info(" " * len(self.msg % 100.0))
class ProgressIndicatorEndless:

5
src/borg/item.py
View file

@ -157,6 +157,11 @@ class Item(PropDict):
part = PropDict._make_property('part', int)
def file_size(self):
if 'chunks' not in self:
return 0
return sum(chunk.size for chunk in self.chunks)
class EncryptedKey(PropDict):
"""

4
src/borg/repository.py
View file

@ -536,7 +536,7 @@ class Repository:
self.prepare_txn(index_transaction_id, do_cleanup=False)
try:
segment_count = sum(1 for _ in self.io.segment_iterator())
pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%", same_line=True)
pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%")
for i, (segment, filename) in enumerate(self.io.segment_iterator()):
pi.show(i)
if index_transaction_id is not None and segment <= index_transaction_id:
@ -636,7 +636,7 @@ class Repository:
segments_transaction_id = self.io.get_segments_transaction_id()
self.prepare_txn(None) # self.index, self.compact, self.segments all empty now!
segment_count = sum(1 for _ in self.io.segment_iterator())
pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.1f%%", step=0.1, same_line=True)
pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.1f%%", step=0.1)
for i, (segment, filename) in enumerate(self.io.segment_iterator()):
pi.show(i)
if segment > transaction_id:

9
src/borg/testsuite/archiver.py
View file

@ -751,6 +751,15 @@ class ArchiverTestCase(ArchiverTestCaseBase):
output = self.cmd('extract', '--list', '--info', self.repository_location + '::test')
self.assert_in("input/file", output)
def test_extract_progress(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file', size=1024 * 80)
self.cmd('create', self.repository_location + '::test', 'input')
with changedir('output'):
output = self.cmd('extract', self.repository_location + '::test', '--progress')
assert 'Extracting files' in output
def _create_test_caches(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)

31
src/borg/testsuite/helpers.py
View file

@ -870,24 +870,9 @@ def test_yes_env_output(capfd, monkeypatch):
assert 'yes' in err
def test_progress_percentage_multiline(capfd):
pi = ProgressIndicatorPercent(1000, step=5, start=0, same_line=False, msg="%3.0f%%")
pi.show(0)
out, err = capfd.readouterr()
assert err == ' 0%\n'
pi.show(420)
out, err = capfd.readouterr()
assert err == ' 42%\n'
pi.show(1000)
out, err = capfd.readouterr()
assert err == '100%\n'
pi.finish()
out, err = capfd.readouterr()
assert err == ''
def test_progress_percentage_sameline(capfd):
pi = ProgressIndicatorPercent(1000, step=5, start=0, same_line=True, msg="%3.0f%%")
pi = ProgressIndicatorPercent(1000, step=5, start=0, msg="%3.0f%%")
pi.logger.setLevel('INFO')
pi.show(0)
out, err = capfd.readouterr()
assert err == ' 0%\r'
@ -904,22 +889,22 @@ def test_progress_percentage_sameline(capfd):
def test_progress_percentage_step(capfd):
pi = ProgressIndicatorPercent(100, step=2, start=0, same_line=False, msg="%3.0f%%")
pi = ProgressIndicatorPercent(100, step=2, start=0, msg="%3.0f%%")
pi.logger.setLevel('INFO')
pi.show()
out, err = capfd.readouterr()
assert err == ' 0%\n'
assert err == ' 0%\r'
pi.show()
out, err = capfd.readouterr()
assert err == '' # no output at 1% as we have step == 2
pi.show()
out, err = capfd.readouterr()
assert err == ' 2%\n'
assert err == ' 2%\r'
def test_progress_percentage_quiet(capfd):
logging.getLogger('borg.output.progress').setLevel(logging.WARN)
pi = ProgressIndicatorPercent(1000, step=5, start=0, same_line=False, msg="%3.0f%%")
pi = ProgressIndicatorPercent(1000, step=5, start=0, msg="%3.0f%%")
pi.logger.setLevel('WARN')
pi.show(0)
out, err = capfd.readouterr()
assert err == ''

14
src/borg/testsuite/item.py
View file

@ -1,5 +1,6 @@
import pytest
from ..cache import ChunkListEntry
from ..item import Item
from ..helpers import StableDict
@ -145,3 +146,16 @@ def test_unknown_property():
item = Item()
with pytest.raises(AttributeError):
item.unknown_attribute = None
def test_item_file_size():
item = Item(chunks=[
ChunkListEntry(csize=1, size=1000, id=None),
ChunkListEntry(csize=1, size=2000, id=None),
])
assert item.file_size() == 3000
def test_item_file_size_no_chunks():
item = Item()
assert item.file_size() == 0

2
src/borg/upgrader.py
View file

@ -77,7 +77,7 @@ class AtticRepositoryUpgrader(Repository):
replace the 8 first bytes of all regular files in there."""
logger.info("converting %d segments..." % len(segments))
segment_count = len(segments)
pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", same_line=True)
pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%")
for i, filename in enumerate(segments):
if progress:
pi.show(i)