diff --git a/src/borg/archive.py b/src/borg/archive.py index d65f2abfc..342f02bfb 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -493,7 +493,7 @@ Utilization of max. archive size: {csize_max:.0%} pass self.manifest.archives[name] = (self.id, metadata.time) self.manifest.write() - self.repository.commit() + self.repository.commit(compact=False) self.cache.commit() def calc_stats(self, cache): @@ -1722,9 +1722,8 @@ class ArchiveChecker: if self.repair: logger.info('Writing Manifest.') self.manifest.write() - logger.info('Committing repo (may take a while, due to compact_segments)...') - self.repository.commit(save_space=save_space) - logger.info('Finished committing repo.') + logger.info('Committing repo.') + self.repository.commit(compact=False, save_space=save_space) class ArchiveRecreater: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 532338da7..000bb993c 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -243,7 +243,7 @@ class Archiver: manifest = Manifest(key, repository) manifest.key = key manifest.write() - repository.commit() + repository.commit(compact=False) with Cache(repository, key, manifest, warn_if_unencrypted=False): pass if key.tam_required: @@ -1012,7 +1012,7 @@ class Archiver: name = replace_placeholders(args.name) archive.rename(name) manifest.write() - repository.commit() + repository.commit(compact=False) cache.commit() return self.exit_code @@ -1062,7 +1062,7 @@ class Archiver: elif deleted: manifest.write() # note: might crash in compact() after committing the repo - repository.commit() + repository.commit(compact=False) logger.info('Done. Run "borg check --repair" to clean up the mess.') else: logger.warning('Aborted.') @@ -1078,7 +1078,7 @@ class Archiver: stats, progress=args.progress, forced=args.forced) if not dry_run: manifest.write() - repository.commit(save_space=args.save_space) + repository.commit(compact=False, save_space=args.save_space) cache.commit() if args.stats: log_multi(DASHES, @@ -1387,7 +1387,7 @@ class Archiver: pi.finish() if to_delete and not args.dry_run: manifest.write() - repository.commit(save_space=args.save_space) + repository.commit(compact=False, save_space=args.save_space) cache.commit() if args.stats: log_multi(DASHES, @@ -1414,7 +1414,7 @@ class Archiver: print(format_archive(archive_info), '[%s]' % bin_to_hex(archive_info.id)) manifest.config[b'tam_required'] = True manifest.write() - repository.commit() + repository.commit(compact=False) if not key.tam_required: key.tam_required = True key.change_passphrase(key._passphrase) @@ -1437,7 +1437,7 @@ class Archiver: print('Key location:', key.find_key()) manifest.config[b'tam_required'] = False manifest.write() - repository.commit() + repository.commit(compact=False) else: # mainly for upgrades from Attic repositories, # but also supports borg 0.xx -> 1.0 upgrade. @@ -1500,7 +1500,7 @@ class Archiver: logger.info('Skipped archive %s: Nothing to do. Archive was not processed.', name) if not args.dry_run: manifest.write() - repository.commit() + repository.commit(compact=False) cache.commit() return self.exit_code @@ -1532,7 +1532,16 @@ class Archiver: # that would be bad if somebody uses rsync with ignore-existing (or # any other mechanism relying on existing segment data not changing). # see issue #1867. - repository.commit() + repository.commit(compact=False) + + @with_repository(manifest=False, exclusive=True) + def do_compact(self, args, repository): + """compact segment files in the repository""" + # see the comment in do_with_lock about why we do it like this: + data = repository.get(Manifest.MANIFEST_ID) + repository.put(Manifest.MANIFEST_ID, data) + repository.commit(compact=True) + return EXIT_SUCCESS @with_repository(exclusive=True, manifest=False) def do_config(self, args, repository): @@ -1788,7 +1797,7 @@ class Archiver: h = hashlib.sha256(data) # XXX hardcoded repository.put(h.digest(), data) print("object %s put." % h.hexdigest()) - repository.commit() + repository.commit(compact=False) return EXIT_SUCCESS @with_repository(manifest=False, exclusive=True) @@ -1808,7 +1817,7 @@ class Archiver: except Repository.ObjectNotFound: print("object %s not found." % hex_id) if modified: - repository.commit() + repository.commit(compact=False) print('Done.') return EXIT_SUCCESS @@ -3686,6 +3695,19 @@ class Archiver: subparser.add_argument('args', metavar='ARGS', nargs=argparse.REMAINDER, help='command arguments') + compact_epilog = process_epilog(""" + This command frees repository space by compacting segments. + """) + subparser = subparsers.add_parser('compact', parents=[common_parser], add_help=False, + description=self.do_compact.__doc__, + epilog=compact_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='compact segment files / free space in repo') + subparser.set_defaults(func=self.do_compact) + subparser.add_argument('location', metavar='REPOSITORY', + type=location_validator(archive=False), + help='repository to compact') + config_epilog = process_epilog(""" This command gets and sets options in a local repository or cache config file. For security reasons, this command only works on local repositories. diff --git a/src/borg/remote.py b/src/borg/remote.py index 622b5ad12..364c42f1f 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -889,8 +889,9 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. def check(self, repair=False, save_space=False): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) - def commit(self, save_space=False): + @api(since=parse_version('1.0.0'), + compact={'since': parse_version('1.2.0a0'), 'previously': True}) + def commit(self, save_space=False, compact=True): """actual remoting is done via self.call in the @api decorator""" @api(since=parse_version('1.0.0')) diff --git a/src/borg/repository.py b/src/borg/repository.py index c1ae6312c..daaea9224 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -416,7 +416,7 @@ class Repository: self.lock.release() self.lock = None - def commit(self, save_space=False): + def commit(self, save_space=False, compact=True): """Commit transaction """ # save_space is not used anymore, but stays for RPC/API compatibility. @@ -427,7 +427,7 @@ class Repository: self.check_free_space() self.log_storage_quota() self.io.write_commit() - if not self.append_only: + if compact and not self.append_only: self.compact_segments() self.write_index() self.rollback() @@ -460,7 +460,7 @@ class Repository: raise self.prepare_txn(self.get_transaction_id()) # don't leave an open transaction around - self.commit() + self.commit(compact=False) return self.open_index(self.get_transaction_id()) def prepare_txn(self, transaction_id, do_cleanup=True): @@ -951,7 +951,6 @@ class Repository: if current_index.get(key, (-1, -1)) != value: report_error('Index mismatch for key {}. {} != {}'.format(key, value, current_index.get(key, (-1, -1)))) if repair: - self.compact_segments() self.write_index() self.rollback() if error_found: