From 2209f56cd53dcb47e8056c20e25968f1e402fa94 Mon Sep 17 00:00:00 2001
From: Thalian <39634976+fantasya-pbem@users.noreply.github.com>
Date: Thu, 24 Oct 2019 10:12:58 +0200
Subject: [PATCH] Feature/4674 compact threshold (#4798)

compact: add --threshold option, fixes #4674
---
 src/borg/archiver.py             |  8 +++++++-
 src/borg/remote.py               |  3 ++-
 src/borg/repository.py           | 10 +++++-----
 src/borg/testsuite/repository.py |  2 +-
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index 2dd730783..176d6afbc 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1616,7 +1616,8 @@ class Archiver:
         # see the comment in do_with_lock about why we do it like this:
         data = repository.get(Manifest.MANIFEST_ID)
         repository.put(Manifest.MANIFEST_ID, data)
-        repository.commit(compact=True, cleanup_commits=args.cleanup_commits)
+        threshold = args.threshold / 100
+        repository.commit(compact=True, threshold=threshold, cleanup_commits=args.cleanup_commits)
         return EXIT_SUCCESS
 
     @with_repository(exclusive=True, manifest=False)
@@ -2875,6 +2876,8 @@ class Archiver:
         Depending on the amount of segments that need compaction, it may take a while,
         so consider using the ``--progress`` option.
 
+        A segment is compacted if the amount of saved space is above the percentage value
+        given by the ``--threshold`` option. If ommitted, a threshold of 10% is used.
         When using ``--verbose``, borg will output an estimate of the freed space.
 
         After upgrading borg (server) to 1.2+, you can use ``borg compact --cleanup-commits``
@@ -2894,6 +2897,9 @@ class Archiver:
                                help='repository to compact')
         subparser.add_argument('--cleanup-commits', dest='cleanup_commits', action='store_true',
                                help='cleanup commit-only 17-byte segment files')
+        subparser.add_argument('--threshold', metavar='PERCENT', dest='threshold',
+                               type=int, default=10,
+                               help='set minimum threshold for saved space in PERCENT (Default: 10)')
 
         # borg config
         config_epilog = process_epilog("""
diff --git a/src/borg/remote.py b/src/borg/remote.py
index dce028801..ecf438367 100644
--- a/src/borg/remote.py
+++ b/src/borg/remote.py
@@ -900,8 +900,9 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
 
     @api(since=parse_version('1.0.0'),
          compact={'since': parse_version('1.2.0a0'), 'previously': True, 'dontcare': True},
+         threshold={'since': parse_version('1.2.0a8'), 'previously': 0.1, 'dontcare': True},
          cleanup_commits={'since': parse_version('1.2.0a0'), 'previously': False, 'dontcare': True})
-    def commit(self, save_space=False, compact=True, cleanup_commits=False):
+    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
         """actual remoting is done via self.call in the @api decorator"""
 
     @api(since=parse_version('1.0.0'))
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 740af95cb..57b43e6fe 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -442,7 +442,7 @@ class Repository:
             self.lock.release()
             self.lock = None
 
-    def commit(self, save_space=False, compact=True, cleanup_commits=False):
+    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
         """Commit transaction
         """
         # save_space is not used anymore, but stays for RPC/API compatibility.
@@ -463,7 +463,7 @@ class Repository:
                     if os.path.getsize(filename) == 17:
                         self.segments[segment] = 0
                         self.compact[segment] = LoggedIO.header_fmt.size
-            self.compact_segments()
+            self.compact_segments(threshold)
         self.write_index()
         self.rollback()
 
@@ -695,7 +695,7 @@ class Repository:
             logger.info('Storage quota: %s out of %s used.',
                         format_file_size(self.storage_quota_use), format_file_size(self.storage_quota))
 
-    def compact_segments(self):
+    def compact_segments(self, threshold):
         """Compact sparse segments by copying data into new segments
         """
         if not self.compact:
@@ -723,7 +723,7 @@ class Repository:
                 del self.compact[segment]
             unused = []
 
-        logger.debug('compaction started.')
+        logger.debug('Compaction started (threshold is %i%%).', threshold * 100)
         pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1,
                                       msgid='repository.compact_segments')
         for segment, freeable_space in sorted(self.compact.items()):
@@ -736,7 +736,7 @@ class Repository:
             freeable_ratio = 1.0 * freeable_space / segment_size
             # we want to compact if:
             # - we can free a considerable relative amount of space (freeable_ratio over some threshold)
-            if not (freeable_ratio > 0.1):
+            if not (freeable_ratio > threshold):
                 logger.debug('not compacting segment %d (freeable: %2.2f%% [%d bytes])',
                              segment, freeable_ratio * 100.0, freeable_space)
                 pi.show()
diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py
index f93d603df..99939a608 100644
--- a/src/borg/testsuite/repository.py
+++ b/src/borg/testsuite/repository.py
@@ -785,7 +785,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         # Simulate a crash before compact
         with patch.object(Repository, 'compact_segments') as compact:
             self.repository.commit(compact=True)
-            compact.assert_called_once_with()
+            compact.assert_called_once_with(0.1)
         self.reopen()
         with self.repository:
             self.check(repair=True)