Feature/4674 compact threshold (#4798)

compact: add --threshold option, fixes #4674

src/borg/archiver.py

@@ -1616,7 +1616,8 @@ class Archiver:
         # see the comment in do_with_lock about why we do it like this:
         data = repository.get(Manifest.MANIFEST_ID)
         repository.put(Manifest.MANIFEST_ID, data)
-        repository.commit(compact=True, cleanup_commits=args.cleanup_commits)
+        threshold = args.threshold / 100
+        repository.commit(compact=True, threshold=threshold, cleanup_commits=args.cleanup_commits)
         return EXIT_SUCCESS
 
     @with_repository(exclusive=True, manifest=False)
@@ -2875,6 +2876,8 @@ class Archiver:
         Depending on the amount of segments that need compaction, it may take a while,
         so consider using the ``--progress`` option.
 
+        A segment is compacted if the amount of saved space is above the percentage value
+        given by the ``--threshold`` option. If ommitted, a threshold of 10% is used.
         When using ``--verbose``, borg will output an estimate of the freed space.
 
         After upgrading borg (server) to 1.2+, you can use ``borg compact --cleanup-commits``
@@ -2894,6 +2897,9 @@ class Archiver:
                                help='repository to compact')
         subparser.add_argument('--cleanup-commits', dest='cleanup_commits', action='store_true',
                                help='cleanup commit-only 17-byte segment files')
+        subparser.add_argument('--threshold', metavar='PERCENT', dest='threshold',
+                               type=int, default=10,
+                               help='set minimum threshold for saved space in PERCENT (Default: 10)')
 
         # borg config
         config_epilog = process_epilog("""

src/borg/remote.py

@@ -900,8 +900,9 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
 
     @api(since=parse_version('1.0.0'),
          compact={'since': parse_version('1.2.0a0'), 'previously': True, 'dontcare': True},
+         threshold={'since': parse_version('1.2.0a8'), 'previously': 0.1, 'dontcare': True},
          cleanup_commits={'since': parse_version('1.2.0a0'), 'previously': False, 'dontcare': True})
-    def commit(self, save_space=False, compact=True, cleanup_commits=False):
+    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
         """actual remoting is done via self.call in the @api decorator"""
 
     @api(since=parse_version('1.0.0'))

src/borg/repository.py

@@ -442,7 +442,7 @@ class Repository:
             self.lock.release()
             self.lock = None
 
-    def commit(self, save_space=False, compact=True, cleanup_commits=False):
+    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
         """Commit transaction
         """
         # save_space is not used anymore, but stays for RPC/API compatibility.
@@ -463,7 +463,7 @@ class Repository:
                     if os.path.getsize(filename) == 17:
                         self.segments[segment] = 0
                         self.compact[segment] = LoggedIO.header_fmt.size
-            self.compact_segments()
+            self.compact_segments(threshold)
         self.write_index()
         self.rollback()
 
@@ -695,7 +695,7 @@ class Repository:
             logger.info('Storage quota: %s out of %s used.',
                         format_file_size(self.storage_quota_use), format_file_size(self.storage_quota))
 
-    def compact_segments(self):
+    def compact_segments(self, threshold):
         """Compact sparse segments by copying data into new segments
         """
         if not self.compact:
@@ -723,7 +723,7 @@ class Repository:
                 del self.compact[segment]
             unused = []
 
-        logger.debug('compaction started.')
+        logger.debug('Compaction started (threshold is %i%%).', threshold * 100)
         pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1,
                                       msgid='repository.compact_segments')
         for segment, freeable_space in sorted(self.compact.items()):
@@ -736,7 +736,7 @@ class Repository:
             freeable_ratio = 1.0 * freeable_space / segment_size
             # we want to compact if:
             # - we can free a considerable relative amount of space (freeable_ratio over some threshold)
-            if not (freeable_ratio > 0.1):
+            if not (freeable_ratio > threshold):
                 logger.debug('not compacting segment %d (freeable: %2.2f%% [%d bytes])',
                              segment, freeable_ratio * 100.0, freeable_space)
                 pi.show()

src/borg/testsuite/repository.py

@@ -785,7 +785,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         # Simulate a crash before compact
         with patch.object(Repository, 'compact_segments') as compact:
             self.repository.commit(compact=True)
-            compact.assert_called_once_with()
+            compact.assert_called_once_with(0.1)
         self.reopen()
         with self.repository:
             self.check(repair=True)