diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index 9dd8bfbfc..7ec4702b7 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -600,13 +600,16 @@ class Archiver:
                 if recurse:
                     tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present)
                     if tag_paths:
-                        if keep_exclude_tags and not dry_run:
-                            archive.process_dir(path, st)
-                            for tag_path in tag_paths:
-                                self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
-                                              keep_exclude_tags, skip_inodes, tag_path, restrict_dev,
-                                              read_special=read_special, dry_run=dry_run)
-                        self.print_file_status('x', path)
+                        # if we are already recursing in an excluded dir, we do not need to do anything else than
+                        # returning (we do not need to archive or recurse into tagged directories), see #3991:
+                        if not recurse_excluded_dir:
+                            if keep_exclude_tags and not dry_run:
+                                archive.process_dir(path, st)
+                                for tag_path in tag_paths:
+                                    self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
+                                                  keep_exclude_tags, skip_inodes, tag_path, restrict_dev,
+                                                  read_special=read_special, dry_run=dry_run)
+                            self.print_file_status('x', path)
                         return
                 if not dry_run:
                     if not recurse_excluded_dir: