Merge f794fabe18 into f000da3b35

changelog/unreleased/issue-5729

@@ -0,0 +1,8 @@
+Enhancement: Support `**` wildcard in `--files-from` patterns
+
+The `--files-from` option now supports the `**` wildcard pattern for
+matching files across directory boundaries. For example,
+`/home/user/**/*.json` will match all `.json` files in any
+subdirectory. Previously, only single `*` wildcards were supported.
+
+https://github.com/restic/restic/issues/5729

cmd/restic/cmd_backup.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"os"
 	"path"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -411,7 +410,7 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
 			}
 
 			var expanded []string
-			expanded, err := filepath.Glob(line)
+			expanded, err := filter.Glob(line)
 			if err != nil {
 				return nil, fmt.Errorf("pattern: %s: %w", line, err)
 			}

internal/filter/glob.go

@@ -0,0 +1,85 @@
+package filter
+
+import (
+	"io/fs"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Glob returns the names of all files matching the pattern, supporting **
+// wildcards for recursive directory matching. If the pattern contains no **,
+// it delegates to filepath.Glob. Otherwise, it walks the filesystem from the
+// longest static prefix directory and uses filter.Match to find matches.
+//
+// The returned matches are sorted in lexical order, consistent with
+// filepath.Glob behavior. If no files match, a nil slice is returned with no
+// error (also consistent with filepath.Glob).
+func Glob(pattern string) ([]string, error) {
+	if !strings.Contains(pattern, "**") {
+		return filepath.Glob(pattern)
+	}
+
+	root := staticPrefix(pattern)
+
+	var matches []string
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		matched, matchErr := Match(pattern, path)
+		if matchErr != nil {
+			return matchErr
+		}
+		if matched {
+			matches = append(matches, path)
+		}
+
+		if d.IsDir() && path != root {
+			childMatched, childErr := ChildMatch(pattern, path)
+			if childErr != nil {
+				return childErr
+			}
+			if !childMatched {
+				return fs.SkipDir
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Strings(matches)
+	return matches, nil
+}
+
+// staticPrefix extracts the longest directory path before the first path
+// component containing a wildcard character. For example, given
+// "/home/user/**/*.json", it returns "/home/user". If no static prefix exists
+// (e.g., "**/*.go"), it returns ".".
+func staticPrefix(pattern string) string {
+	// Clean the pattern to normalize separators
+	pattern = filepath.Clean(pattern)
+	parts := strings.Split(filepath.ToSlash(pattern), "/")
+
+	var prefix []string
+	for _, part := range parts {
+		if strings.ContainsAny(part, "*?[") {
+			break
+		}
+		prefix = append(prefix, part)
+	}
+
+	if len(prefix) == 0 {
+		return "."
+	}
+
+	result := filepath.FromSlash(strings.Join(prefix, "/"))
+	if result == "" {
+		return "."
+	}
+	return result
+}

internal/filter/glob_test.go

@@ -0,0 +1,112 @@
+package filter_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/filter"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func createGlobTestDir(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+
+	dirs := []string{
+		"sub",
+		"sub/deep",
+		filepath.Join("sub", "deep", "nested"),
+	}
+	for _, d := range dirs {
+		err := os.MkdirAll(filepath.Join(dir, d), 0o755)
+		rtest.OK(t, err)
+	}
+
+	files := []string{
+		"file.txt",
+		"file.json",
+		filepath.Join("sub", "file.txt"),
+		filepath.Join("sub", "deep", "file.txt"),
+		filepath.Join("sub", "deep", "other.json"),
+		filepath.Join("sub", "deep", "nested", "file.txt"),
+	}
+	for _, f := range files {
+		err := os.WriteFile(filepath.Join(dir, f), []byte("test"), 0o644)
+		rtest.OK(t, err)
+	}
+
+	return dir
+}
+
+func TestGlobDoublestar(t *testing.T) {
+	dir := createGlobTestDir(t)
+
+	tests := []struct {
+		pattern  string
+		expected []string
+	}{
+		{
+			pattern: filepath.Join(dir, "**", "*.txt"),
+			expected: []string{
+				filepath.Join(dir, "file.txt"),
+				filepath.Join(dir, "sub", "deep", "file.txt"),
+				filepath.Join(dir, "sub", "deep", "nested", "file.txt"),
+				filepath.Join(dir, "sub", "file.txt"),
+			},
+		},
+		{
+			pattern: filepath.Join(dir, "**", "*.json"),
+			expected: []string{
+				filepath.Join(dir, "file.json"),
+				filepath.Join(dir, "sub", "deep", "other.json"),
+			},
+		},
+		{
+			pattern: filepath.Join(dir, "sub", "**", "*.txt"),
+			expected: []string{
+				filepath.Join(dir, "sub", "deep", "file.txt"),
+				filepath.Join(dir, "sub", "deep", "nested", "file.txt"),
+				filepath.Join(dir, "sub", "file.txt"),
+			},
+		},
+		{
+			pattern: filepath.Join(dir, "**", "deep", "*.txt"),
+			expected: []string{
+				filepath.Join(dir, "sub", "deep", "file.txt"),
+			},
+		},
+		{
+			pattern:  filepath.Join(dir, "**", "*.py"),
+			expected: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.pattern, func(t *testing.T) {
+			matches, err := filter.Glob(test.pattern)
+			rtest.OK(t, err)
+			rtest.Equals(t, test.expected, matches)
+		})
+	}
+}
+
+func TestGlobNoDoublestar(t *testing.T) {
+	dir := createGlobTestDir(t)
+
+	// Without **, should delegate to filepath.Glob and only match top-level
+	pattern := filepath.Join(dir, "*.txt")
+	matches, err := filter.Glob(pattern)
+	rtest.OK(t, err)
+
+	expected := []string{filepath.Join(dir, "file.txt")}
+	rtest.Equals(t, expected, matches)
+}
+
+func TestGlobNoMatches(t *testing.T) {
+	dir := createGlobTestDir(t)
+
+	matches, err := filter.Glob(filepath.Join(dir, "**", "*.xyz"))
+	rtest.OK(t, err)
+	rtest.Equals(t, ([]string)(nil), matches)
+}