From f794fabe18b235c7f32bc6cc50b29bb52a1ab184 Mon Sep 17 00:00:00 2001 From: Omar Yusuf Abdi Date: Sat, 14 Mar 2026 18:43:13 +0100 Subject: [PATCH] filter: add Glob function supporting ** wildcard patterns This enables --files-from to use ** for recursive file matching, matching the behavior already available in --exclude patterns. Fixes #5729 --- changelog/unreleased/issue-5729 | 8 +++ cmd/restic/cmd_backup.go | 3 +- internal/filter/glob.go | 85 ++++++++++++++++++++++++ internal/filter/glob_test.go | 112 ++++++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-5729 create mode 100644 internal/filter/glob.go create mode 100644 internal/filter/glob_test.go diff --git a/changelog/unreleased/issue-5729 b/changelog/unreleased/issue-5729 new file mode 100644 index 000000000..c43e65b1c --- /dev/null +++ b/changelog/unreleased/issue-5729 @@ -0,0 +1,8 @@ +Enhancement: Support `**` wildcard in `--files-from` patterns + +The `--files-from` option now supports the `**` wildcard pattern for +matching files across directory boundaries. For example, +`/home/user/**/*.json` will match all `.json` files in any +subdirectory. Previously, only single `*` wildcards were supported. + +https://github.com/restic/restic/issues/5729 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f9b45fe51..06fde1c5f 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -8,7 +8,6 @@ import ( "io" "os" "path" - "path/filepath" "runtime" "strconv" "strings" @@ -403,7 +402,7 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar } var expanded []string - expanded, err := filepath.Glob(line) + expanded, err := filter.Glob(line) if err != nil { return nil, fmt.Errorf("pattern: %s: %w", line, err) } diff --git a/internal/filter/glob.go b/internal/filter/glob.go new file mode 100644 index 000000000..25596f049 --- /dev/null +++ b/internal/filter/glob.go @@ -0,0 +1,85 @@ +package filter + +import ( + "io/fs" + "path/filepath" + "sort" + "strings" +) + +// Glob returns the names of all files matching the pattern, supporting ** +// wildcards for recursive directory matching. If the pattern contains no **, +// it delegates to filepath.Glob. Otherwise, it walks the filesystem from the +// longest static prefix directory and uses filter.Match to find matches. +// +// The returned matches are sorted in lexical order, consistent with +// filepath.Glob behavior. If no files match, a nil slice is returned with no +// error (also consistent with filepath.Glob). +func Glob(pattern string) ([]string, error) { + if !strings.Contains(pattern, "**") { + return filepath.Glob(pattern) + } + + root := staticPrefix(pattern) + + var matches []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + matched, matchErr := Match(pattern, path) + if matchErr != nil { + return matchErr + } + if matched { + matches = append(matches, path) + } + + if d.IsDir() && path != root { + childMatched, childErr := ChildMatch(pattern, path) + if childErr != nil { + return childErr + } + if !childMatched { + return fs.SkipDir + } + } + + return nil + }) + if err != nil { + return nil, err + } + + sort.Strings(matches) + return matches, nil +} + +// staticPrefix extracts the longest directory path before the first path +// component containing a wildcard character. For example, given +// "/home/user/**/*.json", it returns "/home/user". If no static prefix exists +// (e.g., "**/*.go"), it returns ".". +func staticPrefix(pattern string) string { + // Clean the pattern to normalize separators + pattern = filepath.Clean(pattern) + parts := strings.Split(filepath.ToSlash(pattern), "/") + + var prefix []string + for _, part := range parts { + if strings.ContainsAny(part, "*?[") { + break + } + prefix = append(prefix, part) + } + + if len(prefix) == 0 { + return "." + } + + result := filepath.FromSlash(strings.Join(prefix, "/")) + if result == "" { + return "." + } + return result +} diff --git a/internal/filter/glob_test.go b/internal/filter/glob_test.go new file mode 100644 index 000000000..d894e4b10 --- /dev/null +++ b/internal/filter/glob_test.go @@ -0,0 +1,112 @@ +package filter_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/restic/restic/internal/filter" + rtest "github.com/restic/restic/internal/test" +) + +func createGlobTestDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + + dirs := []string{ + "sub", + "sub/deep", + filepath.Join("sub", "deep", "nested"), + } + for _, d := range dirs { + err := os.MkdirAll(filepath.Join(dir, d), 0o755) + rtest.OK(t, err) + } + + files := []string{ + "file.txt", + "file.json", + filepath.Join("sub", "file.txt"), + filepath.Join("sub", "deep", "file.txt"), + filepath.Join("sub", "deep", "other.json"), + filepath.Join("sub", "deep", "nested", "file.txt"), + } + for _, f := range files { + err := os.WriteFile(filepath.Join(dir, f), []byte("test"), 0o644) + rtest.OK(t, err) + } + + return dir +} + +func TestGlobDoublestar(t *testing.T) { + dir := createGlobTestDir(t) + + tests := []struct { + pattern string + expected []string + }{ + { + pattern: filepath.Join(dir, "**", "*.txt"), + expected: []string{ + filepath.Join(dir, "file.txt"), + filepath.Join(dir, "sub", "deep", "file.txt"), + filepath.Join(dir, "sub", "deep", "nested", "file.txt"), + filepath.Join(dir, "sub", "file.txt"), + }, + }, + { + pattern: filepath.Join(dir, "**", "*.json"), + expected: []string{ + filepath.Join(dir, "file.json"), + filepath.Join(dir, "sub", "deep", "other.json"), + }, + }, + { + pattern: filepath.Join(dir, "sub", "**", "*.txt"), + expected: []string{ + filepath.Join(dir, "sub", "deep", "file.txt"), + filepath.Join(dir, "sub", "deep", "nested", "file.txt"), + filepath.Join(dir, "sub", "file.txt"), + }, + }, + { + pattern: filepath.Join(dir, "**", "deep", "*.txt"), + expected: []string{ + filepath.Join(dir, "sub", "deep", "file.txt"), + }, + }, + { + pattern: filepath.Join(dir, "**", "*.py"), + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.pattern, func(t *testing.T) { + matches, err := filter.Glob(test.pattern) + rtest.OK(t, err) + rtest.Equals(t, test.expected, matches) + }) + } +} + +func TestGlobNoDoublestar(t *testing.T) { + dir := createGlobTestDir(t) + + // Without **, should delegate to filepath.Glob and only match top-level + pattern := filepath.Join(dir, "*.txt") + matches, err := filter.Glob(pattern) + rtest.OK(t, err) + + expected := []string{filepath.Join(dir, "file.txt")} + rtest.Equals(t, expected, matches) +} + +func TestGlobNoMatches(t *testing.T) { + dir := createGlobTestDir(t) + + matches, err := filter.Glob(filepath.Join(dir, "**", "*.xyz")) + rtest.OK(t, err) + rtest.Equals(t, ([]string)(nil), matches) +}