From a987c36f6d99e9863c7c10959fb596c5c926a182 Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 11:31:06 +0300 Subject: [PATCH 1/9] add base fs s3 --- cmd/restic/cmd_backup.go | 17 +++ internal/fs/fs_s3.go | 265 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 internal/fs/fs_s3.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f9b45fe51..15b8743b7 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -383,8 +383,15 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu return funcs, nil } +const S3_PREFIX = "s3:/" + // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) { + // example "s3://bucketname/maybe-folder" + //TODO: add collect from many source + if len(args) == 1 && strings.HasPrefix(args[0], S3_PREFIX) { + return []string{strings.Replace(args[0], S3_PREFIX, "", 1)}, nil + } if opts.Stdin || opts.StdinCommand { return nil, nil } @@ -496,6 +503,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te return err } + isS3Source := len(args) == 1 && strings.HasPrefix(args[0], S3_PREFIX) success := true targets, err := collectTargets(opts, args, printer.E, term.InputRaw()) if err != nil { @@ -603,6 +611,15 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te targets = []string{filename} } + if isS3Source { + s3Source := &fs.S3Source{} + err := s3Source.WarmingUp(targets) + if err != nil { + return err + } + targetFS = s3Source + } + if backupFSTestHook != nil { targetFS = backupFSTestHook(targetFS) } diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go new file mode 100644 index 000000000..1e178e211 --- /dev/null +++ b/internal/fs/fs_s3.go @@ -0,0 +1,265 @@ +package fs + +import ( + "context" + "fmt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/restic/restic/internal/data" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" +) + +type S3Source struct { + s3Client *minio.Client + cache map[string]*ExtendedFileInfo + filesByFolder map[string][]string + once sync.Once + targets []string +} + +// statically ensure that S3Source implements FS. +var _ FS = &S3Source{} + +func (fs *S3Source) VolumeName(_ string) string { + return "" +} + +// OpenFile opens a file or directory for reading. +func (fs *S3Source) OpenFile(name string, _ int, metadataOnly bool) (File, error) { + name = s3CleanPath(name) + if name == "/" { + return nil, fmt.Errorf("invalid filename specified") + } + + fi, ok := fs.cache[name] + if !ok { + return nil, pathError("open file", name, os.ErrNotExist) + } + + return newS3SourceFile(name, fi, fs.s3Client, + // is not folder, value is nil + fs.filesByFolder[name], metadataOnly) +} + +func (fs *S3Source) factoryS3Client() (*minio.Client, error) { + endpoint := os.Getenv("AWS_ENDPOINT_URL") + accessKeyID := os.Getenv("AWS_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + if accessKeyID == "" && secretAccessKey != "" { + return nil, errors.Fatalf("no credentials found. $AWS_SECRET_ACCESS_KEY is set but $AWS_ACCESS_KEY_ID is empty") + } else if accessKeyID != "" && secretAccessKey == "" { + return nil, errors.Fatalf("no credentials found. $AWS_ACCESS_KEY_ID is set but $AWS_SECRET_ACCESS_KEY is empty") + } else if endpoint == "" { + return nil, errors.Fatalf("no credentials found. $AWS_ENDPOINT_URL is empty") + } + s3Client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + }) + if err != nil { + return nil, err + } + + return s3Client, nil +} + +func (fs *S3Source) WarmingUp(targets []string) error { + stateDate := time.Now() + defer func() { + debug.Log("s3 duration warming up %s", time.Since(stateDate)) + }() + + var err error + fs.s3Client, err = fs.factoryS3Client() + + if err != nil { + return err + } + + fs.cache = make(map[string]*ExtendedFileInfo) + + filesByFolder := make(map[string][]string) + //TODO: do async by targets + for _, target := range targets { + partPath := strings.Split(target, "/") + // example /bucket-name + bucketName := partPath[1] + prefix := path.Join(partPath[2:]...) + fmt.Println(prefix) + root := path.Join("/", bucketName) + ctx := context.Background() + for obj := range fs.s3Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Recursive: true, Prefix: prefix}) { + if obj.Err != nil { + return obj.Err + } + + absPath := path.Join(root, obj.Key) + for objPath := absPath; ; { + objPath, _ = path.Split(objPath) + objPath = path.Clean(objPath) + if objPath == "/" { + break + } + + if _, ok := fs.cache[objPath]; !ok { + fi := &ExtendedFileInfo{ + Name: path.Base(objPath), + Mode: os.ModeDir | 0755, + ModTime: time.Unix(0, 0), + Size: 0, + } + fs.cache[objPath] = fi + } else { + // this tree already added + break + } + } + { + dir, file := path.Split(absPath) + dir = path.Clean(dir) + filesByFolder[dir] = append(filesByFolder[dir], file) + } + + fs.cache[absPath] = &ExtendedFileInfo{ + Name: path.Base(absPath), + Mode: 0644, + ModTime: obj.LastModified, + Size: obj.Size, + } + } + } + fs.filesByFolder = filesByFolder + return nil +} + +// Lstat returns the FileInfo structure describing the named file. +// If there is an error, it will be of type *os.PathError. +func (fs *S3Source) Lstat(name string) (*ExtendedFileInfo, error) { + name = s3CleanPath(name) + info, ok := fs.cache[name] + if !ok { + return nil, pathError("lstat", name, os.ErrNotExist) + } + return info, nil +} + +func (fs *S3Source) Join(elem ...string) string { + return filepath.Join(elem...) +} + +func (fs *S3Source) Separator() string { + return "/" +} + +func (fs *S3Source) IsAbs(p string) bool { + return p[0] == '/' +} +func (fs *S3Source) Abs(p string) (string, error) { + return s3CleanPath(p), nil +} + +func s3CleanPath(name string) string { + return path.Clean("/" + name) +} + +func (fs *S3Source) Clean(p string) string { + return path.Clean(p) +} + +func (fs *S3Source) Base(p string) string { + return path.Base(p) +} + +func (fs *S3Source) Dir(p string) string { + return path.Dir(p) +} + +type s3SourceFile struct { + rc io.ReadCloser + name string + fi *ExtendedFileInfo + filesInFolder []string + s3Client *minio.Client +} + +// See the File interface for a description of each method +var _ File = &s3SourceFile{} + +func newS3SourceFile(name string, fi *ExtendedFileInfo, s3Client *minio.Client, filesInFolder []string, metadataOnly bool) (*s3SourceFile, error) { + name = s3CleanPath(name) + + if !metadataOnly { + partPath := strings.Split(name, "/") + // example /bucket-name + bucketName := partPath[1] + ctx := context.Background() + objPath := path.Join(partPath[2:]...) + if len(objPath) > 0 { + object, err := s3Client.GetObject(ctx, bucketName, objPath, minio.GetObjectOptions{}) + if err != nil { + return nil, pathError("open file s3", name, os.ErrNotExist) + } + return &s3SourceFile{name: name, fi: fi, rc: object, filesInFolder: filesInFolder, s3Client: s3Client}, nil + } + } + return &s3SourceFile{name: name, fi: fi, rc: nil, filesInFolder: filesInFolder, s3Client: s3Client}, nil + +} + +func (f *s3SourceFile) MakeReadable() error { + if f.rc != nil { + panic("s3 file is already readable") + } + + newF, err := newS3SourceFile(f.name, f.fi, f.s3Client, f.filesInFolder, false) + if err != nil { + return err + } + // replace state and also reset cached FileInfo + *f = *newF + return nil +} + +func (f *s3SourceFile) Stat() (*ExtendedFileInfo, error) { + return f.fi, nil +} + +func (f *s3SourceFile) ToNode(_ bool, _ func(format string, args ...any)) (*data.Node, error) { + node := buildBasicNode(f.name, f.fi) + + //TODO: change on info about owner in repo + node.UID = 0 //uint32(os.Getuid()) + node.GID = 0 //uint32(os.Getgid()) + node.ChangeTime = node.ModTime + + return node, nil +} + +func (f *s3SourceFile) Read(p []byte) (n int, err error) { + if f.rc != nil { + return f.rc.Read(p) + } + + return 0, pathError("read", f.name, os.ErrNotExist) +} + +func (f *s3SourceFile) Readdirnames(_ int) ([]string, error) { + if f.filesInFolder == nil { + return []string{}, pathError("Readdirnames", f.name, os.ErrNotExist) + } + return f.filesInFolder, nil +} + +func (f *s3SourceFile) Close() error { + if f.rc != nil { + return f.rc.Close() + } + return nil +} From 6b19f9a6d33a0c3d81fc8f3557026c0ebfd2484e Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 12:19:32 +0300 Subject: [PATCH 2/9] add async request for many targets --- internal/fs/fs_s3.go | 114 +++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go index 1e178e211..3c318d9ef 100644 --- a/internal/fs/fs_s3.go +++ b/internal/fs/fs_s3.go @@ -19,7 +19,7 @@ import ( type S3Source struct { s3Client *minio.Client - cache map[string]*ExtendedFileInfo + files map[string]*ExtendedFileInfo filesByFolder map[string][]string once sync.Once targets []string @@ -39,7 +39,7 @@ func (fs *S3Source) OpenFile(name string, _ int, metadataOnly bool) (File, error return nil, fmt.Errorf("invalid filename specified") } - fi, ok := fs.cache[name] + fi, ok := fs.files[name] if !ok { return nil, pathError("open file", name, os.ErrNotExist) } @@ -83,59 +83,91 @@ func (fs *S3Source) WarmingUp(targets []string) error { return err } - fs.cache = make(map[string]*ExtendedFileInfo) - + var muFilesByFolder sync.Mutex filesByFolder := make(map[string][]string) - //TODO: do async by targets + var muFiles sync.Mutex + files := make(map[string]*ExtendedFileInfo) + + var wg sync.WaitGroup + wg.Add(len(targets)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errCh := make(chan error, 1) for _, target := range targets { partPath := strings.Split(target, "/") // example /bucket-name bucketName := partPath[1] prefix := path.Join(partPath[2:]...) - fmt.Println(prefix) root := path.Join("/", bucketName) - ctx := context.Background() - for obj := range fs.s3Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Recursive: true, Prefix: prefix}) { - if obj.Err != nil { - return obj.Err - } - absPath := path.Join(root, obj.Key) - for objPath := absPath; ; { - objPath, _ = path.Split(objPath) - objPath = path.Clean(objPath) - if objPath == "/" { - break - } + go func() { + defer wg.Done() + for obj := range fs.s3Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Recursive: true, Prefix: prefix}) { - if _, ok := fs.cache[objPath]; !ok { - fi := &ExtendedFileInfo{ - Name: path.Base(objPath), - Mode: os.ModeDir | 0755, - ModTime: time.Unix(0, 0), - Size: 0, + if obj.Err != nil { + if ctx.Err() == nil { + errCh <- obj.Err } - fs.cache[objPath] = fi - } else { - // this tree already added - break + cancel() + return } - } - { - dir, file := path.Split(absPath) - dir = path.Clean(dir) - filesByFolder[dir] = append(filesByFolder[dir], file) - } - fs.cache[absPath] = &ExtendedFileInfo{ - Name: path.Base(absPath), - Mode: 0644, - ModTime: obj.LastModified, - Size: obj.Size, + absPath := path.Join(root, obj.Key) + for objPath := absPath; ; { + objPath, _ = path.Split(objPath) + objPath = path.Clean(objPath) + if objPath == "/" { + break + } + + if _, ok := files[objPath]; !ok { + fi := &ExtendedFileInfo{ + Name: path.Base(objPath), + Mode: os.ModeDir | 0755, + ModTime: time.Unix(0, 0), + Size: 0, + } + muFiles.Lock() + files[objPath] = fi + muFiles.Unlock() + } else { + // this tree already added + break + } + } + { + dir, file := path.Split(absPath) + dir = path.Clean(dir) + muFilesByFolder.Lock() + filesByFolder[dir] = append(filesByFolder[dir], file) + muFilesByFolder.Unlock() + } + + muFiles.Lock() + files[absPath] = &ExtendedFileInfo{ + Name: path.Base(absPath), + Mode: 0644, + ModTime: obj.LastModified, + Size: obj.Size, + } + muFiles.Unlock() } - } + }() } + wg.Wait() + close(errCh) + + select { + case err, ok := <-errCh: + if err != nil && ok { + return err + } + default: + } + fs.filesByFolder = filesByFolder + fs.files = files return nil } @@ -143,7 +175,7 @@ func (fs *S3Source) WarmingUp(targets []string) error { // If there is an error, it will be of type *os.PathError. func (fs *S3Source) Lstat(name string) (*ExtendedFileInfo, error) { name = s3CleanPath(name) - info, ok := fs.cache[name] + info, ok := fs.files[name] if !ok { return nil, pathError("lstat", name, os.ErrNotExist) } From 9585971669f734c6247b63ee26fcf66ff784fbb4 Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 14:03:59 +0300 Subject: [PATCH 3/9] add parse entry point --- internal/fs/fs_s3.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go index 3c318d9ef..0d0fdbc5b 100644 --- a/internal/fs/fs_s3.go +++ b/internal/fs/fs_s3.go @@ -9,6 +9,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "io" + "net/url" "os" "path" "path/filepath" @@ -60,8 +61,14 @@ func (fs *S3Source) factoryS3Client() (*minio.Client, error) { } else if endpoint == "" { return nil, errors.Fatalf("no credentials found. $AWS_ENDPOINT_URL is empty") } - s3Client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + + urlEndpoint, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + s3Client, err := minio.New(urlEndpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: urlEndpoint.Scheme == "https", }) if err != nil { return nil, err From 16d871a40d67be94961f12301eca9b091bbcea5a Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 14:07:58 +0300 Subject: [PATCH 4/9] add source s3 for many targets --- cmd/restic/cmd_backup.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 15b8743b7..20c249060 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -387,11 +387,6 @@ const S3_PREFIX = "s3:/" // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) { - // example "s3://bucketname/maybe-folder" - //TODO: add collect from many source - if len(args) == 1 && strings.HasPrefix(args[0], S3_PREFIX) { - return []string{strings.Replace(args[0], S3_PREFIX, "", 1)}, nil - } if opts.Stdin || opts.StdinCommand { return nil, nil } @@ -449,6 +444,16 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar return nil, errors.Fatal("nothing to backup, please specify source files/dirs") } + // example "s3://bucketname/maybe-folder" + if strings.HasPrefix(targets[0], S3_PREFIX) { + for _, target := range targets { + if !strings.HasPrefix(target, S3_PREFIX) { + return nil, errors.Fatalf("target=%s has not prefix s3:/", target) + } + } + return targets, nil + } + return filterExisting(targets, warnf) } @@ -503,9 +508,15 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te return err } - isS3Source := len(args) == 1 && strings.HasPrefix(args[0], S3_PREFIX) success := true targets, err := collectTargets(opts, args, printer.E, term.InputRaw()) + isS3Source := strings.HasPrefix(targets[0], S3_PREFIX) + if isS3Source { + for i, target := range targets { + targets[i] = strings.Replace(target, S3_PREFIX, "", 1) + } + } + if err != nil { if errors.Is(err, ErrInvalidSourceData) { success = false From e844950d10f5991da7056bab0032065dc30cd72c Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 14:22:28 +0300 Subject: [PATCH 5/9] move const to fs pakcage --- cmd/restic/cmd_backup.go | 10 ++++------ internal/fs/fs_s3.go | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 20c249060..f691737f3 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -383,8 +383,6 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu return funcs, nil } -const S3_PREFIX = "s3:/" - // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) { if opts.Stdin || opts.StdinCommand { @@ -445,9 +443,9 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar } // example "s3://bucketname/maybe-folder" - if strings.HasPrefix(targets[0], S3_PREFIX) { + if strings.HasPrefix(targets[0], fs.S3_PREFIX) { for _, target := range targets { - if !strings.HasPrefix(target, S3_PREFIX) { + if !strings.HasPrefix(target, fs.S3_PREFIX) { return nil, errors.Fatalf("target=%s has not prefix s3:/", target) } } @@ -510,10 +508,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te success := true targets, err := collectTargets(opts, args, printer.E, term.InputRaw()) - isS3Source := strings.HasPrefix(targets[0], S3_PREFIX) + isS3Source := strings.HasPrefix(targets[0], fs.S3_PREFIX) if isS3Source { for i, target := range targets { - targets[i] = strings.Replace(target, S3_PREFIX, "", 1) + targets[i] = strings.Replace(target, fs.S3_PREFIX, "", 1) } } diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go index 0d0fdbc5b..e0a0873cb 100644 --- a/internal/fs/fs_s3.go +++ b/internal/fs/fs_s3.go @@ -18,6 +18,8 @@ import ( "time" ) +const S3_PREFIX = "s3:/" + type S3Source struct { s3Client *minio.Client files map[string]*ExtendedFileInfo From e03f9e4f1923e9a056153733f5bfff724b31f36a Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 14:48:26 +0300 Subject: [PATCH 6/9] fix detect change --- internal/fs/fs_s3.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go index e0a0873cb..61a2853f9 100644 --- a/internal/fs/fs_s3.go +++ b/internal/fs/fs_s3.go @@ -132,10 +132,11 @@ func (fs *S3Source) WarmingUp(targets []string) error { if _, ok := files[objPath]; !ok { fi := &ExtendedFileInfo{ - Name: path.Base(objPath), - Mode: os.ModeDir | 0755, - ModTime: time.Unix(0, 0), - Size: 0, + Name: path.Base(objPath), + Mode: os.ModeDir | 0755, + ModTime: time.Unix(0, 0), + ChangeTime: time.Unix(0, 0), + Size: 0, } muFiles.Lock() files[objPath] = fi @@ -155,10 +156,11 @@ func (fs *S3Source) WarmingUp(targets []string) error { muFiles.Lock() files[absPath] = &ExtendedFileInfo{ - Name: path.Base(absPath), - Mode: 0644, - ModTime: obj.LastModified, - Size: obj.Size, + Name: path.Base(absPath), + Mode: 0644, + ModTime: obj.LastModified, + ChangeTime: obj.LastModified, + Size: obj.Size, } muFiles.Unlock() } From 96474a5abd2eadb70668cc5e04c8cbcbefa23fa8 Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 15:51:46 +0300 Subject: [PATCH 7/9] required bucket name and fix auto memory --- cmd/restic/cmd_backup.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f691737f3..ab5a17f94 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -447,6 +447,12 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar for _, target := range targets { if !strings.HasPrefix(target, fs.S3_PREFIX) { return nil, errors.Fatalf("target=%s has not prefix s3:/", target) + + } + paths := strings.Split(strings.Replace(target, fs.S3_PREFIX, "", 1), "/") + fmt.Println(strings.Replace(target, fs.S3_PREFIX, "", 1), paths, len(paths)) + if len(paths) < 2 || paths[1] == "" { + return nil, errors.Fatalf("target=%s has not bucketName", target) } } return targets, nil @@ -508,7 +514,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te success := true targets, err := collectTargets(opts, args, printer.E, term.InputRaw()) - isS3Source := strings.HasPrefix(targets[0], fs.S3_PREFIX) + isS3Source := false + if len(targets) > 0 { + isS3Source = strings.HasPrefix(targets[0], fs.S3_PREFIX) + } if isS3Source { for i, target := range targets { targets[i] = strings.Replace(target, fs.S3_PREFIX, "", 1) From 546d4527a76c4f63a960bae2b9266250944d41a5 Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Sun, 23 Nov 2025 15:52:25 +0300 Subject: [PATCH 8/9] remove logs --- cmd/restic/cmd_backup.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index ab5a17f94..d96cc375f 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -450,7 +450,6 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar } paths := strings.Split(strings.Replace(target, fs.S3_PREFIX, "", 1), "/") - fmt.Println(strings.Replace(target, fs.S3_PREFIX, "", 1), paths, len(paths)) if len(paths) < 2 || paths[1] == "" { return nil, errors.Fatalf("target=%s has not bucketName", target) } From d0e0d09ff169991e9edd08fc760fd7a01bff28ae Mon Sep 17 00:00:00 2001 From: BataevDaniil Date: Mon, 24 Nov 2025 13:17:06 +0300 Subject: [PATCH 9/9] refactor --- cmd/restic/cmd_backup.go | 27 +++++++------- internal/fs/fs_s3.go | 79 +++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index d96cc375f..0207c84ac 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -443,13 +443,13 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar } // example "s3://bucketname/maybe-folder" - if strings.HasPrefix(targets[0], fs.S3_PREFIX) { + if strings.HasPrefix(targets[0], fs.S3Prefix) { for _, target := range targets { - if !strings.HasPrefix(target, fs.S3_PREFIX) { - return nil, errors.Fatalf("target=%s has not prefix s3:/", target) + if !strings.HasPrefix(target, fs.S3Prefix) { + return nil, errors.Fatalf("target=%s has not prefix %s", target, fs.S3Prefix) } - paths := strings.Split(strings.Replace(target, fs.S3_PREFIX, "", 1), "/") + paths := strings.Split(strings.TrimPrefix(target, fs.S3Prefix), "/") if len(paths) < 2 || paths[1] == "" { return nil, errors.Fatalf("target=%s has not bucketName", target) } @@ -513,15 +513,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te success := true targets, err := collectTargets(opts, args, printer.E, term.InputRaw()) - isS3Source := false - if len(targets) > 0 { - isS3Source = strings.HasPrefix(targets[0], fs.S3_PREFIX) - } - if isS3Source { - for i, target := range targets { - targets[i] = strings.Replace(target, fs.S3_PREFIX, "", 1) - } - } if err != nil { if errors.Is(err, ErrInvalidSourceData) { @@ -531,6 +522,16 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te } } + isS3Source := false + if len(targets) > 0 { + isS3Source = strings.HasPrefix(targets[0], fs.S3Prefix) + if isS3Source { + for i, target := range targets { + targets[i] = strings.TrimPrefix(target, fs.S3Prefix) + } + } + } + timeStamp := time.Now() backupStart := timeStamp if opts.TimeStamp != "" { diff --git a/internal/fs/fs_s3.go b/internal/fs/fs_s3.go index 61a2853f9..b09445d5d 100644 --- a/internal/fs/fs_s3.go +++ b/internal/fs/fs_s3.go @@ -9,23 +9,23 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "io" + "io/fs" "net/url" "os" "path" - "path/filepath" "strings" "sync" "time" ) -const S3_PREFIX = "s3:/" +const S3Prefix = "s3:/" +const basePermissionFile fs.FileMode = 0644 +const basePermissionFolder fs.FileMode = os.ModeDir | 0755 type S3Source struct { s3Client *minio.Client files map[string]*ExtendedFileInfo filesByFolder map[string][]string - once sync.Once - targets []string } // statically ensure that S3Source implements FS. @@ -101,8 +101,7 @@ func (fs *S3Source) WarmingUp(targets []string) error { wg.Add(len(targets)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - errCh := make(chan error, 1) + errCh := make(chan error, len(targets)) for _, target := range targets { partPath := strings.Split(target, "/") // example /bucket-name @@ -113,38 +112,38 @@ func (fs *S3Source) WarmingUp(targets []string) error { go func() { defer wg.Done() for obj := range fs.s3Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Recursive: true, Prefix: prefix}) { - if obj.Err != nil { if ctx.Err() == nil { - errCh <- obj.Err + select { + case errCh <- obj.Err: + default: + } } cancel() return } absPath := path.Join(root, obj.Key) - for objPath := absPath; ; { - objPath, _ = path.Split(objPath) - objPath = path.Clean(objPath) - if objPath == "/" { + for currPath := absPath; ; { + currPath = path.Clean(path.Dir(currPath)) + if currPath == "/" { break } - if _, ok := files[objPath]; !ok { - fi := &ExtendedFileInfo{ - Name: path.Base(objPath), - Mode: os.ModeDir | 0755, - ModTime: time.Unix(0, 0), - ChangeTime: time.Unix(0, 0), - Size: 0, - } - muFiles.Lock() - files[objPath] = fi + muFiles.Lock() + if _, exists := files[currPath]; exists { muFiles.Unlock() - } else { // this tree already added break } + files[currPath] = &ExtendedFileInfo{ + Name: path.Base(currPath), + Mode: basePermissionFolder, + ModTime: time.Unix(0, 0), + ChangeTime: time.Unix(0, 0), + Size: 0, + } + muFiles.Unlock() } { dir, file := path.Split(absPath) @@ -157,7 +156,7 @@ func (fs *S3Source) WarmingUp(targets []string) error { muFiles.Lock() files[absPath] = &ExtendedFileInfo{ Name: path.Base(absPath), - Mode: 0644, + Mode: basePermissionFile, ModTime: obj.LastModified, ChangeTime: obj.LastModified, Size: obj.Size, @@ -194,7 +193,7 @@ func (fs *S3Source) Lstat(name string) (*ExtendedFileInfo, error) { } func (fs *S3Source) Join(elem ...string) string { - return filepath.Join(elem...) + return path.Join(elem...) } func (fs *S3Source) Separator() string { @@ -202,7 +201,7 @@ func (fs *S3Source) Separator() string { } func (fs *S3Source) IsAbs(p string) bool { - return p[0] == '/' + return path.IsAbs(p) } func (fs *S3Source) Abs(p string) (string, error) { return s3CleanPath(p), nil @@ -237,22 +236,20 @@ var _ File = &s3SourceFile{} func newS3SourceFile(name string, fi *ExtendedFileInfo, s3Client *minio.Client, filesInFolder []string, metadataOnly bool) (*s3SourceFile, error) { name = s3CleanPath(name) - - if !metadataOnly { - partPath := strings.Split(name, "/") - // example /bucket-name - bucketName := partPath[1] - ctx := context.Background() - objPath := path.Join(partPath[2:]...) - if len(objPath) > 0 { - object, err := s3Client.GetObject(ctx, bucketName, objPath, minio.GetObjectOptions{}) - if err != nil { - return nil, pathError("open file s3", name, os.ErrNotExist) - } - return &s3SourceFile{name: name, fi: fi, rc: object, filesInFolder: filesInFolder, s3Client: s3Client}, nil - } + if metadataOnly || fi.Mode.IsDir() { + return &s3SourceFile{name: name, fi: fi, rc: nil, filesInFolder: filesInFolder, s3Client: s3Client}, nil } - return &s3SourceFile{name: name, fi: fi, rc: nil, filesInFolder: filesInFolder, s3Client: s3Client}, nil + + partPath := strings.Split(name, "/") + // example /bucket-name + bucketName := partPath[1] + objPath := path.Join(partPath[2:]...) + ctx := context.Background() + object, err := s3Client.GetObject(ctx, bucketName, objPath, minio.GetObjectOptions{}) + if err != nil { + return nil, pathError("open file s3", name, os.ErrNotExist) + } + return &s3SourceFile{name: name, fi: fi, rc: object, filesInFolder: filesInFolder, s3Client: s3Client}, nil }