mirror of
https://github.com/restic/restic.git
synced 2026-05-28 04:35:41 -04:00
refactor
This commit is contained in:
BataevDaniil
parent
546d4527a7
commit
d0e0d09ff1
2 changed files with 52 additions and 54 deletions
|
|
@ -443,13 +443,13 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
|
|||
}
|
||||
|
||||
// example "s3://bucketname/maybe-folder"
|
||||
if strings.HasPrefix(targets[0], fs.S3_PREFIX) {
|
||||
if strings.HasPrefix(targets[0], fs.S3Prefix) {
|
||||
for _, target := range targets {
|
||||
if !strings.HasPrefix(target, fs.S3_PREFIX) {
|
||||
return nil, errors.Fatalf("target=%s has not prefix s3:/", target)
|
||||
if !strings.HasPrefix(target, fs.S3Prefix) {
|
||||
return nil, errors.Fatalf("target=%s has not prefix %s", target, fs.S3Prefix)
|
||||
|
||||
}
|
||||
paths := strings.Split(strings.Replace(target, fs.S3_PREFIX, "", 1), "/")
|
||||
paths := strings.Split(strings.TrimPrefix(target, fs.S3Prefix), "/")
|
||||
if len(paths) < 2 || paths[1] == "" {
|
||||
return nil, errors.Fatalf("target=%s has not bucketName", target)
|
||||
}
|
||||
|
|
@ -513,15 +513,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
|
|||
|
||||
success := true
|
||||
targets, err := collectTargets(opts, args, printer.E, term.InputRaw())
|
||||
isS3Source := false
|
||||
if len(targets) > 0 {
|
||||
isS3Source = strings.HasPrefix(targets[0], fs.S3_PREFIX)
|
||||
}
|
||||
if isS3Source {
|
||||
for i, target := range targets {
|
||||
targets[i] = strings.Replace(target, fs.S3_PREFIX, "", 1)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrInvalidSourceData) {
|
||||
|
|
@ -531,6 +522,16 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
|
|||
}
|
||||
}
|
||||
|
||||
isS3Source := false
|
||||
if len(targets) > 0 {
|
||||
isS3Source = strings.HasPrefix(targets[0], fs.S3Prefix)
|
||||
if isS3Source {
|
||||
for i, target := range targets {
|
||||
targets[i] = strings.TrimPrefix(target, fs.S3Prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timeStamp := time.Now()
|
||||
backupStart := timeStamp
|
||||
if opts.TimeStamp != "" {
|
||||
|
|
|
|||
|
|
@ -9,23 +9,23 @@ import (
|
|||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const S3_PREFIX = "s3:/"
|
||||
const S3Prefix = "s3:/"
|
||||
const basePermissionFile fs.FileMode = 0644
|
||||
const basePermissionFolder fs.FileMode = os.ModeDir | 0755
|
||||
|
||||
type S3Source struct {
|
||||
s3Client *minio.Client
|
||||
files map[string]*ExtendedFileInfo
|
||||
filesByFolder map[string][]string
|
||||
once sync.Once
|
||||
targets []string
|
||||
}
|
||||
|
||||
// statically ensure that S3Source implements FS.
|
||||
|
|
@ -101,8 +101,7 @@ func (fs *S3Source) WarmingUp(targets []string) error {
|
|||
wg.Add(len(targets))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
errCh := make(chan error, len(targets))
|
||||
for _, target := range targets {
|
||||
partPath := strings.Split(target, "/")
|
||||
// example /bucket-name
|
||||
|
|
@ -113,38 +112,38 @@ func (fs *S3Source) WarmingUp(targets []string) error {
|
|||
go func() {
|
||||
defer wg.Done()
|
||||
for obj := range fs.s3Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Recursive: true, Prefix: prefix}) {
|
||||
|
||||
if obj.Err != nil {
|
||||
if ctx.Err() == nil {
|
||||
errCh <- obj.Err
|
||||
select {
|
||||
case errCh <- obj.Err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
absPath := path.Join(root, obj.Key)
|
||||
for objPath := absPath; ; {
|
||||
objPath, _ = path.Split(objPath)
|
||||
objPath = path.Clean(objPath)
|
||||
if objPath == "/" {
|
||||
for currPath := absPath; ; {
|
||||
currPath = path.Clean(path.Dir(currPath))
|
||||
if currPath == "/" {
|
||||
break
|
||||
}
|
||||
|
||||
if _, ok := files[objPath]; !ok {
|
||||
fi := &ExtendedFileInfo{
|
||||
Name: path.Base(objPath),
|
||||
Mode: os.ModeDir | 0755,
|
||||
ModTime: time.Unix(0, 0),
|
||||
ChangeTime: time.Unix(0, 0),
|
||||
Size: 0,
|
||||
}
|
||||
muFiles.Lock()
|
||||
files[objPath] = fi
|
||||
muFiles.Lock()
|
||||
if _, exists := files[currPath]; exists {
|
||||
muFiles.Unlock()
|
||||
} else {
|
||||
// this tree already added
|
||||
break
|
||||
}
|
||||
files[currPath] = &ExtendedFileInfo{
|
||||
Name: path.Base(currPath),
|
||||
Mode: basePermissionFolder,
|
||||
ModTime: time.Unix(0, 0),
|
||||
ChangeTime: time.Unix(0, 0),
|
||||
Size: 0,
|
||||
}
|
||||
muFiles.Unlock()
|
||||
}
|
||||
{
|
||||
dir, file := path.Split(absPath)
|
||||
|
|
@ -157,7 +156,7 @@ func (fs *S3Source) WarmingUp(targets []string) error {
|
|||
muFiles.Lock()
|
||||
files[absPath] = &ExtendedFileInfo{
|
||||
Name: path.Base(absPath),
|
||||
Mode: 0644,
|
||||
Mode: basePermissionFile,
|
||||
ModTime: obj.LastModified,
|
||||
ChangeTime: obj.LastModified,
|
||||
Size: obj.Size,
|
||||
|
|
@ -194,7 +193,7 @@ func (fs *S3Source) Lstat(name string) (*ExtendedFileInfo, error) {
|
|||
}
|
||||
|
||||
func (fs *S3Source) Join(elem ...string) string {
|
||||
return filepath.Join(elem...)
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
func (fs *S3Source) Separator() string {
|
||||
|
|
@ -202,7 +201,7 @@ func (fs *S3Source) Separator() string {
|
|||
}
|
||||
|
||||
func (fs *S3Source) IsAbs(p string) bool {
|
||||
return p[0] == '/'
|
||||
return path.IsAbs(p)
|
||||
}
|
||||
func (fs *S3Source) Abs(p string) (string, error) {
|
||||
return s3CleanPath(p), nil
|
||||
|
|
@ -237,22 +236,20 @@ var _ File = &s3SourceFile{}
|
|||
|
||||
func newS3SourceFile(name string, fi *ExtendedFileInfo, s3Client *minio.Client, filesInFolder []string, metadataOnly bool) (*s3SourceFile, error) {
|
||||
name = s3CleanPath(name)
|
||||
|
||||
if !metadataOnly {
|
||||
partPath := strings.Split(name, "/")
|
||||
// example /bucket-name
|
||||
bucketName := partPath[1]
|
||||
ctx := context.Background()
|
||||
objPath := path.Join(partPath[2:]...)
|
||||
if len(objPath) > 0 {
|
||||
object, err := s3Client.GetObject(ctx, bucketName, objPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, pathError("open file s3", name, os.ErrNotExist)
|
||||
}
|
||||
return &s3SourceFile{name: name, fi: fi, rc: object, filesInFolder: filesInFolder, s3Client: s3Client}, nil
|
||||
}
|
||||
if metadataOnly || fi.Mode.IsDir() {
|
||||
return &s3SourceFile{name: name, fi: fi, rc: nil, filesInFolder: filesInFolder, s3Client: s3Client}, nil
|
||||
}
|
||||
return &s3SourceFile{name: name, fi: fi, rc: nil, filesInFolder: filesInFolder, s3Client: s3Client}, nil
|
||||
|
||||
partPath := strings.Split(name, "/")
|
||||
// example /bucket-name
|
||||
bucketName := partPath[1]
|
||||
objPath := path.Join(partPath[2:]...)
|
||||
ctx := context.Background()
|
||||
object, err := s3Client.GetObject(ctx, bucketName, objPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, pathError("open file s3", name, os.ErrNotExist)
|
||||
}
|
||||
return &s3SourceFile{name: name, fi: fi, rc: object, filesInFolder: filesInFolder, s3Client: s3Client}, nil
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue