upstream/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-05-28 04:35:41 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into skip-unchanged-ignores-parent-dirs

Browse source
This commit is contained in:
Paulo Saraiva 2026-02-17 09:48:20 +01:00
commit e40975b447
164 changed files with 3400 additions and 1506 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/docker.yml vendored
View file

@ -26,10 +26,10 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Log in to the Container registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

14
.github/workflows/tests.yml vendored
View file

@ -48,10 +48,6 @@ jobs:
go: 1.24.x
os: ubuntu-latest
test_fuse: true
- job_name: Linux
go: 1.23.x
os: ubuntu-latest
test_fuse: true
name: ${{ matrix.job_name }} Go ${{ matrix.go }}
runs-on: ${{ matrix.os }}
@ -61,7 +57,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Set up Go ${{ matrix.go }}
uses: actions/setup-go@v6
@ -224,7 +220,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6
@ -246,7 +242,7 @@ jobs:
checks: write
steps:
- name: Check out code
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6
@ -254,7 +250,7 @@ jobs:
go-version: ${{ env.latest_go }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v8
uses: golangci/golangci-lint-action@v9
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v2.4.0
@ -291,7 +287,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Docker meta
id: meta

16
.golangci.yml
View file

@ -6,6 +6,8 @@ linters:
- asciicheck
# ensure that http response bodies are closed
- bodyclose
# restrict imports from other restic packages for internal/backend (cache exempt)
- depguard
- copyloopvar
# make sure all errors returned by functions are handled
- errcheck
@ -24,6 +26,20 @@ linters:
# find unused variables, functions, structs, types, etc.
- unused
settings:
depguard:
rules:
# Prevent backend packages from importing the internal/restic package to keep the architectural layers intact.
backend-imports:
files:
- "**/internal/backend/**"
- "!**/internal/backend/cache/**"
- "!**/internal/backend/test/**"
- "!**/*_test.go"
deny:
- pkg: "github.com/restic/restic/internal/restic"
desc: "internal/restic should not be imported to keep the architectural layers intact"
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact"
importas:
alias:
- pkg: github.com/restic/restic/internal/test

3
build.go
View file

@ -36,7 +36,6 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//go:build ignore_build_go
// +build ignore_build_go
package main
@ -60,7 +59,7 @@ var config = Config{
// see https://github.com/googleapis/google-cloud-go/issues/11448
DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported
MinVersion: GoVersion{Major: 1, Minor: 24, Patch: 0}, // minimum Go version supported
}
// Config configures the build.

9
changelog/unreleased/issue-3326 Normal file
View file

@ -0,0 +1,9 @@
Enhancement: `restic check` for specified snapshot(s) via snapshot filtering
Snapshots can now be specified for the command `restic check` on the command line
via the standard snapshot filter, (`--tag`, `--host`, `--path` or specifying
snapshot IDs directly) and will be used for checking the packfiles used by these snapshots.
https://github.com/restic/restic/issues/3326
https://github.com/restic/restic/pull/5469
https://github.com/restic/restic/pull/5644

12
changelog/unreleased/issue-4278 Normal file
View file

@ -0,0 +1,12 @@
Enhancement: Support include filters in `rewrite` command
The enhancement enables the standard include filter options
--iinclude pattern same as --include pattern but ignores the casing of filenames
--iinclude-file file same as --include-file but ignores casing of filenames in patterns
-i, --include pattern include a pattern (can be specified multiple times)
--include-file file read include patterns from a file (can be specified multiple times)
The exclusion or inclusion of filter parameters is exclusive, as in other commands.
https://github.com/restic/restic/issues/4278
https://github.com/restic/restic/pull/5191

1
changelog/unreleased/issue-5383
View file

@ -7,3 +7,4 @@ anything in the terminal with certain terminal emulators.
https://github.com/restic/restic/issues/5383
https://github.com/restic/restic/pull/5551
https://github.com/restic/restic/pull/5626

10
changelog/unreleased/issue-5453 Normal file
View file

@ -0,0 +1,10 @@
Enhancement: `copy` copies snapshots in batches
The `copy` command used to copy snapshots individually, even if this resulted in creating pack files
smaller than the target pack size. In particular, this resulted in many small files
when copying small incremental snapshots.
Now, `copy` copies multiple snapshots at once to avoid creating small files.
https://github.com/restic/restic/issues/5175
https://github.com/restic/restic/pull/5464

5
changelog/unreleased/pull-4938 Normal file
View file

@ -0,0 +1,5 @@
Change: Update dependencies and require Go 1.24 or newer
We have updated all dependencies. Restic now requires Go 1.24 or newer to build.
https://github.com/restic/restic/pull/5619

10
changelog/unreleased/pull-5465 Normal file
View file

@ -0,0 +1,10 @@
Bugfix: Correctly restore ACL inheritance state on Windows
Since the introduction of Security Descriptor backups in restic 0.17.0, the inheritance property of Access Control Entries (ACEs) was not restored correctly. This resulted in all restored permissions being marked as explicit (IsInherited: False), even if they were originally inherited from a parent folder.
The issue was caused by sending conflicting inheritance flags (PROTECTED_... and UNPROTECTED_...) to the Windows API during the restore process. The API would default to the more restrictive PROTECTED state, effectively disabling inheritance.
This has been fixed by ensuring that only the correct, non-conflicting inheritance flag is used when applying the security descriptor, preserving the original permission structure from the backup.
https://github.com/restic/restic/pull/5465
https://github.com/restic/restic/issues/5427

10
changelog/unreleased/pull-5588 Normal file
View file

@ -0,0 +1,10 @@
Enhancement: Display timezone information in snapshots output
The `snapshots` command now displays which timezone is being used to show
timestamps. Since snapshots can be created in different timezones but are
always displayed in the local timezone, a footer line is now shown indicating
the timezone used for display (e.g., "Timestamps shown in CET timezone").
This helps prevent confusion when comparing snapshots in a multi-user
environment.
https://github.com/restic/restic/pull/5588

7
changelog/unreleased/pull-5592 Normal file
View file

@ -0,0 +1,7 @@
Bugfix: Return error if `RESTIC_PACK_SIZE` contains invalid value
If the environment variable `RESTIC_PACK_SIZE` could not be parsed, then
restic ignored its value. Now, the restic commands fail with an error, unless
the command-line option `--pack-size` was specified.
https://github.com/restic/restic/pull/5592

7
changelog/unreleased/pull-5610 Normal file
View file

@ -0,0 +1,7 @@
Enhancement: reduce memory usage of check/copy/diff/stats commands
We have optimized the memory usage of the `check`, `copy`, `diff` and
`stats` commands. These now require less memory when processing large
snapshots.
https://github.com/restic/restic/pull/5610

35
cmd/restic/cmd_check.go
View file

@ -15,6 +15,7 @@ import (
"github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
@ -47,6 +48,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
summary, err := runCheck(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
if globalOptions.JSON {
if err != nil && summary.NumErrors == 0 {
@ -71,6 +73,7 @@ type CheckOptions struct {
ReadDataSubset string
CheckUnused bool
WithCache bool
data.SnapshotFilter
}
func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
@ -84,6 +87,7 @@ func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
panic(err)
}
f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
}
func checkFlags(opts CheckOptions) error {
@ -220,9 +224,6 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres
func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args []string, term ui.Terminal) (checkSummary, error) {
summary := checkSummary{MessageType: "summary"}
if len(args) != 0 {
return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
var printer progress.Printer
if !gopts.JSON {
@ -231,11 +232,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
printer = newJSONErrorPrinter(term)
}
readDataFilter, err := buildPacksFilter(opts, printer)
if err != nil {
return summary, err
}
cleanup := prepareCheckCache(opts, &gopts, printer)
defer cleanup()
@ -249,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
defer unlock()
chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(ctx)
err = chkr.LoadSnapshots(ctx, &opts.SnapshotFilter, args)
if err != nil {
return summary, err
}
@ -365,6 +361,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, ctx.Err()
}
// the following block only used for tests
if opts.CheckUnused {
unused, err := chkr.UnusedBlobs(ctx)
if err != nil {
@ -376,6 +373,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
}
}
readDataFilter, err := buildPacksFilter(opts, printer, chkr.IsFiltered())
if err != nil {
return summary, err
}
if readDataFilter != nil {
p := printer.NewCounter("packs")
errChan := make(chan error)
@ -416,11 +418,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, nil
}
func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) {
func buildPacksFilter(opts CheckOptions, printer progress.Printer,
filteredStatus bool) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) {
typeData := ""
if filteredStatus {
typeData = "filtered "
}
switch {
case opts.ReadData:
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read all data\n")
printer.P("read all %sdata", typeData)
return packs
}, nil
case opts.ReadDataSubset != "":
@ -431,7 +438,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
packCount := uint64(len(packs))
packs = selectPacksByBucket(packs, bucket, totalBuckets)
printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, len(packs), packCount, totalBuckets)
printer.P("read group #%d of %d %sdata packs (out of total %d packs in %d groups", bucket, len(packs), typeData, packCount, totalBuckets)
return packs
}, nil
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
@ -440,7 +447,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m
return nil, err
}
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read %.1f%% of data packs\n", percentage)
printer.P("read %.1f%% of %spackfiles", percentage, typeData)
return selectRandomPacksByPercentage(packs, percentage)
}, nil
}
@ -461,7 +468,7 @@ func buildPacksFilter(opts CheckOptions, printer progress.Printer) (func(packs m
if repoSize == 0 {
percentage = 100
}
printer.P("read %d bytes (%.1f%%) of data packs\n", subsetSize, percentage)
printer.P("read %d bytes (%.1f%%) of %sdata packs\n", subsetSize, percentage, typeData)
return packs
}, nil
}

65
cmd/restic/cmd_check_integration_test.go
View file

@ -2,6 +2,7 @@ package main
import (
"context"
"strings"
"testing"
"github.com/restic/restic/internal/global"
@ -34,3 +35,67 @@ func testRunCheckOutput(t testing.TB, gopts global.Options, checkUnused bool) (s
})
return buf.String(), err
}
func testRunCheckOutputWithOpts(t testing.TB, gopts global.Options, opts CheckOptions, args []string) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
gopts.Verbosity = 2
_, err := runCheck(context.TODO(), opts, gopts, args, gopts.Term)
return err
})
return buf.String(), err
}
func TestCheckWithSnaphotFilter(t *testing.T) {
testCases := []struct {
opts CheckOptions
args []string
expectedOutput string
}{
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"4 / 4 packs",
},
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"2 / 2 snapshots",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"2 / 2 packs",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"1 / 1 snapshots",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"1 / 1 packs",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"filtered",
},
}
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
opts := BackupOptions{}
testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
testRunBackup(t, env.testdata+"/0", []string{"0/9"}, opts, env.gopts)
for _, testCase := range testCases {
output, err := testRunCheckOutputWithOpts(t, env.gopts, testCase.opts, testCase.args)
rtest.OK(t, err)
hasOutput := strings.Contains(output, testCase.expectedOutput)
rtest.Assert(t, hasOutput, `expected to find substring %q, but did not find it`, testCase.expectedOutput)
}
}

246
cmd/restic/cmd_copy.go
View file

@ -3,6 +3,9 @@ package main
import (
"context"
"fmt"
"iter"
"sync"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
@ -12,7 +15,6 @@ import (
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -70,6 +72,39 @@ func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
}
// collectAllSnapshots: select all snapshot trees to be copied
func collectAllSnapshots(ctx context.Context, opts CopyOptions,
srcSnapshotLister restic.Lister, srcRepo restic.Repository,
dstSnapshotByOriginal map[restic.ID][]*data.Snapshot, args []string, printer progress.Printer,
) iter.Seq[*data.Snapshot] {
return func(yield func(*data.Snapshot) bool) {
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
printer.V("\n%v", sn)
printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
if !yield(sn) {
return
}
}
}
}
func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
secondaryGopts, isFromRepo, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "destination")
@ -124,49 +159,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
return ctx.Err()
}
// remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet()
selectedSnapshots := collectAllSnapshots(ctx, opts, srcSnapshotLister, srcRepo, dstSnapshotByOriginal, args, printer)
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
printer.V("\n%v", sn)
printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
printer.P("\n%v", sn)
printer.P(" copy started, this may take a while...")
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer); err != nil {
return err
}
debug.Log("tree copied")
// save snapshot
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := data.SaveSnapshot(ctx, dstRepo, sn)
if err != nil {
return err
}
printer.P("snapshot %s saved", newID.Str())
if err := copyTreeBatched(ctx, srcRepo, dstRepo, selectedSnapshots, printer); err != nil {
return err
}
return ctx.Err()
}
@ -189,75 +187,136 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
return true
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.IDSet, rootTreeID restic.ID, printer progress.Printer) error {
// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after
// data equivalent to at least 10 packfiles was written.
func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error {
wg, wgCtx := errgroup.WithContext(ctx)
// remember already processed trees across all snapshots
visitedTrees := srcRepo.NewAssociatedBlobSet()
treeStream := data.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
visited := visitedTrees.Has(treeID)
visitedTrees.Insert(treeID)
return visited
}, nil)
targetSize := uint64(dstRepo.PackSize()) * 100
minDuration := 1 * time.Minute
copyBlobs := restic.NewBlobSet()
packList := restic.NewIDSet()
// use pull-based iterator to allow iteration in multiple steps
next, stop := iter.Pull(selectedSnapshots)
defer stop()
enqueue := func(h restic.BlobHandle) {
pb := srcRepo.LookupBlob(h.Type, h.ID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
for {
var batch []*data.Snapshot
batchSize := uint64(0)
startTime := time.Now()
// call WithBlobUploader() once and then loop over all selectedSnapshots
err := dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for batchSize < targetSize || time.Since(startTime) < minDuration {
sn, ok := next()
if !ok {
break
}
batch = append(batch, sn)
printer.P("\n%v", sn)
printer.P(" copy started, this may take a while...")
sizeBlobs, err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer, uploader)
if err != nil {
return err
}
debug.Log("tree copied")
batchSize += sizeBlobs
}
return nil
})
if err != nil {
return err
}
// if no snapshots were processed in this batch, we're done
if len(batch) == 0 {
break
}
// add a newline to separate saved snapshot messages from the other messages
if len(batch) > 1 {
printer.P("")
}
// save all the snapshots
for _, sn := range batch {
err := copySaveSnapshot(ctx, sn, dstRepo, printer)
if err != nil {
return err
}
}
}
wg.Go(func() error {
for tree := range treeStream {
if tree.Error != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
}
return nil
}
// Do we already have this tree blob?
treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(treeHandle)
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) {
for _, entry := range tree.Nodes {
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range entry.Content {
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
enqueue(h)
}
}
copyBlobs := srcRepo.NewAssociatedBlobSet()
packList := restic.NewIDSet()
var lock sync.Mutex
enqueue := func(h restic.BlobHandle) {
lock.Lock()
defer lock.Unlock()
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
pb := srcRepo.LookupBlob(h.Type, h.ID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
}
}
}
err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool {
handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
visited := visitedTrees.Has(handle)
visitedTrees.Insert(handle)
return visited
}, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error {
if err != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
}
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(restic.BlobHandle{ID: treeID, Type: restic.TreeBlob})
for item := range nodes {
if item.Error != nil {
return item.Error
}
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range item.Node.Content {
enqueue(restic.BlobHandle{Type: restic.DataBlob, ID: blobID})
}
}
return nil
})
err := wg.Wait()
if err != nil {
return err
return 0, err
}
copyStats(srcRepo, copyBlobs, packList, printer)
sizeBlobs := copyStats(srcRepo, copyBlobs, packList, printer)
bar := printer.NewCounter("packs copied")
err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar, printer.P)
err = repository.CopyBlobs(ctx, srcRepo, dstRepo, uploader, packList, copyBlobs, bar, printer.P)
if err != nil {
return errors.Fatalf("%s", err)
return 0, errors.Fatalf("%s", err)
}
return nil
return sizeBlobs, nil
}
// copyStats: print statistics for the blobs to be copied
func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList restic.IDSet, printer progress.Printer) {
func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
// count and size
countBlobs := 0
sizeBlobs := uint64(0)
for blob := range copyBlobs {
for blob := range copyBlobs.Keys() {
for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
countBlobs++
sizeBlobs += uint64(blob.Length)
@ -267,4 +326,19 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList res
printer.V(" copy %d blobs with disk size %s in %d packfiles\n",
countBlobs, ui.FormatBytes(uint64(sizeBlobs)), len(packList))
return sizeBlobs
}
func copySaveSnapshot(ctx context.Context, sn *data.Snapshot, dstRepo restic.Repository, printer progress.Printer) error {
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := data.SaveSnapshot(ctx, dstRepo, sn)
if err != nil {
return err
}
printer.P("snapshot %s saved, copied from source snapshot %s", newID.Str(), sn.ID().Str())
return nil
}

37
cmd/restic/cmd_copy_integration_test.go
View file

@ -7,7 +7,9 @@ import (
"testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func testRunCopy(t testing.TB, srcGopts global.Options, dstGopts global.Options) {
@ -83,6 +85,41 @@ func TestCopy(t *testing.T) {
}
rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots")
// check that snapshots were properly batched while copying
_, _, countBlobs := testPackAndBlobCounts(t, env.gopts)
countTreePacksDst, countDataPacksDst, countBlobsDst := testPackAndBlobCounts(t, env2.gopts)
rtest.Equals(t, countBlobs, countBlobsDst, "expected blob count in boths repos to be equal")
rtest.Equals(t, countTreePacksDst, 1, "expected 1 tree packfile")
rtest.Equals(t, countDataPacksDst, 1, "expected 1 data packfile")
}
func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks int, countDataPacks int, countBlobs int) {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
switch blobs[0].Type {
case restic.TreeBlob:
countTreePacks++
case restic.DataBlob:
countDataPacks++
}
countBlobs += len(blobs)
return nil
}))
return nil
}))
return countTreePacks, countDataPacks, countBlobs
}
func TestCopyIncremental(t *testing.T) {

3
cmd/restic/cmd_debug.go
View file

@ -1,5 +1,4 @@
//go:build debug
// +build debug
package main
@ -353,7 +352,7 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) {

91
cmd/restic/cmd_diff.go
View file

@ -5,7 +5,6 @@ import (
"encoding/json"
"path"
"reflect"
"sort"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
@ -124,7 +123,7 @@ func (s *DiffStat) Add(node *data.Node) {
}
// addBlobs adds the blobs of node to s.
func addBlobs(bs restic.BlobSet, node *data.Node) {
func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
if node == nil {
return
}
@ -148,18 +147,18 @@ func addBlobs(bs restic.BlobSet, node *data.Node) {
}
type DiffStatsContainer struct {
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"`
}
// updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, printError func(string, ...interface{})) {
for h := range blobs {
func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) {
for h := range blobs.Keys() {
switch h.Type {
case restic.DataBlob:
stats.DataBlobs++
@ -177,18 +176,21 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, prin
}
}
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error {
debug.Log("print %v tree %v", mode, id)
tree, err := data.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
for _, node := range tree.Nodes {
for item := range tree {
if item.Error != nil {
return item.Error
}
if ctx.Err() != nil {
return ctx.Err()
}
node := item.Node
name := path.Join(prefix, node.Name)
if node.Type == data.NodeTypeDir {
name += "/"
@ -208,18 +210,22 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
return ctx.Err()
}
func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error {
debug.Log("print tree %v", id)
tree, err := data.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
for _, node := range tree.Nodes {
for item := range tree {
if item.Error != nil {
return item.Error
}
if ctx.Err() != nil {
return ctx.Err()
}
node := item.Node
addBlobs(blobs, node)
if node.Type == data.NodeTypeDir {
@ -233,29 +239,6 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest
return ctx.Err()
}
func uniqueNodeNames(tree1, tree2 *data.Tree) (tree1Nodes, tree2Nodes map[string]*data.Node, uniqueNames []string) {
names := make(map[string]struct{})
tree1Nodes = make(map[string]*data.Node)
for _, node := range tree1.Nodes {
tree1Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
tree2Nodes = make(map[string]*data.Node)
for _, node := range tree2.Nodes {
tree2Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
uniqueNames = make([]string, 0, len(names))
for name := range names {
uniqueNames = append(uniqueNames, name)
}
sort.Strings(uniqueNames)
return tree1Nodes, tree2Nodes, uniqueNames
}
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
debug.Log("diffing %v to %v", id1, id2)
tree1, err := data.LoadTree(ctx, c.repo, id1)
@ -268,21 +251,29 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return err
}
tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)
for _, name := range names {
for dt := range data.DualTreeIterator(tree1, tree2) {
if dt.Error != nil {
return dt.Error
}
if ctx.Err() != nil {
return ctx.Err()
}
node1, t1 := tree1Nodes[name]
node2, t2 := tree2Nodes[name]
node1 := dt.Tree1
node2 := dt.Tree2
var name string
if node1 != nil {
name = node1.Name
} else {
name = node2.Name
}
addBlobs(stats.BlobsBefore, node1)
addBlobs(stats.BlobsAfter, node2)
switch {
case t1 && t2:
case node1 != nil && node2 != nil:
name := path.Join(prefix, name)
mod := ""
@ -328,7 +319,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
c.printError("error: %v", err)
}
}
case t1 && !t2:
case node1 != nil && node2 == nil:
prefix := path.Join(prefix, name)
if node1.Type == data.NodeTypeDir {
prefix += "/"
@ -342,7 +333,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
c.printError("error: %v", err)
}
}
case !t1 && t2:
case node1 == nil && node2 != nil:
prefix := path.Join(prefix, name)
if node2.Type == data.NodeTypeDir {
prefix += "/"
@ -442,9 +433,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
MessageType: "statistics",
SourceSnapshot: args[0],
TargetSnapshot: args[1],
BlobsBefore: restic.NewBlobSet(),
BlobsAfter: restic.NewBlobSet(),
BlobsCommon: restic.NewBlobSet(),
BlobsBefore: repo.NewAssociatedBlobSet(),
BlobsAfter: repo.NewAssociatedBlobSet(),
BlobsCommon: repo.NewAssociatedBlobSet(),
}
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})

15
cmd/restic/cmd_dump.go
View file

@ -80,7 +80,7 @@ func splitPath(p string) []string {
return append(s, f)
}
func printFromTree(ctx context.Context, tree *data.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
// If we print / we need to assume that there are multiple nodes at that
// level in the tree.
if pathComponents[0] == "" {
@ -92,11 +92,14 @@ func printFromTree(ctx context.Context, tree *data.Tree, repo restic.BlobLoader,
item := filepath.Join(prefix, pathComponents[0])
l := len(pathComponents)
for _, node := range tree.Nodes {
for it := range tree {
if it.Error != nil {
return it.Error
}
if ctx.Err() != nil {
return ctx.Err()
}
node := it.Node
// If dumping something in the highest level it will just take the
// first item it finds and dump that according to the switch case below.
if node.Name == pathComponents[0] {
@ -154,11 +157,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
}
defer unlock()
sn, subfolder, err := (&data.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo, repo, snapshotIDString)
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

6
cmd/restic/cmd_ls.go
View file

@ -406,11 +406,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
}
}
sn, subfolder, err := (&data.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, snapshotLister, repo, args[0])
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, snapshotLister, repo, args[0])
if err != nil {
return err
}

1
cmd/restic/cmd_mount.go
View file

@ -1,5 +1,4 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package main

1
cmd/restic/cmd_mount_disabled.go
View file

@ -1,5 +1,4 @@
//go:build !darwin && !freebsd && !linux
// +build !darwin,!freebsd,!linux
package main

1
cmd/restic/cmd_mount_integration_test.go
View file

@ -1,5 +1,4 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package main

46
cmd/restic/cmd_recover.go
View file

@ -97,7 +97,11 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
continue
}
for _, node := range tree.Nodes {
for item := range tree {
if item.Error != nil {
return item.Error
}
node := item.Node
if node.Type == data.NodeTypeDir && node.Subtree != nil {
trees[*node.Subtree] = true
}
@ -134,28 +138,28 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
return ctx.Err()
}
tree := data.NewTree(len(roots))
for id := range roots {
var subtreeID = id
node := data.Node{
Type: data.NodeTypeDir,
Name: id.Str(),
Mode: 0755,
Subtree: &subtreeID,
AccessTime: time.Now(),
ModTime: time.Now(),
ChangeTime: time.Now(),
}
err := tree.Insert(&node)
if err != nil {
return err
}
}
var treeID restic.ID
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
var err error
treeID, err = data.SaveTree(ctx, uploader, tree)
tw := data.NewTreeWriter(uploader)
for id := range roots {
var subtreeID = id
node := data.Node{
Type: data.NodeTypeDir,
Name: id.Str(),
Mode: 0755,
Subtree: &subtreeID,
AccessTime: time.Now(),
ModTime: time.Now(),
ChangeTime: time.Now(),
}
err := tw.AddNode(&node)
if err != nil {
return err
}
}
treeID, err = tw.Finalize(ctx)
if err != nil {
return errors.Fatalf("unable to save new tree to the repository: %v", err)
}

7
cmd/restic/cmd_repair_snapshots.go
View file

@ -2,6 +2,7 @@ package main
import (
"context"
"slices"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
@ -130,7 +131,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
node.Size = newSize
return node
},
RewriteFailedTree: func(_ restic.ID, path string, _ error) (*data.Tree, error) {
RewriteFailedTree: func(_ restic.ID, path string, _ error) (data.TreeNodeIterator, error) {
if path == "/" {
printer.P(" dir %q: not readable", path)
// remove snapshots with invalid root node
@ -138,7 +139,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
}
// If a subtree fails to load, remove it
printer.P(" dir %q: replaced with empty directory", path)
return &data.Tree{}, nil
return slices.Values([]data.NodeOrError{}), nil
},
AllowUnstableSerialization: true,
})
@ -150,7 +151,7 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, uploader, "/", *sn.Tree)
return id, nil, err
}, opts.DryRun, opts.Forget, nil, "repaired", printer)
}, opts.DryRun, opts.Forget, nil, "repaired", printer, false)
if err != nil {
return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
}

6
cmd/restic/cmd_restore.go
View file

@ -151,11 +151,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
}
defer unlock()
sn, subfolder, err := (&data.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo, repo, snapshotIDString)
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

129
cmd/restic/cmd_rewrite.go
View file

@ -30,6 +30,9 @@ The "rewrite" command excludes files from existing snapshots. It creates new
snapshots containing the same data as the original ones, but without the files
you specify to exclude. All metadata (time, host, tags) will be preserved.
Alternatively you can use one of the --include variants to only include files
in the new snapshot which you want to preserve.
The snapshots to rewrite are specified using the --host, --tag and --path options,
or by providing a list of snapshot IDs. Please note that specifying neither any of
these options nor a snapshot ID will cause the command to rewrite all snapshots.
@ -46,8 +49,8 @@ When rewrite is used with the --snapshot-summary option, a new snapshot is
created containing statistics summary data. Only two fields in the summary will
be non-zero: TotalFilesProcessed and TotalBytesProcessed.
When rewrite is called with one of the --exclude options, TotalFilesProcessed
and TotalBytesProcessed will be updated in the snapshot summary.
When rewrite is called with one of the --exclude or --include options,
TotalFilesProcessed and TotalBytesProcessed will be updated in the snapshot summary.
EXIT STATUS
===========
@ -109,6 +112,7 @@ type RewriteOptions struct {
Metadata snapshotMetadataArgs
data.SnapshotFilter
filter.ExcludePatternOptions
filter.IncludePatternOptions
}
func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
@ -120,6 +124,7 @@ func (opts *RewriteOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
opts.ExcludePatternOptions.Add(f)
opts.IncludePatternOptions.Add(f)
}
// rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will
@ -136,33 +141,31 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *data.
return false, err
}
includeByNameFuncs, err := opts.IncludePatternOptions.CollectPatterns(printer.E)
if err != nil {
return false, err
}
metadata, err := opts.Metadata.convert()
if err != nil {
return false, err
}
condInclude := len(includeByNameFuncs) > 0
condExclude := len(rejectByNameFuncs) > 0
var filter rewriteFilterFunc
if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary {
selectByName := func(nodepath string) bool {
for _, reject := range rejectByNameFuncs {
if reject(nodepath) {
return false
}
}
return true
if condInclude || condExclude || opts.SnapshotSummary {
var rewriteNode walker.NodeRewriteFunc
var keepEmptyDirectoryFunc walker.NodeKeepEmptyDirectoryFunc
if condInclude {
rewriteNode, keepEmptyDirectoryFunc = gatherIncludeFilters(includeByNameFuncs, printer)
} else {
rewriteNode = gatherExcludeFilters(rejectByNameFuncs, printer)
}
rewriteNode := func(node *data.Node, path string) *data.Node {
if selectByName(path) {
return node
}
printer.P("excluding %s", path)
return nil
}
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode)
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode, keepEmptyDirectoryFunc)
filter = func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, uploader, "/", *sn.Tree)
@ -186,15 +189,16 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *data.
}
return filterAndReplaceSnapshot(ctx, repo, sn,
filter, opts.DryRun, opts.Forget, metadata, "rewrite", printer)
filter, opts.DryRun, opts.Forget, metadata, "rewrite", printer, len(includeByNameFuncs) > 0)
}
func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *data.Snapshot,
filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string, printer progress.Printer) (bool, error) {
filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string, printer progress.Printer,
keepEmptySnapshot bool) (bool, error) {
var filteredTree restic.ID
var summary *data.SnapshotSummary
err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
var err error
filteredTree, summary, err = filter(ctx, sn, uploader)
return err
@ -204,6 +208,10 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *d
}
if filteredTree.IsNull() {
if keepEmptySnapshot {
debug.Log("Snapshot %v not modified", sn)
return false, nil
}
if dryRun {
printer.P("would delete empty snapshot")
} else {
@ -284,8 +292,12 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *d
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts global.Options, args []string, term ui.Terminal) error {
if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
hasExcludes := !opts.ExcludePatternOptions.Empty()
hasIncludes := !opts.IncludePatternOptions.Empty()
if !opts.SnapshotSummary && !hasExcludes && !hasIncludes && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes/includes provided and no new metadata provided")
} else if hasExcludes && hasIncludes {
return errors.Fatal("exclude and include patterns are mutually exclusive")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@ -348,3 +360,72 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts global.Options,
return nil
}
func gatherIncludeFilters(includeByNameFuncs []filter.IncludeByNameFunc, printer progress.Printer) (rewriteNode walker.NodeRewriteFunc, keepEmptyDirectory walker.NodeKeepEmptyDirectoryFunc) {
inSelectByName := func(nodepath string, node *data.Node) bool {
for _, include := range includeByNameFuncs {
matched, childMayMatch := include(nodepath)
if node.Type == data.NodeTypeDir {
// include directories if they or some of their children may be included
if matched || childMayMatch {
return true
}
} else if matched {
return true
}
}
return false
}
rewriteNode = func(node *data.Node, path string) *data.Node {
if inSelectByName(path, node) {
if node.Type != data.NodeTypeDir {
printer.VV("including %q\n", path)
}
return node
}
return nil
}
inSelectByNameDir := func(nodepath string) bool {
for _, include := range includeByNameFuncs {
matched, _ := include(nodepath)
if matched {
return matched
}
}
return false
}
keepEmptyDirectory = func(path string) bool {
keep := inSelectByNameDir(path)
if keep {
printer.VV("including directory %q\n", path)
}
return keep
}
return rewriteNode, keepEmptyDirectory
}
func gatherExcludeFilters(excludeByNameFuncs []filter.RejectByNameFunc, printer progress.Printer) (rewriteNode walker.NodeRewriteFunc) {
exSelectByName := func(nodepath string) bool {
for _, reject := range excludeByNameFuncs {
if reject(nodepath) {
return false
}
}
return true
}
rewriteNode = func(node *data.Node, path string) *data.Node {
if exSelectByName(path) {
return node
}
printer.VV("excluding %q\n", path)
return nil
}
return rewriteNode
}

156
cmd/restic/cmd_rewrite_integration_test.go
View file

@ -2,7 +2,9 @@ package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/restic/restic/internal/data"
@ -27,6 +29,27 @@ func testRunRewriteExclude(t testing.TB, gopts global.Options, excludes []string
}))
}
func testRunRewriteWithOpts(t testing.TB, opts RewriteOptions, gopts global.Options, args []string) error {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runRewrite(context.TODO(), opts, gopts, args, gopts.Term)
}))
return nil
}
// testLsOutputContainsCount runs restic ls with the given options and asserts that
// exactly expectedCount lines of the output contain substring.
func testLsOutputContainsCount(t testing.TB, gopts global.Options, lsOpts LsOptions, lsArgs []string, substring string, expectedCount int) {
t.Helper()
out := testRunLsWithOpts(t, gopts, lsOpts, lsArgs)
count := 0
for _, line := range strings.Split(string(out), "\n") {
if strings.Contains(line, substring) {
count++
}
}
rtest.Assert(t, count == expectedCount, "expected %d lines containing %q, but got %d", expectedCount, substring, count)
}
func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID {
testSetupBackupData(t, env)
@ -39,6 +62,20 @@ func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID {
return snapshotIDs[0]
}
func createBasicRewriteRepoWithEmptyDirectory(t testing.TB, env *testEnvironment) restic.ID {
testSetupBackupData(t, env)
// make an empty directory named "empty-directory"
rtest.OK(t, os.Mkdir(filepath.Join(env.testdata, "/0/tests", "empty-directory"), 0755))
// create backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
snapshotIDs := testRunList(t, env.gopts, "snapshots")
rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs)
return snapshotIDs[0]
}
func getSnapshot(t testing.TB, snapshotID restic.ID, env *testEnvironment) *data.Snapshot {
t.Helper()
@ -195,3 +232,122 @@ func TestRewriteSnaphotSummary(t *testing.T) {
rtest.Equals(t, oldSummary.TotalBytesProcessed, newSn.Summary.TotalBytesProcessed, "unexpected TotalBytesProcessed value")
rtest.Equals(t, oldSummary.TotalFilesProcessed, newSn.Summary.TotalFilesProcessed, "unexpected TotalFilesProcessed value")
}
func TestRewriteInclude(t *testing.T) {
for _, tc := range []struct {
name string
opts RewriteOptions
lsSubstring string
lsExpectedCount int
summaryFilesExpected uint
}{
{"relative", RewriteOptions{
Forget: true,
IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*.txt"}},
}, ".txt", 2, 2},
{"absolute", RewriteOptions{
Forget: true,
// test that childMatches are working by only matching a subdirectory
IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"/testdata/0/for_cmd_ls"}},
}, "/testdata/0", 5, 3},
} {
t.Run(tc.name, func(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createBasicRewriteRepo(t, env)
snapshots := testListSnapshots(t, env.gopts, 1)
rtest.OK(t, testRunRewriteWithOpts(t, tc.opts, env.gopts, []string{"latest"}))
newSnapshots := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, snapshots[0] != newSnapshots[0], "snapshot id should have changed")
testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, tc.lsSubstring, tc.lsExpectedCount)
sn := testLoadSnapshot(t, env.gopts, newSnapshots[0])
rtest.Assert(t, sn.Summary != nil, "snapshot should have a summary attached")
rtest.Assert(t, sn.Summary.TotalFilesProcessed == tc.summaryFilesExpected,
"there should be %d files in the snapshot, but there are %d files", tc.summaryFilesExpected, sn.Summary.TotalFilesProcessed)
})
}
}
func TestRewriteExcludeFiles(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createBasicRewriteRepo(t, env)
snapshots := testListSnapshots(t, env.gopts, 1)
// exclude txt files
err := testRunRewriteWithOpts(t,
RewriteOptions{
Forget: true,
ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*.txt"}},
},
env.gopts,
[]string{"latest"})
rtest.OK(t, err)
newSnapshots := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, snapshots[0] != newSnapshots[0], "snapshot id should have changed")
testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, ".txt", 0)
}
func TestRewriteExcludeIncludeContradiction(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
// test contradiction
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runRewrite(ctx,
RewriteOptions{
ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"nonsense"}},
IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"not allowed"}},
},
gopts, []string{"quack"}, env.gopts.Term)
})
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"), `expected to fail command with message "exclude and include patterns are mutually exclusive"`)
}
func TestRewriteIncludeEmptyDirectory(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
snapIDEmpty := createBasicRewriteRepoWithEmptyDirectory(t, env)
// restic rewrite <snapshots[0]> -i empty-directory --forget
// exclude txt files
err := testRunRewriteWithOpts(t,
RewriteOptions{
Forget: true,
IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"empty-directory"}},
},
env.gopts,
[]string{"latest"})
rtest.OK(t, err)
newSnapshots := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, snapIDEmpty != newSnapshots[0], "snapshot id should have changed")
testLsOutputContainsCount(t, env.gopts, LsOptions{}, []string{"latest"}, "empty-directory", 1)
}
// TestRewriteIncludeNothing makes sure when nothing is included, the original snapshot stays untouched
func TestRewriteIncludeNothing(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createBasicRewriteRepo(t, env)
snapsBefore := testListSnapshots(t, env.gopts, 1)
// restic rewrite latest -i nothing-whatsoever --forget
err := testRunRewriteWithOpts(t,
RewriteOptions{
Forget: true,
IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"nothing-whatsoever"}},
},
env.gopts,
[]string{"latest"})
rtest.OK(t, err)
snapsAfter := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, snapsBefore[0] == snapsAfter[0], "snapshots should be identical but are %s and %s",
snapsBefore[0].Str(), snapsAfter[0].Str())
}

11
cmd/restic/cmd_snapshots.go
View file

@ -7,6 +7,7 @@ import (
"io"
"sort"
"strings"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/global"
@ -240,7 +241,15 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe
tab.AddRow(data)
}
tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))
// Add timezone information to prevent confusion:
// Each snapshot can be registered in different timezones,
// but we display them all in local timezone on this output.
footer := fmt.Sprintf("%d snapshots", len(list))
zoneName, _ := time.Now().Local().Zone()
if zoneName != "" {
footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer)
}
tab.AddFooter(footer)
if multiline {
// print an additional blank line between snapshots

6
cmd/restic/cmd_stats.go
View file

@ -130,7 +130,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
stats := &statsContainer{
uniqueFiles: make(map[fileID]struct{}),
fileBlobs: make(map[string]restic.IDSet),
blobs: restic.NewBlobSet(),
blobs: repo.NewAssociatedBlobSet(),
SnapshotsCount: 0,
}
@ -146,7 +146,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
if opts.countMode == countModeRawData {
// the blob handles have been collected, but not yet counted
for blobHandle := range stats.blobs {
for blobHandle := range stats.blobs.Keys() {
pbs := repo.LookupBlob(blobHandle.Type, blobHandle.ID)
if len(pbs) == 0 {
return fmt.Errorf("blob %v not found", blobHandle)
@ -350,7 +350,7 @@ type statsContainer struct {
// blobs is used to count individual unique blobs,
// independent of references to files
blobs restic.BlobSet
blobs restic.AssociatedBlobSet
}
// fileID is a 256-bit hash that distinguishes unique files.

2
cmd/restic/cmd_unlock.go
View file

@ -19,6 +19,8 @@ func newUnlockCommand(globalOptions *global.Options) *cobra.Command {
Long: `
The "unlock" command removes stale locks that have been created by other restic processes.
Removing locks works even with repositories served in append-only mode from restic's rest-server.
EXIT STATUS
===========

1
cmd/restic/integration_helpers_unix_test.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package main

1
cmd/restic/integration_helpers_windows_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package main

6
cmd/restic/main.go
View file

@ -49,6 +49,10 @@ The full documentation can be found at https://restic.readthedocs.io/ .
DisableAutoGenTag: true,
PersistentPreRunE: func(c *cobra.Command, _ []string) error {
switch c.Name() {
case "__complete", "__completeNoDesc":
return nil
}
return globalOptions.PreRun(needsPassword(c.Name()))
},
}
@ -114,7 +118,7 @@ The full documentation can be found at https://restic.readthedocs.io/ .
// user for authentication).
func needsPassword(cmd string) bool {
switch cmd {
case "cache", "generate", "help", "options", "self-update", "version", "__complete":
case "cache", "generate", "help", "options", "self-update", "version", "__complete", "__completeNoDesc":
return false
default:
return true

2
doc/020_installation.rst
View file

@ -315,7 +315,7 @@ From Source
***********
restic is written in the Go programming language and you need at least
Go version 1.23. Building restic may also work with older versions of Go,
Go version 1.24. Building restic may also work with older versions of Go,
but that's not supported. See the `Getting
started <https://go.dev/doc/install>`__ guide of the Go project for
instructions how to install Go.

85
doc/040_backup.rst
View file

@ -710,87 +710,4 @@ Environment Variables
*********************
In addition to command-line options, restic supports passing various options in
environment variables. The following lists these environment variables:
.. code-block:: console
RESTIC_REPOSITORY_FILE Name of file containing the repository location (replaces --repository-file)
RESTIC_REPOSITORY Location of repository (replaces -r)
RESTIC_PASSWORD_FILE Location of password file (replaces --password-file)
RESTIC_PASSWORD The actual password for the repository
RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout
RESTIC_KEY_HINT ID of key to try decrypting first, before other keys
RESTIC_CACERT Location(s) of certificate file(s), comma separated if multiple (replaces --cacert)
RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert)
RESTIC_CACHE_DIR Location of the cache directory
RESTIC_COMPRESSION Compression mode (only available for repository format version 2)
RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host)
RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated
RESTIC_PACK_SIZE Target size for pack files
RESTIC_READ_CONCURRENCY Concurrency for file reads
TMPDIR Location for temporary files (except Windows)
TMP Location for temporary files (only Windows)
AWS_ACCESS_KEY_ID Amazon S3 access key ID
AWS_SECRET_ACCESS_KEY Amazon S3 secret access key
AWS_SESSION_TOKEN Amazon S3 temporary session token
AWS_DEFAULT_REGION Amazon S3 default region
AWS_PROFILE Amazon credentials profile (alternative to specifying key and region)
AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials)
RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials
RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption
RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption
RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy
RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1)
RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only.
AZURE_ACCOUNT_NAME Account name for Azure
AZURE_ACCOUNT_KEY Account key for Azure
AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure
AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net)
AZURE_FORCE_CLI_CREDENTIAL Force the use of Azure CLI credentials for authentication
B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2
B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2
GOOGLE_PROJECT_ID Project ID for Google Cloud Storage
GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json)
OS_AUTH_URL Auth URL for keystone authentication
OS_REGION_NAME Region name for keystone authentication
OS_USERNAME Username for keystone authentication
OS_USER_ID User ID for keystone v3 authentication
OS_PASSWORD Password for keystone authentication
OS_TENANT_ID Tenant ID for keystone v2 authentication
OS_TENANT_NAME Tenant name for keystone v2 authentication
OS_USER_DOMAIN_NAME User domain name for keystone authentication
OS_USER_DOMAIN_ID User domain ID for keystone v3 authentication
OS_PROJECT_NAME Project name for keystone authentication
OS_PROJECT_DOMAIN_NAME Project domain name for keystone authentication
OS_PROJECT_DOMAIN_ID Project domain ID for keystone v3 authentication
OS_TRUST_ID Trust ID for keystone v3 authentication
OS_APPLICATION_CREDENTIAL_ID Application Credential ID (keystone v3)
OS_APPLICATION_CREDENTIAL_NAME Application Credential Name (keystone v3)
OS_APPLICATION_CREDENTIAL_SECRET Application Credential Secret (keystone v3)
OS_STORAGE_URL Storage URL for token authentication
OS_AUTH_TOKEN Auth token for token authentication
RCLONE_BWLIMIT rclone bandwidth limit
RESTIC_REST_USERNAME Restic REST Server username
RESTIC_REST_PASSWORD Restic REST Server password
ST_AUTH Auth URL for keystone v1 authentication
ST_USER Username for keystone v1 authentication
ST_KEY Password for keystone v1 authentication
See :ref:`caching` for the rules concerning cache locations when
``RESTIC_CACHE_DIR`` is not set.
The external programs that restic may execute include ``rclone`` (for rclone
backends) and ``ssh`` (for the SFTP backend). These may respond to further
environment variables and configuration files; see their respective manuals.
environment variables. See :ref:`environment-variables` for a list.

22
doc/045_working_with_repos.rst
View file

@ -205,21 +205,28 @@ example from a local to a remote repository, you can use the ``copy`` command:
.. code-block:: console
$ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo
$ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo --verbose
repository d6504c63 opened successfully
repository 3dd0878c opened successfully
[0:00] 100.00% 2 / 2 index files loaded
[0:00] 100.00% 7 / 7 index files loaded
snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir
copy started, this may take a while...
snapshot 7a746a07 saved
[0:00] 100.00% 13 / 13 packs copied
snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir
skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7
snapshot 7a746a07 saved, copied from source snapshot 410b18a2
The example command copies all snapshots from the source repository
``/srv/restic-repo`` to the destination repository ``/srv/restic-repo-copy``.
Snapshots which have previously been copied between repositories will
be skipped by later copy runs.
be skipped by later copy runs. Information about skipped snapshots is only
printed when ``--verbose`` is passed to the command. For efficiency reasons,
the snapshots are copied in batches, such that the ``snapshot [...] saved``
messages can appear some time after the snapshot content was copied.
.. important:: This process will have to both download (read) and upload (write)
the entire snapshot(s) due to the different encryption keys used in the
@ -329,6 +336,13 @@ The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and
``--iexclude-file`` are supported. They behave the same way as for the backup
command, see :ref:`backup-excluding-files` for details.
The options ``--include``, ``--include-file``, ``--iinclude`` and
``--iinclude-file`` are supported as well.
The ``--include`` variants allow you to reduce an existing snapshot or a set of snapshots
to those files that you are really interested in. An example could be all pictures
from a snapshot:
``restic rewrite -r ... --iinclude "*.jpg" --iinclude "*.jpeg" --iinclude "*.png"``.
It is possible to rewrite only a subset of snapshots by filtering them the same
way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
@ -353,7 +367,7 @@ modifying the repository. Instead restic will only print the actions it would
perform.
.. note:: The ``rewrite`` command verifies that it does not modify snapshots in
unexpected ways and fails with an ``cannot encode tree at "[...]" without losing information``
unexpected ways and fails with an ``cannot encode tree at "[...]" without loosing information``
error otherwise. This can occur when rewriting a snapshot created by a newer
version of restic or some third-party implementation.

17
doc/047_tuning_backup_parameters.rst → doc/047_tuning_parameters.rst
View file

@ -9,14 +9,14 @@
^ for subsubsections
" for paragraphs
########################
Tuning Backup Parameters
########################
#################
Tuning Parameters
#################
Restic offers a few parameters that allow tuning the backup. The default values should
work well in general although specific use cases can benefit from different non-default
values. As the restic commands evolve over time, the optimal value for each parameter
can also change across restic versions.
Restic offers a few parameters that allow tuning the backup and other operations.
The default values should work well in general although specific use cases can
benefit from different non-default values. As the restic commands evolve over
time, the optimal value for each parameter can also change across restic versions.
Disabling Backup Progress Estimation
@ -99,7 +99,8 @@ Swift and some Google Drive Team accounts, where there are hard limits on the to
number of files. Larger pack sizes can also improve the backup speed for a repository
stored on a local HDD. This can be achieved by either using the ``--pack-size`` option
or defining the ``$RESTIC_PACK_SIZE`` environment variable, using an integer value for the
pack size in MiB. Restic currently defaults to a 16 MiB pack size.
pack size in MiB. Restic currently defaults to a 16 MiB pack size. Note that this
setting should be specified for each restic command that modifies the repository.
The side effect of increasing the pack size is requiring more disk space for temporary pack
files created before uploading. The space must be available in the system default temp

105
doc/075_scripting.rst
View file

@ -14,8 +14,99 @@
Scripting
#########################
This is a list of how certain tasks may be accomplished when you use
restic via scripts.
This section covers environment variables and how certain tasks may be accomplished
when you use restic via scripts.
.. _environment-variables:
Environment Variables
*********************
In addition to command-line options, restic supports passing various options in
environment variables, which are listed below.
.. code-block:: console
RESTIC_REPOSITORY_FILE Name of file containing the repository location (replaces --repository-file)
RESTIC_REPOSITORY Location of repository (replaces -r)
RESTIC_PASSWORD_FILE Location of password file (replaces --password-file)
RESTIC_PASSWORD The actual password for the repository
RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout
RESTIC_KEY_HINT ID of key to try decrypting first, before other keys
RESTIC_CACERT Location(s) of certificate file(s), comma separated if multiple (replaces --cacert)
RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert)
RESTIC_CACHE_DIR Location of the cache directory
RESTIC_COMPRESSION Compression mode (only available for repository format version 2)
RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host)
RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated
RESTIC_PACK_SIZE Target size for pack files
RESTIC_READ_CONCURRENCY Concurrency for file reads
TMPDIR Location for temporary files (except Windows)
TMP Location for temporary files (only Windows)
AWS_ACCESS_KEY_ID Amazon S3 access key ID
AWS_SECRET_ACCESS_KEY Amazon S3 secret access key
AWS_SESSION_TOKEN Amazon S3 temporary session token
AWS_DEFAULT_REGION Amazon S3 default region
AWS_PROFILE Amazon credentials profile (alternative to specifying key and region)
AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials)
RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials
RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption
RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption
RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy
RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1)
RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only.
AZURE_ACCOUNT_NAME Account name for Azure
AZURE_ACCOUNT_KEY Account key for Azure
AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure
AZURE_ENDPOINT_SUFFIX Endpoint suffix for Azure Storage (default: core.windows.net)
AZURE_FORCE_CLI_CREDENTIAL Force the use of Azure CLI credentials for authentication
B2_ACCOUNT_ID Account ID or applicationKeyId for Backblaze B2
B2_ACCOUNT_KEY Account Key or applicationKey for Backblaze B2
GOOGLE_PROJECT_ID Project ID for Google Cloud Storage
GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json)
OS_AUTH_URL Auth URL for keystone authentication
OS_REGION_NAME Region name for keystone authentication
OS_USERNAME Username for keystone authentication
OS_USER_ID User ID for keystone v3 authentication
OS_PASSWORD Password for keystone authentication
OS_TENANT_ID Tenant ID for keystone v2 authentication
OS_TENANT_NAME Tenant name for keystone v2 authentication
OS_USER_DOMAIN_NAME User domain name for keystone authentication
OS_USER_DOMAIN_ID User domain ID for keystone v3 authentication
OS_PROJECT_NAME Project name for keystone authentication
OS_PROJECT_DOMAIN_NAME Project domain name for keystone authentication
OS_PROJECT_DOMAIN_ID Project domain ID for keystone v3 authentication
OS_TRUST_ID Trust ID for keystone v3 authentication
OS_APPLICATION_CREDENTIAL_ID Application Credential ID (keystone v3)
OS_APPLICATION_CREDENTIAL_NAME Application Credential Name (keystone v3)
OS_APPLICATION_CREDENTIAL_SECRET Application Credential Secret (keystone v3)
OS_STORAGE_URL Storage URL for token authentication
OS_AUTH_TOKEN Auth token for token authentication
RCLONE_BWLIMIT rclone bandwidth limit
RESTIC_REST_USERNAME Restic REST Server username
RESTIC_REST_PASSWORD Restic REST Server password
ST_AUTH Auth URL for keystone v1 authentication
ST_USER Username for keystone v1 authentication
ST_KEY Password for keystone v1 authentication
See :ref:`caching` for the rules concerning cache locations when
``RESTIC_CACHE_DIR`` is not set.
The external programs that restic may execute include ``rclone`` (for rclone
backends) and ``ssh`` (for the SFTP backend). These may respond to further
environment variables and configuration files; see their respective manuals.
Check if a repository is already initialized
********************************************
@ -810,11 +901,11 @@ Changed
Summary
^^^^^^^
+----------------------------+-----------------------------------+--------+
| ``message_type`` | Always "summary" | string |
+----------------------------+-----------------------------------+--------+
| ``changed_snapshot_count`` | Total number of changed snapshots | int64 |
+----------------------------+-----------------------------------+--------+
+-----------------------+-----------------------------------+--------+
| ``message_type`` | Always "summary" | string |
+-----------------------+-----------------------------------+--------+
| ``changed_snapshots`` | Total number of changed snapshots | int64 |
+-----------------------+-----------------------------------+--------+
version
-------

6
doc/077_troubleshooting.rst
View file

@ -82,6 +82,12 @@ If ``check`` detects damaged pack files, it will show instructions on how to rep
them using the ``repair pack`` command. Use that command instead of the "Repair the
index" section in this guide.
If you are interested to check only specific snapshots, you can now
use the standard snapshot filter method specifying ``--host``, ``--path``, ``--tag`` or
alternatively naming snapshot ID(s) explicitely. The selected subset of packfiles
will then be checked for consistency and read when either ``--read-data`` or
``--read-data-subset`` is given.
2. Backup the repository
************************

1
doc/100_references.rst
View file

@ -19,5 +19,6 @@ Design
******
.. include:: design.rst
.. include:: view_repository.rst
.. include:: cache.rst
.. include:: REST_backend.rst

4
doc/_static/css/restic.css vendored
View file

@ -8,3 +8,7 @@
height: 50% !important;
width: 50% !important;
}
.wy-table-responsive table td {
white-space: normal;
}

2
doc/index.rst
View file

@ -9,7 +9,7 @@ Restic Documentation
030_preparing_a_new_repo
040_backup
045_working_with_repos
047_tuning_backup_parameters
047_tuning_parameters
050_restore
060_forget
070_encryption

137
doc/view_repository.rst Normal file
View file

@ -0,0 +1,137 @@
..
Normally, there are no heading levels assigned to certain characters as the structure is
determined from the succession of headings. However, this convention is used in Python’s
Style Guide for documenting which you may follow:
# with overline, for parts
* for chapters
= for sections
- for subsections
^ for subsubsections
" for paragraphs
************************
Diving into a Repository
************************
The following section dives into the commands developers could use
to extract certain data from a repository.
Listing different file types in the repository
==============================================
The ``restic list`` allows listing objects in the repository based on type.
The allowed types are (in alphabetic order):
- blobs
- index
- keys
- locks
- packs
- snapshots
With the exception of ``blobs`` all output - in text mode - contains zero or more
``IDs`` of the given type, one ``ID`` per output line.
The output for ``blobs`` contains one or more lines of output of the form
``blob-type blob-ID``, where ``blob-type`` is either ``data`` or ``tree``, and ``blob-ID``
is the ``sha256sum`` of the ``blob``.
The output of the ``restic list 'type-plural'`` is most commonly used for the ``restic cat 'type' ID``
command to study an ``type`` object with an ``ID`` in more detail. The only exception to
this singular/plural ``type`` is ``ìndex``, which is used in both commands ``restic list index`` and
``restic cat index <ID>``.
The examples below are using part of the standard file structure for testing restic itself.
Here is the ``ls`` output of the one and only snapshot in this test repository:
.. code-block:: console
$ restic -r /srv/restic-repo ls 4254d65c
snapshot 4254d65c of [/srv/restic-repo/testdata/0/for_cmd_ls] at 2026-01-17 17:26:41.972899252 +0000 UTC by user@kasimir filtered by []:
/srv/restic-repo/testdata
/srv/restic-repo/testdata/0
/srv/restic-repo/testdata/0/for_cmd_ls
/srv/restic-repo/testdata/0/for_cmd_ls/file1.txt
/srv/restic-repo/testdata/0/for_cmd_ls/file2.txt
/srv/restic-repo/testdata/0/for_cmd_ls/python.py
Inspecting this repository with ``restic list snapshots`` produces:
.. code-block:: console
$ restic -r /srv/restic-repo list snapshots -q
4254d65c92208eda22b852b390bd5401ca4c500be7a022c70e7c33de68ca2143
$ restic -r /srv/restic-repo cat snapshot 4254d65c92208eda22b852b390bd5401ca4c500be7a022c70e7c33de68ca2143 -q
{
"time": "2026-01-17T17:26:41.972899252Z",
"tree": "db9e90f7f1761ab892b3ae25e3838bbd697499b985e9b47d3a1da09e0bd8ca68",
...
"summary": {
"backup_start": "2026-01-17T17:26:41.972899252Z",
"backup_end": "2026-01-17T17:26:42.581012438Z",
...
}
}
The index contains 2 packfiles, one for trees and one for the actual file data:
.. code-block:: console
$ restic -r /srv/restic-repo list index -q
a1828d209e760f0fd143aa79e530de0a377d7affd1cd0964d9cb2ad3c77e0d8b
$ restic -r /srv/restic-repo cat index a1828d209e760f0fd143aa79e530de0a377d7affd1cd0964d9cb2ad3c77e0d8b | jq
{
"packs": [
{
"id": "953e5381138bdc44da23740a83065809dd4021f45ce4e351b577dc4c07f81314",
"blobs": [
{
"id": "124323c57d74fb8944c98fb69ce67a41a107cb6d2ed304cf50c8529cc137aafd",
"type": "data",
"offset": 0,
"length": 59,
"uncompressed_length": 18
},
...
]
},
{
"id": "75bca8556f47d16362e58e757ea89a34b28fb96aedcc314bea35d468e5cb665c",
"blobs": [
{
"id": "6dfdc53cc3b45a6bf519a7fb80a54f6ef3e3ea859f51d3e85a6235177606f1f9",
"type": "tree",
"offset": 0,
"length": 271,
"uncompressed_length": 353
},
...
]
}
]
}
And this is the list of blobs:
.. code-block:: console
$ restic -r /srv/restic-repo list blobs -q
data 124323c57d74fb8944c98fb69ce67a41a107cb6d2ed304cf50c8529cc137aafd
data 37cc0b45af245d93abaecba73a600a8d577b39e4a1fdc2dcdf93ad63b1e167bd
data 5dfb8bc8a35175bf011d10ac7bc3a6b8d42b7743ac188be8c1bf0b215f9b7bf5
tree 6dfdc53cc3b45a6bf519a7fb80a54f6ef3e3ea859f51d3e85a6235177606f1f9
tree 73947e98d4025179347363401eb41f148dc29a1d1735bfb96a08a6036422108c
tree 6d1daddbb3f280be0f25e708618576e003c2a87516a9aa31e98205ae0a152ab5
tree 2e89c815e31c377629ef77fa1c156d1ad794b9f09d9d3b113e00e8eab36ceb98
tree db9e90f7f1761ab892b3ae25e3838bbd697499b985e9b47d3a1da09e0bd8ca68
tree d2524f3358bffbfe7349ca73df4bd3f23f5b252a9ba887481eda7e696b506dd4
tree 4d8f5a6c6e90a2d69ae4b2f8e4f7f5851ccc4fa2cd3314f81de6c929453994fe
The other types ``keys``, ``locks`` and ``packs`` are used in the same way as the type ``index``.
.. code-block:: console
$ restic -r /srv/restic-repo list packs -q
953e5381138bdc44da23740a83065809dd4021f45ce4e351b577dc4c07f81314
75bca8556f47d16362e58e757ea89a34b28fb96aedcc314bea35d468e5cb665c

90
go.mod
View file

@ -1,61 +1,61 @@
module github.com/restic/restic
go 1.23.0
go 1.24.0
// keep the old behavior for reparse points on windows until handling reparse points has been improved in restic
// https://forum.restic.net/t/windows-junction-backup-with-go1-23-or-later/8940
godebug winsymlink=0
require (
cloud.google.com/go/storage v1.56.1
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
cloud.google.com/go/storage v1.59.2
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/Backblaze/blazer v0.7.2
github.com/Microsoft/go-winio v0.6.2
github.com/anacrolix/fuse v0.3.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/cespare/xxhash/v2 v2.3.0
github.com/elithrar/simple-scrypt v1.3.0
github.com/elithrar/simple-scrypt v1.4.0
github.com/go-ole/go-ole v1.3.0
github.com/google/go-cmp v0.7.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/compress v1.18.0
github.com/minio/minio-go/v7 v7.0.95
github.com/ncw/swift/v2 v2.0.4
github.com/klauspost/compress v1.18.3
github.com/minio/minio-go/v7 v7.0.98
github.com/ncw/swift/v2 v2.0.5
github.com/peterbourgon/unixtransport v0.0.7
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.10
github.com/pkg/xattr v0.4.12
github.com/restic/chunker v0.4.0
github.com/spf13/cobra v1.10.1
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
go.uber.org/automaxprocs v1.6.0
golang.org/x/crypto v0.41.0
golang.org/x/net v0.43.0
golang.org/x/oauth2 v0.30.0
golang.org/x/sync v0.16.0
golang.org/x/sys v0.35.0
golang.org/x/term v0.34.0
golang.org/x/text v0.28.0
golang.org/x/time v0.12.0
google.golang.org/api v0.248.0
golang.org/x/crypto v0.47.0
golang.org/x/net v0.49.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sync v0.19.0
golang.org/x/sys v0.40.0
golang.org/x/term v0.39.0
golang.org/x/text v0.33.0
golang.org/x/time v0.14.0
google.golang.org/api v0.256.0
)
require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go v0.121.6 // indirect
cloud.google.com/go/auth v0.16.5 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.8.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@ -64,21 +64,21 @@ require (
github.com/felixge/fgprof v0.9.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-jose/go-jose/v4 v4.1.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/minio/crc64nvme v1.0.2 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@ -86,21 +86,21 @@ require (
github.com/rs/xid v1.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/tinylib/msgp v1.3.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
google.golang.org/grpc v1.74.2 // indirect
google.golang.org/protobuf v1.36.7 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/grpc v1.76.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
)

192
go.sum
View file

@ -1,51 +1,51 @@
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c=
cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI=
cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=
cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE=
cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY=
cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E=
cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY=
cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
cloud.google.com/go/storage v1.56.1 h1:n6gy+yLnHn0hTwBFzNn8zJ1kqWfR91wzdM8hjRF4wP0=
cloud.google.com/go/storage v1.56.1/go.mod h1:C9xuCZgFl3buo2HZU/1FncgvvOgTAs/rnh4gF4lMg0s=
cloud.google.com/go/storage v1.59.2 h1:gmOAuG1opU8YvycMNpP+DvHfT9BfzzK5Cy+arP+Nocw=
cloud.google.com/go/storage v1.59.2/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI=
cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM=
github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 h1:lhhYARPUu3LmHysQ/igznQphfzynnqI3D75oUyw1HXk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0/go.mod h1:l9rva3ApbBpEJxSNYnwT9N4CDLrWgtq3u8736C5hyJw=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0 h1:xfK3bbi6F2RDtaZFtUdKO3osOBIhNb+xTs8lFW6yx9o=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 h1:s0WlVbf9qpvkh1c/uDAPElam0WrL7fHRIidgZJ7UqZI=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
@ -78,8 +78,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dvyukov/go-fuzz v0.0.0-20220726122315-1d375ef9f9f6/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg=
github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1qPwrl/EJwEqnZo=
github.com/elithrar/simple-scrypt v1.4.0 h1:5sZ4st4bK5wBymv7DaGZFbRasbAj0WrhxECkLSm6q6Y=
github.com/elithrar/simple-scrypt v1.4.0/go.mod h1:dtDmWT+ijC9sdIbf50kQjgI9xOFmA4LG/8/T6TbmtFY=
github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
@ -95,8 +95,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@ -104,8 +104,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@ -122,8 +120,8 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@ -133,11 +131,13 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -151,14 +151,14 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w=
github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8=
github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
@ -190,14 +190,14 @@ github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@ -214,41 +214,43 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw=
go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4=
golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
@ -261,16 +263,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -284,22 +286,22 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
@ -308,18 +310,20 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.248.0 h1:hUotakSkcwGdYUqzCRc5yGYsg4wXxpkKlW5ryVqvC1Y=
google.golang.org/api v0.248.0/go.mod h1:yAFUAF56Li7IuIQbTFoLwXTCI6XCFKueOlS7S9e4F9k=
google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ=
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI=
google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4=
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

62
internal/archiver/archiver.go
View file

@ -96,7 +96,6 @@ type Archiver struct {
FS fs.FS
Options Options
blobSaver *blobSaver
fileSaver *fileSaver
treeSaver *treeSaver
mu sync.Mutex
@ -145,11 +144,6 @@ type Options struct {
// turned out to be a good default for most situations).
ReadConcurrency uint
// SaveBlobConcurrency sets how many blobs are hashed and saved
// concurrently. If it's set to zero, the default is the number of CPUs
// available in the system.
SaveBlobConcurrency uint
// SaveTreeConcurrency sets how many trees are marshalled and saved to the
// repo concurrently.
SaveTreeConcurrency uint
@ -165,12 +159,6 @@ func (o Options) ApplyDefaults() Options {
o.ReadConcurrency = 2
}
if o.SaveBlobConcurrency == 0 {
// blob saving is CPU bound due to hash checking and encryption
// the actual upload is handled by the repository itself
o.SaveBlobConcurrency = uint(runtime.GOMAXPROCS(0))
}
if o.SaveTreeConcurrency == 0 {
// can either wait for a file, wait for a tree, serialize a tree or wait for saveblob
// the last two are cpu-bound and thus mutually exclusive.
@ -293,7 +281,7 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ig
// loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.
// If there is no node to load, then nil is returned without an error.
func (arch *Archiver) loadSubtree(ctx context.Context, node *data.Node) (*data.Tree, error) {
func (arch *Archiver) loadSubtree(ctx context.Context, node *data.Node) (data.TreeNodeIterator, error) {
if node == nil || node.Type != data.NodeTypeDir || node.Subtree == nil {
return nil, nil
}
@ -319,7 +307,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *data.Tree, complete fileCompleteFunc) (d futureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous data.TreeNodeIterator, complete fileCompleteFunc) (d futureNode, err error) {
debug.Log("%v %v", snPath, dir)
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
@ -329,6 +317,9 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
nodes := make([]futureNode, 0, len(names))
finder := data.NewTreeFinder(previous)
defer finder.Close()
for _, name := range names {
// test if context has been cancelled
if ctx.Err() != nil {
@ -337,7 +328,11 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
}
pathname := arch.FS.Join(dir, name)
oldNode := previous.Find(name)
oldNode, err := finder.Find(name)
err = arch.error(pathname, err)
if err != nil {
return futureNode{}, err
}
snItem := join(snPath, name)
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
@ -657,7 +652,7 @@ func join(elem ...string) string {
// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *data.Tree, complete fileCompleteFunc) (futureNode, int, error) {
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous data.TreeNodeIterator, complete fileCompleteFunc) (futureNode, int, error) {
var node *data.Node
if snPath != "/" {
@ -675,10 +670,13 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
node = &data.Node{}
}
debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
debug.Log("%v (%v nodes)", snPath, len(atree.Nodes))
nodeNames := atree.NodeNames()
nodes := make([]futureNode, 0, len(nodeNames))
finder := data.NewTreeFinder(previous)
defer finder.Close()
// iterate over the nodes of atree in lexicographic (=deterministic) order
for _, name := range nodeNames {
subatree := atree.Nodes[name]
@ -690,7 +688,13 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
// this is a leaf node
if subatree.Leaf() {
fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
pathname := join(snPath, name)
oldNode, err := finder.Find(name)
err = arch.error(pathname, err)
if err != nil {
return futureNode{}, 0, err
}
fn, excluded, err := arch.save(ctx, pathname, subatree.Path, oldNode)
if err != nil {
err = arch.error(subatree.Path, err)
@ -710,7 +714,11 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
snItem := join(snPath, name) + "/"
start := time.Now()
oldNode := previous.Find(name)
oldNode, err := finder.Find(name)
err = arch.error(snItem, err)
if err != nil {
return futureNode{}, 0, err
}
oldSubtree, err := arch.loadSubtree(ctx, oldNode)
if err != nil {
err = arch.error(join(snPath, name), err)
@ -842,7 +850,7 @@ type SnapshotOptions struct {
}
// loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned.
func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) *data.Tree {
func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) data.TreeNodeIterator {
if sn == nil {
return nil
}
@ -863,24 +871,20 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *data.Snapshot) *da
}
// runWorkers starts the worker pools, which are stopped when the context is cancelled.
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaver) {
arch.blobSaver = newBlobSaver(ctx, wg, uploader, arch.Options.SaveBlobConcurrency)
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync) {
arch.fileSaver = newFileSaver(ctx, wg,
arch.blobSaver.Save,
uploader,
arch.Repo.Config().ChunkerPolynomial,
arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency)
arch.Options.ReadConcurrency)
arch.fileSaver.CompleteBlob = arch.CompleteBlob
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error)
arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, uploader, arch.Error)
}
func (arch *Archiver) stopWorkers() {
arch.blobSaver.TriggerShutdown()
arch.fileSaver.TriggerShutdown()
arch.treeSaver.TriggerShutdown()
arch.blobSaver = nil
arch.fileSaver = nil
arch.treeSaver = nil
}
@ -903,7 +907,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
var rootTreeID restic.ID
err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err = arch.Repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, wgCtx := errgroup.WithContext(ctx)
start := time.Now()

136
internal/archiver/archiver_test.go
View file

@ -56,7 +56,7 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS
return err
}
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
@ -219,7 +219,7 @@ func TestArchiverSave(t *testing.T) {
arch.summary = &Summary{}
var fnr futureNodeResult
err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
@ -296,7 +296,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
arch.summary = &Summary{}
var fnr futureNodeResult
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
@ -415,29 +415,39 @@ type blobCountingRepo struct {
saved map[restic.BlobHandle]uint
}
func (repo *blobCountingRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaver) error) error {
return repo.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
func (repo *blobCountingRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaverWithAsync) error) error {
return repo.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
return fn(ctx, &blobCountingSaver{saver: uploader, blobCountingRepo: repo})
})
}
type blobCountingSaver struct {
saver restic.BlobSaver
saver restic.BlobSaverWithAsync
blobCountingRepo *blobCountingRepo
}
func (repo *blobCountingSaver) count(exists bool, h restic.BlobHandle) {
if exists {
return
}
repo.blobCountingRepo.m.Lock()
repo.blobCountingRepo.saved[h]++
repo.blobCountingRepo.m.Unlock()
}
func (repo *blobCountingSaver) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) {
id, exists, size, err := repo.saver.SaveBlob(ctx, t, buf, id, storeDuplicate)
if exists {
return id, exists, size, err
}
h := restic.BlobHandle{ID: id, Type: t}
repo.blobCountingRepo.m.Lock()
repo.blobCountingRepo.saved[h]++
repo.blobCountingRepo.m.Unlock()
repo.count(exists, restic.BlobHandle{ID: id, Type: t})
return id, exists, size, err
}
func (repo *blobCountingSaver) SaveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) {
repo.saver.SaveBlobAsync(ctx, t, buf, id, storeDuplicate, func(newID restic.ID, known bool, size int, err error) {
repo.count(known, restic.BlobHandle{ID: newID, Type: t})
cb(newID, known, size, err)
})
}
func appendToFile(t testing.TB, filename string, data []byte) {
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
@ -840,7 +850,7 @@ func TestArchiverSaveDir(t *testing.T) {
defer back()
var treeID restic.ID
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true)
@ -867,11 +877,7 @@ func TestArchiverSaveDir(t *testing.T) {
}
node.Name = targetNodeName
tree := &data.Tree{Nodes: []*data.Node{node}}
treeID, err = data.SaveTree(ctx, uploader, tree)
if err != nil {
t.Fatal(err)
}
treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
arch.stopWorkers()
return wg.Wait()
})
@ -906,7 +912,7 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
arch.summary = &Summary{}
var fnr futureNodeResult
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true)
@ -1096,7 +1102,7 @@ func TestArchiverSaveTree(t *testing.T) {
}
var treeID restic.ID
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
@ -2077,8 +2083,6 @@ func TestArchiverContextCanceled(t *testing.T) {
type TrackFS struct {
fs.FS
errorOn map[string]error
opened map[string]uint
m sync.Mutex
}
@ -2094,38 +2098,61 @@ func (m *TrackFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, e
type failSaveRepo struct {
archiverRepo
failAfter int32
cnt int32
cnt atomic.Int32
err error
}
func (f *failSaveRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaver) error) error {
return f.archiverRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
return fn(ctx, &failSaveSaver{saver: uploader, failSaveRepo: f})
func (f *failSaveRepo) WithBlobUploader(ctx context.Context, fn func(ctx context.Context, uploader restic.BlobSaverWithAsync) error) error {
outerCtx, outerCancel := context.WithCancelCause(ctx)
defer outerCancel(f.err)
return f.archiverRepo.WithBlobUploader(outerCtx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
return fn(ctx, &failSaveSaver{saver: uploader, failSaveRepo: f, semaphore: make(chan struct{}, 1), outerCancel: outerCancel})
})
}
type failSaveSaver struct {
saver restic.BlobSaver
saver restic.BlobSaverWithAsync
failSaveRepo *failSaveRepo
semaphore chan struct{}
outerCancel context.CancelCauseFunc
}
func (f *failSaveSaver) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) {
val := atomic.AddInt32(&f.failSaveRepo.cnt, 1)
val := f.failSaveRepo.cnt.Add(1)
if val >= f.failSaveRepo.failAfter {
return restic.Hash(buf), false, 0, f.failSaveRepo.err
return restic.ID{}, false, 0, f.failSaveRepo.err
}
return f.saver.SaveBlob(ctx, t, buf, id, storeDuplicate)
}
func TestArchiverAbortEarlyOnError(t *testing.T) {
var testErr = errors.New("test error")
func (f *failSaveSaver) SaveBlobAsync(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, size int, err error)) {
// limit concurrency to make test reliable
f.semaphore <- struct{}{}
val := f.failSaveRepo.cnt.Add(1)
if val >= f.failSaveRepo.failAfter {
// kill the outer context to make SaveBlobAsync fail
// precisely injecting a specific error into the repository is not possible, so just cancel the context
f.outerCancel(f.failSaveRepo.err)
}
f.saver.SaveBlobAsync(ctx, t, buf, id, storeDuplicate, func(newID restic.ID, known bool, size int, err error) {
if val >= f.failSaveRepo.failAfter {
if err == nil {
panic("expected error")
}
}
cb(newID, known, size, err)
<-f.semaphore
})
}
func TestArchiverAbortEarlyOnError(t *testing.T) {
var tests = []struct {
src TestDir
wantOpen map[string]uint
failAfter uint // error after so many blobs have been saved to the repo
err error
}{
{
src: TestDir{
@ -2137,8 +2164,6 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
},
wantOpen: map[string]uint{
filepath.FromSlash("dir/bar"): 1,
filepath.FromSlash("dir/baz"): 1,
filepath.FromSlash("dir/foo"): 1,
},
},
{
@ -2165,9 +2190,8 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
filepath.FromSlash("dir/file9"): 0,
},
// fails after four to seven files were opened, as the ReadConcurrency allows for
// two queued files and SaveBlobConcurrency for one blob queued for saving.
// two queued files and one blob queued for saving.
failAfter: 4,
err: testErr,
},
}
@ -2186,25 +2210,25 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
opened: make(map[string]uint),
}
if testFS.errorOn == nil {
testFS.errorOn = make(map[string]error)
}
testErr := context.Canceled
testRepo := &failSaveRepo{
archiverRepo: repo,
failAfter: int32(test.failAfter),
err: test.err,
err: testErr,
}
// at most two files may be queued
arch := New(testRepo, testFS, Options{
ReadConcurrency: 2,
SaveBlobConcurrency: 1,
ReadConcurrency: 2,
})
arch.Error = func(item string, err error) error {
t.Logf("archiver error for %q: %v", item, err)
return err
}
_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if !errors.Is(err, test.err) {
t.Errorf("expected error (%v) not found, got %v", test.err, err)
if !errors.Is(err, testErr) {
t.Errorf("expected error (%v) not found, got %v", testErr, err)
}
t.Logf("Snapshot return error: %v", err)
@ -2231,19 +2255,15 @@ func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *data.Snapshot,
ParentSnapshot: parent,
}
snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
if err != nil {
t.Fatal(err)
}
rtest.OK(t, err)
tree, err := data.LoadTree(ctx, repo, *snapshot.Tree)
if err != nil {
t.Fatal(err)
}
rtest.OK(t, err)
node := tree.Find(filename)
if node == nil {
t.Fatalf("unable to find node for testfile in snapshot")
}
finder := data.NewTreeFinder(tree)
defer finder.Close()
node, err := finder.Find(filename)
rtest.OK(t, err)
rtest.Assert(t, node != nil, "unable to find node for testfile in snapshot")
return snapshot, node
}
@ -2428,7 +2448,7 @@ func TestRacyFileTypeSwap(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
_ = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch := New(repo, fs.Track{FS: statfs}, Options{})

1
internal/archiver/archiver_unix_test.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package archiver

105
internal/archiver/blob_saver.go
View file

@ -1,105 +0,0 @@
package archiver
import (
"context"
"fmt"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/sync/errgroup"
)
// saver allows saving a blob.
type saver interface {
SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error)
}
// blobSaver concurrently saves incoming blobs to the repo.
type blobSaver struct {
repo saver
ch chan<- saveBlobJob
}
// newBlobSaver returns a new blob. A worker pool is started, it is stopped
// when ctx is cancelled.
func newBlobSaver(ctx context.Context, wg *errgroup.Group, repo saver, workers uint) *blobSaver {
ch := make(chan saveBlobJob)
s := &blobSaver{
repo: repo,
ch: ch,
}
for i := uint(0); i < workers; i++ {
wg.Go(func() error {
return s.worker(ctx, ch)
})
}
return s
}
func (s *blobSaver) TriggerShutdown() {
close(s.ch)
}
// Save stores a blob in the repo. It checks the index and the known blobs
// before saving anything. It takes ownership of the buffer passed in.
func (s *blobSaver) Save(ctx context.Context, t restic.BlobType, buf *buffer, filename string, cb func(res saveBlobResponse)) {
select {
case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}:
case <-ctx.Done():
debug.Log("not sending job, context is cancelled")
}
}
type saveBlobJob struct {
restic.BlobType
buf *buffer
fn string
cb func(res saveBlobResponse)
}
type saveBlobResponse struct {
id restic.ID
length int
sizeInRepo int
known bool
}
func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) {
id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false)
if err != nil {
return saveBlobResponse{}, err
}
return saveBlobResponse{
id: id,
length: len(buf),
sizeInRepo: sizeInRepo,
known: known,
}, nil
}
func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
for {
var job saveBlobJob
var ok bool
select {
case <-ctx.Done():
return nil
case job, ok = <-jobs:
if !ok {
return nil
}
}
res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data)
if err != nil {
debug.Log("saveBlob returned error, exiting: %v", err)
return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err)
}
job.cb(res)
job.buf.Release()
}
}

116
internal/archiver/blob_saver_test.go
View file

@ -1,116 +0,0 @@
package archiver
import (
"context"
"fmt"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
var errTest = errors.New("test error")
type saveFail struct {
cnt int32
failAt int32
}
func (b *saveFail) SaveBlob(_ context.Context, _ restic.BlobType, _ []byte, id restic.ID, _ bool) (restic.ID, bool, int, error) {
val := atomic.AddInt32(&b.cnt, 1)
if val == b.failAt {
return restic.ID{}, false, 0, errTest
}
return id, false, 0, nil
}
func TestBlobSaver(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{}
b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
var wait sync.WaitGroup
var results []saveBlobResponse
var lock sync.Mutex
wait.Add(20)
for i := 0; i < 20; i++ {
buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
idx := i
lock.Lock()
results = append(results, saveBlobResponse{})
lock.Unlock()
b.Save(ctx, restic.DataBlob, buf, "file", func(res saveBlobResponse) {
lock.Lock()
results[idx] = res
lock.Unlock()
wait.Done()
})
}
wait.Wait()
for i, sbr := range results {
if sbr.known {
t.Errorf("blob %v is known, that should not be the case", i)
}
}
b.TriggerShutdown()
err := wg.Wait()
if err != nil {
t.Fatal(err)
}
}
func TestBlobSaverError(t *testing.T) {
var tests = []struct {
blobs int
failAt int
}{
{20, 2},
{20, 5},
{20, 15},
{200, 150},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{
failAt: int32(test.failAt),
}
b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
for i := 0; i < test.blobs; i++ {
buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
b.Save(ctx, restic.DataBlob, buf, "errfile", func(res saveBlobResponse) {})
}
b.TriggerShutdown()
err := wg.Wait()
if err == nil {
t.Errorf("expected error not found")
}
rtest.Assert(t, errors.Is(err, errTest), "unexpected error %v", err)
rtest.Assert(t, strings.Contains(err.Error(), "errfile"), "expected error to contain 'errfile' got: %v", err)
})
}
}

31
internal/archiver/buffer.go
View file

@ -1,5 +1,7 @@
package archiver
import "sync"
// buffer is a reusable buffer. After the buffer has been used, Release should
// be called so the underlying slice is put back into the pool.
type buffer struct {
@ -14,41 +16,32 @@ func (b *buffer) Release() {
return
}
select {
case pool.ch <- b:
default:
}
pool.pool.Put(b)
}
// bufferPool implements a limited set of reusable buffers.
type bufferPool struct {
ch chan *buffer
pool sync.Pool
defaultSize int
}
// newBufferPool initializes a new buffer pool. The pool stores at most max
// items. New buffers are created with defaultSize. Buffers that have grown
// larger are not put back.
func newBufferPool(max int, defaultSize int) *bufferPool {
func newBufferPool(defaultSize int) *bufferPool {
b := &bufferPool{
ch: make(chan *buffer, max),
defaultSize: defaultSize,
}
b.pool = sync.Pool{New: func() any {
return &buffer{
Data: make([]byte, defaultSize),
pool: b,
}
}}
return b
}
// Get returns a new buffer, either from the pool or newly allocated.
func (pool *bufferPool) Get() *buffer {
select {
case buf := <-pool.ch:
return buf
default:
}
b := &buffer{
Data: make([]byte, pool.defaultSize),
pool: pool,
}
return b
return pool.pool.Get().(*buffer)
}

58
internal/archiver/buffer_test.go Normal file
View file

@ -0,0 +1,58 @@
package archiver
import (
"testing"
)
func TestBufferPoolReuse(t *testing.T) {
success := false
// retries to avoid flakiness. The test can fail depending on the GC.
for i := 0; i < 100; i++ {
// Test that buffers are actually reused from the pool
pool := newBufferPool(1024)
// Get a buffer and modify it
buf1 := pool.Get()
buf1.Data[0] = 0xFF
originalAddr := &buf1.Data[0]
buf1.Release()
// Get another buffer and check if it's the same underlying slice
buf2 := pool.Get()
if &buf2.Data[0] == originalAddr {
success = true
break
}
buf2.Release()
}
if !success {
t.Error("buffer was not reused from pool")
}
}
func TestBufferPoolLargeBuffers(t *testing.T) {
success := false
// retries to avoid flakiness. The test can fail depending on the GC.
for i := 0; i < 100; i++ {
// Test that buffers larger than defaultSize are not returned to pool
pool := newBufferPool(1024)
buf := pool.Get()
// Grow the buffer beyond default size
buf.Data = append(buf.Data, make([]byte, 2048)...)
originalCap := cap(buf.Data)
buf.Release()
// Get a new buffer - should not be the same slice
newBuf := pool.Get()
if cap(newBuf.Data) != originalCap {
success = true
break
}
}
if !success {
t.Error("large buffer was incorrectly returned to pool")
}
}

35
internal/archiver/file_saver.go
View file

@ -15,13 +15,10 @@ import (
"golang.org/x/sync/errgroup"
)
// saveBlobFn saves a blob to a repo.
type saveBlobFn func(context.Context, restic.BlobType, *buffer, string, func(res saveBlobResponse))
// fileSaver concurrently saves incoming files to the repo.
type fileSaver struct {
saveFilePool *bufferPool
saveBlob saveBlobFn
uploader restic.BlobSaverAsync
pol chunker.Pol
@ -34,16 +31,13 @@ type fileSaver struct {
// newFileSaver returns a new file saver. A worker pool with fileWorkers is
// started, it is stopped when ctx is cancelled.
func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *fileSaver {
func newFileSaver(ctx context.Context, wg *errgroup.Group, uploader restic.BlobSaverAsync, pol chunker.Pol, fileWorkers uint) *fileSaver {
ch := make(chan saveFileJob)
debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers)
poolSize := fileWorkers + blobWorkers
debug.Log("new file saver with %v file workers", fileWorkers)
s := &fileSaver{
saveBlob: save,
saveFilePool: newBufferPool(int(poolSize), chunker.MaxSize),
uploader: uploader,
saveFilePool: newBufferPool(chunker.MaxSize),
pol: pol,
ch: ch,
@ -203,15 +197,20 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
node.Content = append(node.Content, restic.ID{})
lock.Unlock()
s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr saveBlobResponse) {
lock.Lock()
if !sbr.known {
fnr.stats.DataBlobs++
fnr.stats.DataSize += uint64(sbr.length)
fnr.stats.DataSizeInRepo += uint64(sbr.sizeInRepo)
s.uploader.SaveBlobAsync(ctx, restic.DataBlob, buf.Data, restic.ID{}, false, func(newID restic.ID, known bool, sizeInRepo int, err error) {
defer buf.Release()
if err != nil {
completeError(err)
return
}
node.Content[pos] = sbr.id
lock.Lock()
if !known {
fnr.stats.DataBlobs++
fnr.stats.DataSize += uint64(len(buf.Data))
fnr.stats.DataSizeInRepo += uint64(sizeInRepo)
}
node.Content[pos] = newID
lock.Unlock()
completeBlob()

24
internal/archiver/file_saver_test.go
View file

@ -11,7 +11,6 @@ import (
"github.com/restic/chunker"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
@ -31,44 +30,35 @@ func createTestFiles(t testing.TB, num int) (files []string) {
return files
}
func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, context.Context, *errgroup.Group) {
func startFileSaver(ctx context.Context, t testing.TB, _ fs.FS) (*fileSaver, *mockSaver, context.Context, *errgroup.Group) {
wg, ctx := errgroup.WithContext(ctx)
saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *buffer, _ string, cb func(saveBlobResponse)) {
cb(saveBlobResponse{
id: restic.Hash(buf.Data),
length: len(buf.Data),
sizeInRepo: len(buf.Data),
known: false,
})
}
workers := uint(runtime.NumCPU())
pol, err := chunker.RandomPolynomial()
if err != nil {
t.Fatal(err)
}
s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers)
saver := &mockSaver{saved: make(map[string]int)}
s := newFileSaver(ctx, wg, saver, pol, workers)
s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*data.Node, error) {
return meta.ToNode(ignoreXattrListError, t.Logf)
}
return s, ctx, wg
return s, saver, ctx, wg
}
func TestFileSaver(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
files := createTestFiles(t, 15)
startFn := func() {}
completeReadingFn := func() {}
completeFn := func(*data.Node, ItemStats) {}
files := createTestFiles(t, 15)
testFs := fs.Local{}
s, ctx, wg := startFileSaver(ctx, t, testFs)
s, saver, ctx, wg := startFileSaver(ctx, t, testFs)
var results []futureNode
@ -89,6 +79,8 @@ func TestFileSaver(t *testing.T) {
}
}
test.Assert(t, len(saver.saved) == len(files), "expected %d saved files, got %d", len(files), len(saver.saved))
s.TriggerShutdown()
err := wg.Wait()

17
internal/archiver/testing.go
View file

@ -15,6 +15,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
// TestSnapshot creates a new snapshot of path.
@ -265,19 +266,14 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
t.Helper()
tree, err := data.LoadTree(ctx, repo, treeID)
if err != nil {
t.Fatal(err)
return
}
rtest.OK(t, err)
var nodeNames []string
for _, node := range tree.Nodes {
nodeNames = append(nodeNames, node.Name)
}
debug.Log("%v (%v) %v", prefix, treeID.Str(), nodeNames)
checked := make(map[string]struct{})
for _, node := range tree.Nodes {
for item := range tree {
rtest.OK(t, item.Error)
node := item.Node
nodeNames = append(nodeNames, node.Name)
nodePrefix := path.Join(prefix, node.Name)
entry, ok := dir[node.Name]
@ -316,6 +312,7 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
}
}
}
debug.Log("%v (%v) %v", prefix, treeID.Str(), nodeNames)
for name := range dir {
_, ok := checked[name]

38
internal/archiver/tree_saver.go
View file

@ -12,7 +12,7 @@ import (
// treeSaver concurrently saves incoming trees to the repo.
type treeSaver struct {
saveBlob saveBlobFn
uploader restic.BlobSaverAsync
errFn ErrorFunc
ch chan<- saveTreeJob
@ -20,12 +20,12 @@ type treeSaver struct {
// newTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// started, it is stopped when ctx is cancelled.
func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob saveBlobFn, errFn ErrorFunc) *treeSaver {
func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, uploader restic.BlobSaverAsync, errFn ErrorFunc) *treeSaver {
ch := make(chan saveTreeJob)
s := &treeSaver{
ch: ch,
saveBlob: saveBlob,
uploader: uploader,
errFn: errFn,
}
@ -129,21 +129,35 @@ func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*data.Node, Ite
return nil, stats, err
}
b := &buffer{Data: buf}
ch := make(chan saveBlobResponse, 1)
s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res saveBlobResponse) {
ch <- res
var (
known bool
length int
sizeInRepo int
id restic.ID
)
ch := make(chan struct{}, 1)
s.uploader.SaveBlobAsync(ctx, restic.TreeBlob, buf, restic.ID{}, false, func(newID restic.ID, cbKnown bool, cbSizeInRepo int, cbErr error) {
known = cbKnown
length = len(buf)
sizeInRepo = cbSizeInRepo
id = newID
err = cbErr
ch <- struct{}{}
})
select {
case sbr := <-ch:
if !sbr.known {
case <-ch:
if err != nil {
return nil, stats, err
}
if !known {
stats.TreeBlobs++
stats.TreeSize += uint64(sbr.length)
stats.TreeSizeInRepo += uint64(sbr.sizeInRepo)
stats.TreeSize += uint64(length)
stats.TreeSizeInRepo += uint64(sizeInRepo)
}
node.Subtree = &sbr.id
node.Subtree = &id
return node, stats, nil
case <-ctx.Done():
return nil, stats, ctx.Err()

24
internal/archiver/tree_saver_test.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"runtime"
"sync"
"testing"
"github.com/restic/restic/internal/data"
@ -13,13 +14,20 @@ import (
"golang.org/x/sync/errgroup"
)
func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *buffer, _ string, cb func(res saveBlobResponse)) {
cb(saveBlobResponse{
id: restic.NewRandomID(),
known: false,
length: len(buf.Data),
sizeInRepo: len(buf.Data),
})
type mockSaver struct {
saved map[string]int
mutex sync.Mutex
}
func (m *mockSaver) SaveBlobAsync(_ context.Context, _ restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool, cb func(newID restic.ID, known bool, sizeInRepo int, err error)) {
// Fake async operation
go func() {
m.mutex.Lock()
m.saved[string(buf)]++
m.mutex.Unlock()
cb(restic.Hash(buf), false, len(buf), nil)
}()
}
func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() error) {
@ -30,7 +38,7 @@ func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() e
return err
}
b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn)
b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), &mockSaver{saved: make(map[string]int)}, errFn)
shutdown := func() error {
b.TriggerShutdown()

1
internal/backend/local/local_unix.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package local

2
internal/backend/location/location_test.go
View file

@ -29,7 +29,7 @@ func TestParse(t *testing.T) {
u, err := location.Parse(registry, path)
test.OK(t, err)
test.Equals(t, "local", u.Scheme)
test.Equals(t, &testConfig{loc: path}, u.Config)
test.Equals(t, any(&testConfig{loc: path}), u.Config)
}
func TestParseFallback(t *testing.T) {

1
internal/backend/rest/rest_unix_test.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package rest_test

2
internal/backend/util/defaults_test.go
View file

@ -37,7 +37,7 @@ func TestDefaultLoad(t *testing.T) {
return rd, nil
}, func(ird io.Reader) error {
rtest.Equals(t, rd, ird)
rtest.Equals(t, io.Reader(rd), ird)
return nil
})
rtest.OK(t, err)

149
internal/checker/checker.go
View file

@ -3,7 +3,6 @@ package checker
import (
"context"
"fmt"
"runtime"
"sync"
"github.com/restic/restic/internal/data"
@ -12,7 +11,6 @@ import (
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
)
// Checker runs various checks on a repository. It is advisable to create an
@ -24,13 +22,17 @@ type Checker struct {
*repository.Checker
blobRefs struct {
sync.Mutex
M restic.BlobSet
M restic.AssociatedBlobSet
}
trackUnused bool
snapshots restic.Lister
repo restic.Repository
// when snapshot filtering is being used
snapshotFilter *data.SnapshotFilter
args []string
}
type checkerRepository interface {
@ -46,17 +48,24 @@ func New(repo checkerRepository, trackUnused bool) *Checker {
trackUnused: trackUnused,
}
c.blobRefs.M = restic.NewBlobSet()
c.blobRefs.M = c.repo.NewAssociatedBlobSet()
return c
}
func (c *Checker) LoadSnapshots(ctx context.Context) error {
func (c *Checker) LoadSnapshots(ctx context.Context, snapshotFilter *data.SnapshotFilter, args []string) error {
var err error
c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile)
c.args = args
c.snapshotFilter = snapshotFilter
return err
}
// IsFiltered returns true if snapshot filtering is active
func (c *Checker) IsFiltered() bool {
return len(c.args) != 0 || !c.snapshotFilter.Empty()
}
// Error is an error that occurred while checking a repository.
type Error struct {
TreeID restic.ID
@ -81,31 +90,6 @@ func (e *TreeError) Error() string {
return fmt.Sprintf("tree %v: %v", e.ID, e.Errors)
}
// checkTreeWorker checks the trees received and sends out errors to errChan.
func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan data.TreeItem, out chan<- error) {
for job := range trees {
debug.Log("check tree %v (tree %v, err %v)", job.ID, job.Tree, job.Error)
var errs []error
if job.Error != nil {
errs = append(errs, job.Error)
} else {
errs = c.checkTree(job.ID, job.Tree)
}
if len(errs) == 0 {
continue
}
treeError := &TreeError{ID: job.ID, Errors: errs}
select {
case <-ctx.Done():
return
case out <- treeError:
debug.Log("tree %v: sent %d errors", treeError.ID, len(treeError.Errors))
}
}
}
func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) {
err := data.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *data.Snapshot, err error) error {
if err != nil {
@ -124,14 +108,43 @@ func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.
return ids, errs
}
func (c *Checker) loadActiveTrees(ctx context.Context, snapshotFilter *data.SnapshotFilter, args []string) (trees restic.IDs, errs []error) {
trees = []restic.ID{}
errs = []error{}
if !c.IsFiltered() {
return loadSnapshotTreeIDs(ctx, c.snapshots, c.repo)
}
err := snapshotFilter.FindAll(ctx, c.snapshots, c.repo, args, func(_ string, sn *data.Snapshot, err error) error {
if err != nil {
errs = append(errs, err)
return err
} else if sn != nil {
trees = append(trees, *sn.Tree)
}
return nil
})
if err != nil {
errs = append(errs, err)
return nil, errs
}
// track blobs to learn which packs need to be checked
c.trackUnused = true
return trees, errs
}
// Structure checks that for all snapshots all referenced data blobs and
// subtrees are available in the index. errChan is closed after all trees have
// been traversed.
func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan chan<- error) {
trees, errs := loadSnapshotTreeIDs(ctx, c.snapshots, c.repo)
trees, errs := c.loadActiveTrees(ctx, c.snapshotFilter, c.args)
p.SetMax(uint64(len(trees)))
debug.Log("need to check %d trees from snapshots, %d errs returned", len(trees), len(errs))
defer close(errChan)
for _, err := range errs {
select {
case <-ctx.Done():
@ -140,8 +153,7 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch
}
}
wg, ctx := errgroup.WithContext(ctx)
treeStream := data.StreamTrees(ctx, wg, c.repo, trees, func(treeID restic.ID) bool {
err := data.StreamTrees(ctx, c.repo, trees, p, func(treeID restic.ID) bool {
// blobRefs may be accessed in parallel by checkTree
c.blobRefs.Lock()
h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
@ -150,30 +162,46 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch
c.blobRefs.M.Insert(h)
c.blobRefs.Unlock()
return blobReferenced
}, p)
}, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error {
debug.Log("check tree %v (err %v)", treeID, err)
defer close(errChan)
// The checkTree worker only processes already decoded trees and is thus CPU-bound
workerCount := runtime.GOMAXPROCS(0)
for i := 0; i < workerCount; i++ {
wg.Go(func() error {
c.checkTreeWorker(ctx, treeStream, errChan)
var errs []error
if err != nil {
errs = append(errs, err)
} else {
errs = c.checkTree(treeID, nodes)
}
if len(errs) == 0 {
return nil
})
}
}
// the wait group should not return an error because no worker returns an
treeError := &TreeError{ID: treeID, Errors: errs}
select {
case <-ctx.Done():
return nil
case errChan <- treeError:
debug.Log("tree %v: sent %d errors", treeError.ID, len(treeError.Errors))
}
return nil
})
// StreamTrees should not return an error because no worker returns an
// error, so panic if that has changed somehow.
err := wg.Wait()
if err != nil {
panic(err)
}
}
func (c *Checker) checkTree(id restic.ID, tree *data.Tree) (errs []error) {
func (c *Checker) checkTree(id restic.ID, tree data.TreeNodeIterator) (errs []error) {
debug.Log("checking tree %v", id)
for _, node := range tree.Nodes {
for item := range tree {
if item.Error != nil {
errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("failed to decode tree %v: %w", id, item.Error)})
break
}
node := item.Node
switch node.Type {
case data.NodeTypeFile:
if node.Content == nil {
@ -245,7 +273,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
c.blobRefs.Lock()
defer c.blobRefs.Unlock()
debug.Log("checking %d blobs", len(c.blobRefs.M))
debug.Log("checking %d blobs", c.blobRefs.M.Len())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -259,3 +287,30 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
return blobs, err
}
// ReadPacks wraps repository.ReadPacks:
// in case snapshot filtering is not active it calls repository.ReadPacks()
// with an unmodified parameter list
// Otherwise it calculates the packfiles needed, gets their sizes from the full
// packfile set and submits them to repository.ReadPacks()
func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p *progress.Counter, errChan chan<- error) {
// no snapshot filtering, pass through
if !c.IsFiltered() {
c.Checker.ReadPacks(ctx, filter, p, errChan)
return
}
packfileFilter := func(allPacks map[restic.ID]int64) map[restic.ID]int64 {
filteredPacks := make(map[restic.ID]int64)
// convert used blobs into their encompassing packfiles
for bh := range c.blobRefs.M.Keys() {
for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) {
filteredPacks[pb.PackID] = allPacks[pb.PackID]
}
}
return filter(filteredPacks)
}
c.Checker.ReadPacks(ctx, packfileFilter, p, errChan)
}

26
internal/checker/checker_test.go
View file

@ -46,7 +46,7 @@ func checkPacks(chkr *checker.Checker) []error {
}
func checkStruct(chkr *checker.Checker) []error {
err := chkr.LoadSnapshots(context.TODO())
err := chkr.LoadSnapshots(context.TODO(), &data.SnapshotFilter{}, nil)
if err != nil {
return []error{err}
}
@ -522,21 +522,18 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
Size: 42,
Content: restic.IDs{restic.TestParseID("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")},
}
damagedTree := &data.Tree{
Nodes: []*data.Node{damagedNode},
}
damagedNodes := []*data.Node{damagedNode}
var id restic.ID
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
var err error
id, err = data.SaveTree(ctx, uploader, damagedTree)
return err
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
id = data.TestSaveNodes(t, ctx, uploader, damagedNodes)
return nil
}))
buf, err := repo.LoadBlob(ctx, restic.TreeBlob, id, nil)
test.OK(t, err)
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
var err error
_, _, _, err = uploader.SaveBlob(ctx, restic.DataBlob, buf, id, false)
return err
@ -556,15 +553,12 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
Subtree: &id,
}
rootTree := &data.Tree{
Nodes: []*data.Node{malNode, dirNode},
}
rootNodes := []*data.Node{malNode, dirNode}
var rootID restic.ID
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaver) error {
var err error
rootID, err = data.SaveTree(ctx, uploader, rootTree)
return err
test.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
rootID = data.TestSaveNodes(t, ctx, uploader, rootNodes)
return nil
}))
snapshot, err := data.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now())

3
internal/checker/testing.go
View file

@ -4,6 +4,7 @@ import (
"context"
"testing"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
)
@ -20,7 +21,7 @@ func TestCheckRepo(t testing.TB, repo checkerRepository) {
t.Fatalf("errors loading index: %v", hints)
}
err := chkr.LoadSnapshots(context.TODO())
err := chkr.LoadSnapshots(context.TODO(), &data.SnapshotFilter{}, nil)
if err != nil {
t.Error(err)
}

28
internal/data/find.go
View file

@ -6,7 +6,6 @@ import (
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
)
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data
@ -14,8 +13,7 @@ import (
func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs restic.FindBlobSet, p *progress.Counter) error {
var lock sync.Mutex
wg, ctx := errgroup.WithContext(ctx)
treeStream := StreamTrees(ctx, wg, repo, treeIDs, func(treeID restic.ID) bool {
return StreamTrees(ctx, repo, treeIDs, p, func(treeID restic.ID) bool {
// locking is necessary the goroutine below concurrently adds data blobs
lock.Lock()
h := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
@ -24,26 +22,24 @@ func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs,
blobs.Insert(h)
lock.Unlock()
return blobReferenced
}, p)
}, func(_ restic.ID, err error, nodes TreeNodeIterator) error {
if err != nil {
return err
}
wg.Go(func() error {
for tree := range treeStream {
if tree.Error != nil {
return tree.Error
for item := range nodes {
if item.Error != nil {
return item.Error
}
lock.Lock()
for _, node := range tree.Nodes {
switch node.Type {
case NodeTypeFile:
for _, blob := range node.Content {
blobs.Insert(restic.BlobHandle{ID: blob, Type: restic.DataBlob})
}
switch item.Node.Type {
case NodeTypeFile:
for _, blob := range item.Node.Content {
blobs.Insert(restic.BlobHandle{ID: blob, Type: restic.DataBlob})
}
}
lock.Unlock()
}
return nil
})
return wg.Wait()
}

6
internal/data/testdata/used_blobs_snapshot0 vendored
View file

@ -1,9 +1,9 @@
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"}
{"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"}
{"ID":"3cb26562e6849003adffc5e1dcf9a58a9d151ea4203bd451f6359d7cc0328104","Type":"tree"}
{"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
{"ID":"4bb52083c8a467921e8ed4139f7be3e282bad8d25d0056145eadd3962aed0127","Type":"tree"}
{"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"}
{"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"}
{"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"}
@ -14,10 +14,10 @@
{"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
{"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"}
{"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"}
{"ID":"b326b56e1b4c5c3b80e449fc40abcada21b5bd7ff12ce02236a2d289b89dcea7","Type":"tree"}
{"ID":"b5ba06039224566a09555abd089de7a693660154991295122fa72b0a3adc4150","Type":"data"}
{"ID":"b7040572b44cbfea8b784ecf8679c3d75cefc1cd3d12ed783ca0d8e5d124a60f","Type":"data"}
{"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"}
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

4
internal/data/testdata/used_blobs_snapshot1 vendored
View file

@ -1,6 +1,8 @@
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"}
{"ID":"428b3f50bcfdcb9a85b87f9401d8947b2c8a2f807c19c00491626e3ee890075e","Type":"tree"}
{"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
{"ID":"55416727dd211e5f208b70fc9a3d60d34484626279717be87843a7535f997404","Type":"tree"}
{"ID":"6824d08e63a598c02b364e25f195e64758494b5944f06c921ff30029e1e4e4bf","Type":"data"}
{"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"}
{"ID":"8192279e4b56e1644dcff715d5e08d875cd5713349139d36d142ed28364d8e00","Type":"data"}
@ -10,6 +12,4 @@
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"}

10
internal/data/testdata/used_blobs_snapshot2 vendored
View file

@ -1,24 +1,24 @@
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
{"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"}
{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"}
{"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
{"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"}
{"ID":"2b8ebd79732fcfec50ec94429cfd404d531c93defed78832a597d0fe8de64b96","Type":"tree"}
{"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"}
{"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
{"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"}
{"ID":"6824d08e63a598c02b364e25f195e64758494b5944f06c921ff30029e1e4e4bf","Type":"data"}
{"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"}
{"ID":"721d803612a2565f9be9581048c5d899c14a65129dabafbb0e43bba89684a63a","Type":"tree"}
{"ID":"9103a50221ecf6684c1e1652adacb4a85afb322d81a74ecd7477930ecf4774fc","Type":"tree"}
{"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"}
{"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"}
{"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"}
{"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
{"ID":"ac08ce34ba4f8123618661bef2425f7028ffb9ac740578a3ee88684d2523fee8","Type":"tree"}
{"ID":"b326b56e1b4c5c3b80e449fc40abcada21b5bd7ff12ce02236a2d289b89dcea7","Type":"tree"}
{"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"}
{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"}
{"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"}
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"}
{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

80
internal/data/testing.go
View file

@ -5,12 +5,14 @@ import (
"fmt"
"io"
"math/rand"
"slices"
"strings"
"testing"
"time"
"github.com/restic/chunker"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
rtest "github.com/restic/restic/internal/test"
)
// fakeFile returns a reader which yields deterministic pseudo-random data.
@ -72,22 +74,21 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, uploader restic.BlobSave
rnd := rand.NewSource(seed)
numNodes := int(rnd.Int63() % maxNodes)
var tree Tree
var nodes []*Node
for i := 0; i < numNodes; i++ {
// randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4).
if depth > 1 && rnd.Int63()%4 == 0 {
treeSeed := rnd.Int63() % maxSeed
id := fs.saveTree(ctx, uploader, treeSeed, depth-1)
node := &Node{
Name: fmt.Sprintf("dir-%v", treeSeed),
Name: fmt.Sprintf("dir-%v", i),
Type: NodeTypeDir,
Mode: 0755,
Subtree: &id,
}
tree.Nodes = append(tree.Nodes, node)
nodes = append(nodes, node)
continue
}
@ -95,22 +96,31 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, uploader restic.BlobSave
fileSize := (maxFileSize / maxSeed) * fileSeed
node := &Node{
Name: fmt.Sprintf("file-%v", fileSeed),
Name: fmt.Sprintf("file-%v", i),
Type: NodeTypeFile,
Mode: 0644,
Size: uint64(fileSize),
}
node.Content = fs.saveFile(ctx, uploader, fakeFile(fileSeed, fileSize))
tree.Nodes = append(tree.Nodes, node)
nodes = append(nodes, node)
}
tree.Sort()
return TestSaveNodes(fs.t, ctx, uploader, nodes)
}
id, err := SaveTree(ctx, uploader, &tree)
if err != nil {
fs.t.Fatalf("SaveTree returned error: %v", err)
//nolint:revive // as this is a test helper, t should go first
func TestSaveNodes(t testing.TB, ctx context.Context, uploader restic.BlobSaver, nodes []*Node) restic.ID {
slices.SortFunc(nodes, func(a, b *Node) int {
return strings.Compare(a.Name, b.Name)
})
treeWriter := NewTreeWriter(uploader)
for _, node := range nodes {
err := treeWriter.AddNode(node)
rtest.OK(t, err)
}
id, err := treeWriter.Finalize(ctx)
rtest.OK(t, err)
return id
}
@ -136,7 +146,7 @@ func TestCreateSnapshot(t testing.TB, repo restic.Repository, at time.Time, dept
}
var treeID restic.ID
test.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
treeID = fs.saveTree(ctx, uploader, seed, depth)
return nil
}))
@ -188,3 +198,49 @@ func TestLoadAllSnapshots(ctx context.Context, repo restic.ListerLoaderUnpacked,
return snapshots, nil
}
// TestTreeMap returns the trees from the map on LoadTree.
type TestTreeMap map[restic.ID][]byte
func (t TestTreeMap) LoadBlob(_ context.Context, tpe restic.BlobType, id restic.ID, _ []byte) ([]byte, error) {
if tpe != restic.TreeBlob {
return nil, fmt.Errorf("can only load trees")
}
tree, ok := t[id]
if !ok {
return nil, fmt.Errorf("tree not found")
}
return tree, nil
}
func (t TestTreeMap) Connections() uint {
return 2
}
// TestWritableTreeMap also support saving
type TestWritableTreeMap struct {
TestTreeMap
}
func (t TestWritableTreeMap) SaveBlob(_ context.Context, tpe restic.BlobType, buf []byte, id restic.ID, _ bool) (newID restic.ID, known bool, size int, err error) {
if tpe != restic.TreeBlob {
return restic.ID{}, false, 0, fmt.Errorf("can only save trees")
}
if id.IsNull() {
id = restic.Hash(buf)
}
_, ok := t.TestTreeMap[id]
if ok {
return id, false, 0, nil
}
t.TestTreeMap[id] = append([]byte{}, buf...)
return id, true, len(buf), nil
}
func (t TestWritableTreeMap) Dump(test testing.TB) {
for k, v := range t.TestTreeMap {
test.Logf("%v: %v", k, string(v))
}
}

478
internal/data/tree.go
View file

@ -5,146 +5,247 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"iter"
"path"
"sort"
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/debug"
)
// Tree is an ordered list of nodes.
type Tree struct {
Nodes []*Node `json:"nodes"`
}
// NewTree creates a new tree object with the given initial capacity.
func NewTree(capacity int) *Tree {
return &Tree{
Nodes: make([]*Node, 0, capacity),
}
}
func (t *Tree) String() string {
return fmt.Sprintf("Tree<%d nodes>", len(t.Nodes))
}
// Equals returns true if t and other have exactly the same nodes.
func (t *Tree) Equals(other *Tree) bool {
if len(t.Nodes) != len(other.Nodes) {
debug.Log("tree.Equals(): trees have different number of nodes")
return false
}
for i := 0; i < len(t.Nodes); i++ {
if !t.Nodes[i].Equals(*other.Nodes[i]) {
debug.Log("tree.Equals(): node %d is different:", i)
debug.Log(" %#v", t.Nodes[i])
debug.Log(" %#v", other.Nodes[i])
return false
}
}
return true
}
// Insert adds a new node at the correct place in the tree.
func (t *Tree) Insert(node *Node) error {
pos, found := t.find(node.Name)
if found != nil {
return errors.Errorf("node %q already present", node.Name)
}
// https://github.com/golang/go/wiki/SliceTricks
t.Nodes = append(t.Nodes, nil)
copy(t.Nodes[pos+1:], t.Nodes[pos:])
t.Nodes[pos] = node
return nil
}
func (t *Tree) find(name string) (int, *Node) {
pos := sort.Search(len(t.Nodes), func(i int) bool {
return t.Nodes[i].Name >= name
})
if pos < len(t.Nodes) && t.Nodes[pos].Name == name {
return pos, t.Nodes[pos]
}
return pos, nil
}
// Find returns a node with the given name, or nil if none could be found.
func (t *Tree) Find(name string) *Node {
if t == nil {
return nil
}
_, node := t.find(name)
return node
}
// Sort sorts the nodes by name.
func (t *Tree) Sort() {
list := Nodes(t.Nodes)
sort.Sort(list)
t.Nodes = list
}
// Subtrees returns a slice of all subtree IDs of the tree.
func (t *Tree) Subtrees() (trees restic.IDs) {
for _, node := range t.Nodes {
if node.Type == NodeTypeDir && node.Subtree != nil {
trees = append(trees, *node.Subtree)
}
}
return trees
}
// LoadTree loads a tree from the repository.
func LoadTree(ctx context.Context, r restic.BlobLoader, id restic.ID) (*Tree, error) {
debug.Log("load tree %v", id)
buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil)
if err != nil {
return nil, err
}
t := &Tree{}
err = json.Unmarshal(buf, t)
if err != nil {
return nil, err
}
return t, nil
}
// SaveTree stores a tree into the repository and returns the ID. The ID is
// checked against the index. The tree is only stored when the index does not
// contain the ID.
func SaveTree(ctx context.Context, r restic.BlobSaver, t *Tree) (restic.ID, error) {
buf, err := json.Marshal(t)
if err != nil {
return restic.ID{}, errors.Wrap(err, "MarshalJSON")
}
// append a newline so that the data is always consistent (json.Encoder
// adds a newline after each object)
buf = append(buf, '\n')
id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
return id, err
}
// For documentation purposes only:
// // Tree is an ordered list of nodes.
// type Tree struct {
// Nodes []*Node `json:"nodes"`
// }
var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate")
type treeIterator struct {
dec json.Decoder
started bool
}
type NodeOrError struct {
Node *Node
Error error
}
type TreeNodeIterator = iter.Seq[NodeOrError]
func NewTreeNodeIterator(rd io.Reader) (TreeNodeIterator, error) {
t := &treeIterator{
dec: *json.NewDecoder(rd),
}
err := t.init()
if err != nil {
return nil, err
}
return func(yield func(NodeOrError) bool) {
if t.started {
panic("tree iterator is single use only")
}
t.started = true
for {
n, err := t.next()
if err != nil && errors.Is(err, io.EOF) {
return
}
if !yield(NodeOrError{Node: n, Error: err}) {
return
}
// errors are final
if err != nil {
return
}
}
}, nil
}
func (t *treeIterator) init() error {
// A tree is expected to be encoded as a JSON object with a single key "nodes".
// However, for future-proofness, we allow unknown keys before and after the "nodes" key.
// The following is the expected format:
// `{"nodes":[...]}`
if err := t.assertToken(json.Delim('{')); err != nil {
return err
}
// Skip unknown keys until we find "nodes"
for {
token, err := t.dec.Token()
if err != nil {
return err
}
key, ok := token.(string)
if !ok {
return errors.Errorf("error decoding tree: expected string key, got %v", token)
}
if key == "nodes" {
// Found "nodes", proceed to read the array
if err := t.assertToken(json.Delim('[')); err != nil {
return err
}
return nil
}
// Unknown key, decode its value into RawMessage and discard it
var raw json.RawMessage
if err := t.dec.Decode(&raw); err != nil {
return err
}
}
}
func (t *treeIterator) next() (*Node, error) {
if t.dec.More() {
var n Node
err := t.dec.Decode(&n)
if err != nil {
return nil, err
}
return &n, nil
}
if err := t.assertToken(json.Delim(']')); err != nil {
return nil, err
}
// Skip unknown keys after the array until we find the closing brace
for {
token, err := t.dec.Token()
if err != nil {
return nil, err
}
if token == json.Delim('}') {
return nil, io.EOF
}
// We have an unknown key, decode its value into RawMessage and discard it
var raw json.RawMessage
if err := t.dec.Decode(&raw); err != nil {
return nil, err
}
}
}
func (t *treeIterator) assertToken(token json.Token) error {
to, err := t.dec.Token()
if err != nil {
return err
}
if to != token {
return errors.Errorf("error decoding tree: expected %v, got %v", token, to)
}
return nil
}
func LoadTree(ctx context.Context, loader restic.BlobLoader, content restic.ID) (TreeNodeIterator, error) {
rd, err := loader.LoadBlob(ctx, restic.TreeBlob, content, nil)
if err != nil {
return nil, err
}
return NewTreeNodeIterator(bytes.NewReader(rd))
}
type TreeFinder struct {
next func() (NodeOrError, bool)
stop func()
current *Node
last string
}
func NewTreeFinder(tree TreeNodeIterator) *TreeFinder {
if tree == nil {
return &TreeFinder{stop: func() {}}
}
next, stop := iter.Pull(tree)
return &TreeFinder{next: next, stop: stop}
}
// Find finds the node with the given name. If the node is not found, it returns nil.
// If Find was called before, the new name must be strictly greater than the last name.
func (t *TreeFinder) Find(name string) (*Node, error) {
if t.next == nil {
return nil, nil
}
if name <= t.last {
return nil, errors.Errorf("name %q is not greater than last name %q", name, t.last)
}
t.last = name
// loop until `t.current.Name` is >= name
for t.current == nil || t.current.Name < name {
current, ok := t.next()
if current.Error != nil {
return nil, current.Error
}
if !ok {
return nil, nil
}
t.current = current.Node
}
if t.current.Name == name {
// forget the current node to free memory as early as possible
current := t.current
t.current = nil
return current, nil
}
// we have already passed the name
return nil, nil
}
func (t *TreeFinder) Close() {
t.stop()
}
type TreeWriter struct {
builder *TreeJSONBuilder
saver restic.BlobSaver
}
func NewTreeWriter(saver restic.BlobSaver) *TreeWriter {
builder := NewTreeJSONBuilder()
return &TreeWriter{builder: builder, saver: saver}
}
func (t *TreeWriter) AddNode(node *Node) error {
return t.builder.AddNode(node)
}
func (t *TreeWriter) Finalize(ctx context.Context) (restic.ID, error) {
buf, err := t.builder.Finalize()
if err != nil {
return restic.ID{}, err
}
id, _, _, err := t.saver.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false)
return id, err
}
// Count returns the number of nodes in the tree
func (t *TreeWriter) Count() int {
return t.builder.Count()
}
func SaveTree(ctx context.Context, saver restic.BlobSaver, nodes TreeNodeIterator) (restic.ID, error) {
treeWriter := NewTreeWriter(saver)
for item := range nodes {
if item.Error != nil {
return restic.ID{}, item.Error
}
err := treeWriter.AddNode(item.Node)
if err != nil {
return restic.ID{}, err
}
}
return treeWriter.Finalize(ctx)
}
type TreeJSONBuilder struct {
buf bytes.Buffer
lastName string
buf bytes.Buffer
lastName string
countNodes int
}
func NewTreeJSONBuilder() *TreeJSONBuilder {
@ -167,6 +268,7 @@ func (builder *TreeJSONBuilder) AddNode(node *Node) error {
return err
}
_, _ = builder.buf.Write(val)
builder.countNodes++
return nil
}
@ -180,6 +282,11 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) {
return buf, nil
}
// Count returns the number of nodes in the tree
func (builder *TreeJSONBuilder) Count() int {
return builder.countNodes
}
func FindTreeDirectory(ctx context.Context, repo restic.BlobLoader, id *restic.ID, dir string) (*restic.ID, error) {
if id == nil {
return nil, errors.New("tree id is null")
@ -197,7 +304,12 @@ func FindTreeDirectory(ctx context.Context, repo restic.BlobLoader, id *restic.I
if err != nil {
return nil, fmt.Errorf("path %s: %w", subfolder, err)
}
node := tree.Find(name)
finder := NewTreeFinder(tree)
node, err := finder.Find(name)
finder.Close()
if err != nil {
return nil, fmt.Errorf("path %s: %w", subfolder, err)
}
if node == nil {
return nil, fmt.Errorf("path %s: not found", subfolder)
}
@ -218,5 +330,109 @@ func FindTreeNode(ctx context.Context, repo restic.BlobLoader, id *restic.ID, no
if err != nil {
return nil, err
}
return tree.Find(path.Base(nodepath)), nil
finder := NewTreeFinder(tree)
defer finder.Close()
return finder.Find(path.Base(nodepath))
}
type peekableNodeIterator struct {
iter func() (NodeOrError, bool)
stop func()
value *Node
}
func newPeekableNodeIterator(tree TreeNodeIterator) (*peekableNodeIterator, error) {
iter, stop := iter.Pull(tree)
it := &peekableNodeIterator{iter: iter, stop: stop}
err := it.Next()
if err != nil {
it.Close()
return nil, err
}
return it, nil
}
func (i *peekableNodeIterator) Next() error {
item, ok := i.iter()
if item.Error != nil || !ok {
i.value = nil
return item.Error
}
i.value = item.Node
return nil
}
func (i *peekableNodeIterator) Peek() *Node {
return i.value
}
func (i *peekableNodeIterator) Close() {
i.stop()
}
type DualTree struct {
Tree1 *Node
Tree2 *Node
Error error
}
// DualTreeIterator iterates over two trees in parallel. It returns a sequence of DualTree structs.
// The sequence is terminated when both trees are exhausted. The error field must be checked before
// accessing any of the nodes.
func DualTreeIterator(tree1, tree2 TreeNodeIterator) iter.Seq[DualTree] {
started := false
return func(yield func(DualTree) bool) {
if started {
panic("tree iterator is single use only")
}
started = true
iter1, err := newPeekableNodeIterator(tree1)
if err != nil {
yield(DualTree{Tree1: nil, Tree2: nil, Error: err})
return
}
defer iter1.Close()
iter2, err := newPeekableNodeIterator(tree2)
if err != nil {
yield(DualTree{Tree1: nil, Tree2: nil, Error: err})
return
}
defer iter2.Close()
for {
node1 := iter1.Peek()
node2 := iter2.Peek()
if node1 == nil && node2 == nil {
// both iterators are exhausted
break
} else if node1 != nil && node2 != nil {
// if both nodes have a different name, only keep the first one
if node1.Name < node2.Name {
node2 = nil
} else if node1.Name > node2.Name {
node1 = nil
}
}
// non-nil nodes will be processed in the following, so advance the corresponding iterator
if node1 != nil {
if err = iter1.Next(); err != nil {
break
}
}
if node2 != nil {
if err = iter2.Next(); err != nil {
break
}
}
if !yield(DualTree{Tree1: node1, Tree2: node2, Error: err}) {
return
}
}
if err != nil {
yield(DualTree{Tree1: nil, Tree2: nil, Error: err})
return
}
}
}

154
internal/data/tree_stream.go
View file

@ -2,26 +2,20 @@ package data
import (
"context"
"errors"
"runtime"
"sync"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
)
// TreeItem is used to return either an error or the tree for a tree id
type TreeItem struct {
restic.ID
Error error
*Tree
}
type trackedTreeItem struct {
TreeItem
rootIdx int
restic.ID
Subtrees restic.IDs
rootIdx int
}
type trackedID struct {
@ -29,35 +23,87 @@ type trackedID struct {
rootIdx int
}
// subtreesCollector wraps a TreeNodeIterator and returns a new iterator that collects the subtrees.
func subtreesCollector(tree TreeNodeIterator) (TreeNodeIterator, func() restic.IDs) {
subtrees := restic.IDs{}
isComplete := false
return func(yield func(NodeOrError) bool) {
for item := range tree {
if !yield(item) {
return
}
// be defensive and check for nil subtree as this code is also used by the checker
if item.Node != nil && item.Node.Type == NodeTypeDir && item.Node.Subtree != nil {
subtrees = append(subtrees, *item.Node.Subtree)
}
}
isComplete = true
}, func() restic.IDs {
if !isComplete {
panic("tree was not read completely")
}
return subtrees
}
}
// loadTreeWorker loads trees from repo and sends them to out.
func loadTreeWorker(ctx context.Context, repo restic.Loader,
in <-chan trackedID, out chan<- trackedTreeItem) {
func loadTreeWorker(
ctx context.Context,
repo restic.Loader,
in <-chan trackedID,
process func(id restic.ID, error error, nodes TreeNodeIterator) error,
out chan<- trackedTreeItem,
) error {
for treeID := range in {
tree, err := LoadTree(ctx, repo, treeID.ID)
if tree == nil && err == nil {
err = errors.New("tree is nil and error is nil")
}
debug.Log("load tree %v (%v) returned err: %v", tree, treeID, err)
job := trackedTreeItem{TreeItem: TreeItem{ID: treeID.ID, Error: err, Tree: tree}, rootIdx: treeID.rootIdx}
// wrap iterator to collect subtrees while `process` iterates over `tree`
var collectSubtrees func() restic.IDs
if tree != nil {
tree, collectSubtrees = subtreesCollector(tree)
}
err = process(treeID.ID, err, tree)
if err != nil {
return err
}
// assume that the number of subtrees is within reasonable limits, such that the memory usage is not a problem
var subtrees restic.IDs
if collectSubtrees != nil {
subtrees = collectSubtrees()
}
job := trackedTreeItem{ID: treeID.ID, Subtrees: subtrees, rootIdx: treeID.rootIdx}
select {
case <-ctx.Done():
return
return nil
case out <- job:
}
}
return nil
}
// filterTree receives the result of a tree load and queues new trees for loading and processing.
func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID,
in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree restic.ID) bool, p *progress.Counter) {
in <-chan trackedTreeItem, skip func(tree restic.ID) bool, p *progress.Counter) {
var (
inCh = in
outCh chan<- TreeItem
loadCh chan<- trackedID
job TreeItem
nextTreeID trackedID
outstandingLoadTreeJobs = 0
)
// tracks how many trees are currently waiting to be processed for a given root tree
rootCounter := make([]int, len(trees))
// build initial backlog
backlog := make([]trackedID, 0, len(trees))
for idx, id := range trees {
backlog = append(backlog, trackedID{ID: id, rootIdx: idx})
@ -65,6 +111,7 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
}
for {
// if no tree is waiting to be sent, pick the next one
if loadCh == nil && len(backlog) > 0 {
// process last added ids first, that is traverse the tree in depth-first order
ln := len(backlog) - 1
@ -86,7 +133,8 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
}
}
if loadCh == nil && outCh == nil && outstandingLoadTreeJobs == 0 {
// loadCh is only nil at this point if the backlog is empty
if loadCh == nil && outstandingLoadTreeJobs == 0 {
debug.Log("backlog is empty, all channels nil, exiting")
return
}
@ -103,7 +151,6 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
if !ok {
debug.Log("input channel closed")
inCh = nil
in = nil
continue
}
@ -111,58 +158,47 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
rootCounter[j.rootIdx]--
debug.Log("input job tree %v", j.ID)
if j.Error != nil {
debug.Log("received job with error: %v (tree %v, ID %v)", j.Error, j.Tree, j.ID)
} else if j.Tree == nil {
debug.Log("received job with nil tree pointer: %v (ID %v)", j.Error, j.ID)
// send a new job with the new error instead of the old one
j = trackedTreeItem{TreeItem: TreeItem{ID: j.ID, Error: errors.New("tree is nil and error is nil")}, rootIdx: j.rootIdx}
} else {
subtrees := j.Tree.Subtrees()
debug.Log("subtrees for tree %v: %v", j.ID, subtrees)
// iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection
for i := len(subtrees) - 1; i >= 0; i-- {
id := subtrees[i]
if id.IsNull() {
// We do not need to raise this error here, it is
// checked when the tree is checked. Just make sure
// that we do not add any null IDs to the backlog.
debug.Log("tree %v has nil subtree", j.ID)
continue
}
backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx})
rootCounter[j.rootIdx]++
// iterate backwards over subtree to compensate backwards traversal order of nextTreeID selection
for i := len(j.Subtrees) - 1; i >= 0; i-- {
id := j.Subtrees[i]
if id.IsNull() {
// We do not need to raise this error here, it is
// checked when the tree is checked. Just make sure
// that we do not add any null IDs to the backlog.
debug.Log("tree %v has nil subtree", j.ID)
continue
}
backlog = append(backlog, trackedID{ID: id, rootIdx: j.rootIdx})
rootCounter[j.rootIdx]++
}
// the progress check must happen after j.Subtrees was added to the backlog
if p != nil && rootCounter[j.rootIdx] == 0 {
p.Add(1)
}
job = j.TreeItem
outCh = out
inCh = nil
case outCh <- job:
debug.Log("tree sent to process: %v", job.ID)
outCh = nil
inCh = in
}
}
}
// StreamTrees iteratively loads the given trees and their subtrees. The skip method
// is guaranteed to always be called from the same goroutine. To shutdown the started
// goroutines, either read all items from the channel or cancel the context. Then `Wait()`
// on the errgroup until all goroutines were stopped.
func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, trees restic.IDs, skip func(tree restic.ID) bool, p *progress.Counter) <-chan TreeItem {
// is guaranteed to always be called from the same goroutine. The process function is
// directly called from the worker goroutines. It MUST read `nodes` until it returns an
// error or completes. If the process function returns an error, then StreamTrees will
// abort and return the error.
func StreamTrees(
ctx context.Context,
repo restic.Loader,
trees restic.IDs,
p *progress.Counter,
skip func(tree restic.ID) bool,
process func(id restic.ID, error error, nodes TreeNodeIterator) error,
) error {
loaderChan := make(chan trackedID)
hugeTreeChan := make(chan trackedID, 10)
loadedTreeChan := make(chan trackedTreeItem)
treeStream := make(chan TreeItem)
var loadTreeWg sync.WaitGroup
wg, ctx := errgroup.WithContext(ctx)
// decoding a tree can take quite some time such that this can be both CPU- or IO-bound
// one extra worker to handle huge tree blobs
workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) + 1
@ -174,8 +210,7 @@ func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, tr
loadTreeWg.Add(1)
wg.Go(func() error {
defer loadTreeWg.Done()
loadTreeWorker(ctx, repo, workerLoaderChan, loadedTreeChan)
return nil
return loadTreeWorker(ctx, repo, workerLoaderChan, process, loadedTreeChan)
})
}
@ -189,9 +224,8 @@ func StreamTrees(ctx context.Context, wg *errgroup.Group, repo restic.Loader, tr
wg.Go(func() error {
defer close(loaderChan)
defer close(hugeTreeChan)
defer close(treeStream)
filterTrees(ctx, repo, trees, loaderChan, hugeTreeChan, loadedTreeChan, treeStream, skip, p)
filterTrees(ctx, repo, trees, loaderChan, hugeTreeChan, loadedTreeChan, skip, p)
return nil
})
return treeStream
return wg.Wait()
}

390
internal/data/tree_test.go
View file

@ -4,9 +4,12 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"testing"
"github.com/restic/restic/internal/archiver"
@ -105,37 +108,45 @@ func TestNodeComparison(t *testing.T) {
func TestEmptyLoadTree(t *testing.T) {
repo := repository.TestRepository(t)
tree := data.NewTree(0)
nodes := []*data.Node{}
var id restic.ID
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaver) error {
var err error
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
// save tree
id, err = data.SaveTree(ctx, uploader, tree)
return err
id = data.TestSaveNodes(t, ctx, uploader, nodes)
return nil
}))
// load tree again
tree2, err := data.LoadTree(context.TODO(), repo, id)
it, err := data.LoadTree(context.TODO(), repo, id)
rtest.OK(t, err)
nodes2 := []*data.Node{}
for item := range it {
rtest.OK(t, item.Error)
nodes2 = append(nodes2, item.Node)
}
rtest.Assert(t, tree.Equals(tree2),
"trees are not equal: want %v, got %v",
tree, tree2)
rtest.Assert(t, slices.Equal(nodes, nodes2),
"tree nodes are not equal: want %v, got %v",
nodes, nodes2)
}
// Basic type for comparing the serialization of the tree
type Tree struct {
Nodes []*data.Node `json:"nodes"`
}
func TestTreeEqualSerialization(t *testing.T) {
files := []string{"node.go", "tree.go", "tree_test.go"}
for i := 1; i <= len(files); i++ {
tree := data.NewTree(i)
tree := Tree{Nodes: make([]*data.Node, 0, i)}
builder := data.NewTreeJSONBuilder()
for _, fn := range files[:i] {
node := nodeForFile(t, fn)
rtest.OK(t, tree.Insert(node))
tree.Nodes = append(tree.Nodes, node)
rtest.OK(t, builder.AddNode(node))
rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node")
rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node")
rtest.Assert(t, errors.Is(builder.AddNode(node), data.ErrTreeNotOrdered), "wrong error returned")
}
@ -144,14 +155,34 @@ func TestTreeEqualSerialization(t *testing.T) {
treeBytes = append(treeBytes, '\n')
rtest.OK(t, err)
stiBytes, err := builder.Finalize()
buf, err := builder.Finalize()
rtest.OK(t, err)
// compare serialization of an individual node and the SaveTreeIterator
rtest.Equals(t, treeBytes, stiBytes)
rtest.Equals(t, treeBytes, buf)
}
}
func TestTreeLoadSaveCycle(t *testing.T) {
files := []string{"node.go", "tree.go", "tree_test.go"}
builder := data.NewTreeJSONBuilder()
for _, fn := range files {
node := nodeForFile(t, fn)
rtest.OK(t, builder.AddNode(node))
}
buf, err := builder.Finalize()
rtest.OK(t, err)
tm := data.TestTreeMap{restic.Hash(buf): buf}
it, err := data.LoadTree(context.TODO(), tm, restic.Hash(buf))
rtest.OK(t, err)
mtm := data.TestWritableTreeMap{TestTreeMap: data.TestTreeMap{}}
id, err := data.SaveTree(context.TODO(), mtm, it)
rtest.OK(t, err)
rtest.Equals(t, restic.Hash(buf), id, "saved tree id mismatch")
}
func BenchmarkBuildTree(b *testing.B) {
const size = 100 // Directories of this size are not uncommon.
@ -165,11 +196,12 @@ func BenchmarkBuildTree(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
t := data.NewTree(size)
t := data.NewTreeJSONBuilder()
for i := range nodes {
_ = t.Insert(&nodes[i])
rtest.OK(b, t.AddNode(&nodes[i]))
}
_, err := t.Finalize()
rtest.OK(b, err)
}
}
@ -186,8 +218,80 @@ func testLoadTree(t *testing.T, version uint) {
repo, _, _ := repository.TestRepositoryWithVersion(t, version)
sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
_, err := data.LoadTree(context.TODO(), repo, *sn.Tree)
nodes, err := data.LoadTree(context.TODO(), repo, *sn.Tree)
rtest.OK(t, err)
for item := range nodes {
rtest.OK(t, item.Error)
}
}
func TestTreeIteratorUnknownKeys(t *testing.T) {
tests := []struct {
name string
jsonData string
wantNodes []string
}{
{
name: "unknown key before nodes",
jsonData: `{"extra": "value", "nodes": [{"name": "test1"}, {"name": "test2"}]}`,
wantNodes: []string{"test1", "test2"},
},
{
name: "unknown key after nodes",
jsonData: `{"nodes": [{"name": "test1"}, {"name": "test2"}], "extra": "value"}`,
wantNodes: []string{"test1", "test2"},
},
{
name: "multiple unknown keys before nodes",
jsonData: `{"key1": "value1", "key2": 42, "nodes": [{"name": "test1"}]}`,
wantNodes: []string{"test1"},
},
{
name: "multiple unknown keys after nodes",
jsonData: `{"nodes": [{"name": "test1"}], "key1": "value1", "key2": 42}`,
wantNodes: []string{"test1"},
},
{
name: "unknown keys before and after nodes",
jsonData: `{"before": "value", "nodes": [{"name": "test1"}], "after": "value"}`,
wantNodes: []string{"test1"},
},
{
name: "nested object as unknown value",
jsonData: `{"extra": {"nested": "value"}, "nodes": [{"name": "test1"}]}`,
wantNodes: []string{"test1"},
},
{
name: "nested array as unknown value",
jsonData: `{"extra": [1, 2, 3], "nodes": [{"name": "test1"}]}`,
wantNodes: []string{"test1"},
},
{
name: "complex nested structure as unknown value",
jsonData: `{"extra": {"obj": {"arr": [1, {"nested": true}]}}, "nodes": [{"name": "test1"}]}`,
wantNodes: []string{"test1"},
},
{
name: "empty nodes array with unknown keys",
jsonData: `{"extra": "value", "nodes": []}`,
wantNodes: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
it, err := data.NewTreeNodeIterator(strings.NewReader(tt.jsonData + "\n"))
rtest.OK(t, err)
var gotNodes []string
for item := range it {
rtest.OK(t, item.Error)
gotNodes = append(gotNodes, item.Node.Name)
}
rtest.Equals(t, tt.wantNodes, gotNodes, "nodes mismatch")
})
}
}
func BenchmarkLoadTree(t *testing.B) {
@ -211,6 +315,58 @@ func benchmarkLoadTree(t *testing.B, version uint) {
}
}
func TestTreeFinderNilIterator(t *testing.T) {
finder := data.NewTreeFinder(nil)
defer finder.Close()
node, err := finder.Find("foo")
rtest.OK(t, err)
rtest.Equals(t, node, nil, "finder should return nil node")
}
func TestTreeFinderError(t *testing.T) {
testErr := errors.New("error")
finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{
{Node: &data.Node{Name: "a"}, Error: nil},
{Node: &data.Node{Name: "b"}, Error: nil},
{Node: nil, Error: testErr},
}))
defer finder.Close()
node, err := finder.Find("b")
rtest.OK(t, err)
rtest.Equals(t, node.Name, "b", "finder should return node with name b")
node, err = finder.Find("c")
rtest.Equals(t, err, testErr, "finder should return correcterror")
rtest.Equals(t, node, nil, "finder should return nil node")
}
func TestTreeFinderNotFound(t *testing.T) {
finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{
{Node: &data.Node{Name: "a"}, Error: nil},
}))
defer finder.Close()
node, err := finder.Find("b")
rtest.OK(t, err)
rtest.Equals(t, node, nil, "finder should return nil node")
// must also be ok multiple times
node, err = finder.Find("c")
rtest.OK(t, err)
rtest.Equals(t, node, nil, "finder should return nil node")
}
func TestTreeFinderWrongOrder(t *testing.T) {
finder := data.NewTreeFinder(slices.Values([]data.NodeOrError{
{Node: &data.Node{Name: "d"}, Error: nil},
}))
defer finder.Close()
node, err := finder.Find("b")
rtest.OK(t, err)
rtest.Equals(t, node, nil, "finder should return nil node")
node, err = finder.Find("a")
rtest.Assert(t, strings.Contains(err.Error(), "is not greater than"), "unexpected error: %v", err)
rtest.Equals(t, node, nil, "finder should return nil node")
}
func TestFindTreeDirectory(t *testing.T) {
repo := repository.TestRepository(t)
sn := data.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3)
@ -220,15 +376,15 @@ func TestFindTreeDirectory(t *testing.T) {
id restic.ID
err error
}{
{"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{"", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
{"/", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
{".", restic.TestParseID("8804a5505fc3012e7d08b2843e9bda1bf3dc7644f64b542470340e1b4059f09f"), nil},
{"..", restic.ID{}, errors.New("path ..: not found")},
{"file-1", restic.ID{}, errors.New("path file-1: not a directory")},
{"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil},
{"dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
{"/dir-7", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
{"dir-7/", restic.TestParseID("1af51eb70cd4457d51db40d649bb75446a3eaa29b265916d411bb7ae971d4849"), nil},
{"dir-7/dir-5", restic.TestParseID("f05534d2673964de698860e5069da1ee3c198acf21c187975c6feb49feb8e9c9"), nil},
} {
t.Run("", func(t *testing.T) {
id, err := data.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder)
@ -244,3 +400,187 @@ func TestFindTreeDirectory(t *testing.T) {
_, err := data.FindTreeDirectory(context.TODO(), repo, nil, "")
rtest.Assert(t, err != nil, "missing error on null tree id")
}
func TestDualTreeIterator(t *testing.T) {
testErr := errors.New("test error")
tests := []struct {
name string
tree1 []data.NodeOrError
tree2 []data.NodeOrError
expected []data.DualTree
}{
{
name: "both empty",
tree1: []data.NodeOrError{},
tree2: []data.NodeOrError{},
expected: []data.DualTree{},
},
{
name: "tree1 empty",
tree1: []data.NodeOrError{},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "b"}},
},
expected: []data.DualTree{
{Tree1: nil, Tree2: &data.Node{Name: "a"}, Error: nil},
{Tree1: nil, Tree2: &data.Node{Name: "b"}, Error: nil},
},
},
{
name: "tree2 empty",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "b"}},
},
tree2: []data.NodeOrError{},
expected: []data.DualTree{
{Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil},
{Tree1: &data.Node{Name: "b"}, Tree2: nil, Error: nil},
},
},
{
name: "identical trees",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "b"}},
},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "b"}},
},
expected: []data.DualTree{
{Tree1: &data.Node{Name: "a"}, Tree2: &data.Node{Name: "a"}, Error: nil},
{Tree1: &data.Node{Name: "b"}, Tree2: &data.Node{Name: "b"}, Error: nil},
},
},
{
name: "disjoint trees",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "c"}},
},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "b"}},
{Node: &data.Node{Name: "d"}},
},
expected: []data.DualTree{
{Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil},
{Tree1: nil, Tree2: &data.Node{Name: "b"}, Error: nil},
{Tree1: &data.Node{Name: "c"}, Tree2: nil, Error: nil},
{Tree1: nil, Tree2: &data.Node{Name: "d"}, Error: nil},
},
},
{
name: "overlapping trees",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Node: &data.Node{Name: "b"}},
{Node: &data.Node{Name: "d"}},
},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "b"}},
{Node: &data.Node{Name: "c"}},
{Node: &data.Node{Name: "d"}},
},
expected: []data.DualTree{
{Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil},
{Tree1: &data.Node{Name: "b"}, Tree2: &data.Node{Name: "b"}, Error: nil},
{Tree1: nil, Tree2: &data.Node{Name: "c"}, Error: nil},
{Tree1: &data.Node{Name: "d"}, Tree2: &data.Node{Name: "d"}, Error: nil},
},
},
{
name: "error in tree1 during iteration",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
{Error: testErr},
},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "c"}},
},
expected: []data.DualTree{
{Tree1: nil, Tree2: nil, Error: testErr},
},
},
{
name: "error in tree2 during iteration",
tree1: []data.NodeOrError{
{Node: &data.Node{Name: "a"}},
},
tree2: []data.NodeOrError{
{Node: &data.Node{Name: "b"}},
{Error: testErr},
},
expected: []data.DualTree{
{Tree1: &data.Node{Name: "a"}, Tree2: nil, Error: nil},
{Tree1: nil, Tree2: nil, Error: testErr},
},
},
{
name: "error at start of tree1",
tree1: []data.NodeOrError{{Error: testErr}},
tree2: []data.NodeOrError{{Node: &data.Node{Name: "b"}}},
expected: []data.DualTree{
{Tree1: nil, Tree2: nil, Error: testErr},
},
},
{
name: "error at start of tree2",
tree1: []data.NodeOrError{{Node: &data.Node{Name: "a"}}},
tree2: []data.NodeOrError{{Error: testErr}},
expected: []data.DualTree{
{Tree1: nil, Tree2: nil, Error: testErr},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
iter1 := slices.Values(tt.tree1)
iter2 := slices.Values(tt.tree2)
dualIter := data.DualTreeIterator(iter1, iter2)
var results []data.DualTree
for dt := range dualIter {
results = append(results, dt)
}
rtest.Equals(t, len(tt.expected), len(results), "unexpected number of results")
for i, exp := range tt.expected {
rtest.Equals(t, exp.Error, results[i].Error, fmt.Sprintf("error mismatch at index %d", i))
rtest.Equals(t, exp.Tree1, results[i].Tree1, fmt.Sprintf("Tree1 mismatch at index %d", i))
rtest.Equals(t, exp.Tree2, results[i].Tree2, fmt.Sprintf("Tree2 mismatch at index %d", i))
}
})
}
t.Run("single use restriction", func(t *testing.T) {
iter1 := slices.Values([]data.NodeOrError{{Node: &data.Node{Name: "a"}}})
iter2 := slices.Values([]data.NodeOrError{{Node: &data.Node{Name: "b"}}})
dualIter := data.DualTreeIterator(iter1, iter2)
// First use should work
var count int
for range dualIter {
count++
}
rtest.Assert(t, count > 0, "first iteration should produce results")
// Second use should panic
func() {
defer func() {
if r := recover(); r == nil {
t.Fatal("expected panic on second use")
}
}()
count = 0
for range dualIter {
// Should panic before reaching here
count++
}
rtest.Equals(t, count, 0, "expected count to be 0")
}()
})
}

1
internal/debug/round_tripper_debug.go
View file

@ -1,5 +1,4 @@
//go:build debug
// +build debug
package debug

1
internal/debug/round_tripper_release.go
View file

@ -1,5 +1,4 @@
//go:build !debug
// +build !debug
package debug

43
internal/dump/common.go
View file

@ -30,33 +30,42 @@ func New(format string, repo restic.Loader, w io.Writer) *Dumper {
}
}
func (d *Dumper) DumpTree(ctx context.Context, tree *data.Tree, rootPath string) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
func (d *Dumper) DumpTree(ctx context.Context, tree data.TreeNodeIterator, rootPath string) error {
wg, ctx := errgroup.WithContext(ctx)
// ch is buffered to deal with variable download/write speeds.
ch := make(chan *data.Node, 10)
go sendTrees(ctx, d.repo, tree, rootPath, ch)
wg.Go(func() error {
return sendTrees(ctx, d.repo, tree, rootPath, ch)
})
switch d.format {
case "tar":
return d.dumpTar(ctx, ch)
case "zip":
return d.dumpZip(ctx, ch)
default:
panic("unknown dump format")
}
wg.Go(func() error {
switch d.format {
case "tar":
return d.dumpTar(ctx, ch)
case "zip":
return d.dumpZip(ctx, ch)
default:
panic("unknown dump format")
}
})
return wg.Wait()
}
func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *data.Tree, rootPath string, ch chan *data.Node) {
func sendTrees(ctx context.Context, repo restic.BlobLoader, nodes data.TreeNodeIterator, rootPath string, ch chan *data.Node) error {
defer close(ch)
for _, root := range tree.Nodes {
root.Path = path.Join(rootPath, root.Name)
if sendNodes(ctx, repo, root, ch) != nil {
break
for item := range nodes {
if item.Error != nil {
return item.Error
}
node := item.Node
node.Path = path.Join(rootPath, node.Name)
if err := sendNodes(ctx, repo, node, ch); err != nil {
return err
}
}
return nil
}
func sendNodes(ctx context.Context, repo restic.BlobLoader, root *data.Node, ch chan *data.Node) error {

27
internal/dump/common_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
@ -13,13 +14,13 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.Repository) {
func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.Repository, backend.Backend) {
tempdir := rtest.TempDir(t)
repo := repository.TestRepository(t)
repo, _, be := repository.TestRepositoryWithVersion(t, 0)
archiver.TestCreateFiles(t, tempdir, src)
return tempdir, repo
return tempdir, repo, be
}
type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error
@ -67,13 +68,22 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
},
target: "/",
},
{
name: "directory only",
args: archiver.TestDir{
"firstDir": archiver.TestDir{
"secondDir": archiver.TestDir{},
},
},
target: "/",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
tmpdir, repo := prepareTempdirRepoSrc(t, tt.args)
tmpdir, repo, be := prepareTempdirRepoSrc(t, tt.args)
arch := archiver.New(repo, fs.Track{FS: fs.Local{}}, archiver.Options{})
back := rtest.Chdir(t, tmpdir)
@ -93,6 +103,15 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
if err := cd(t, tmpdir, dst); err != nil {
t.Errorf("WriteDump() = does not match: %v", err)
}
// test that dump returns an error if the repository is broken
tree, err = data.LoadTree(ctx, repo, *sn.Tree)
rtest.OK(t, err)
rtest.OK(t, be.Delete(ctx))
// use new dumper as the old one has the blobs cached
d = New(format, repo, dst)
err = d.DumpTree(ctx, tree, tt.target)
rtest.Assert(t, err != nil, "expected error, got nil")
})
}
}

4
internal/filter/include.go
View file

@ -25,6 +25,10 @@ func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
}
func (opts *IncludePatternOptions) Empty() bool {
return len(opts.Includes) == 0 && len(opts.InsensitiveIncludes) == 0 && len(opts.IncludeFiles) == 0 && len(opts.InsensitiveIncludeFiles) == 0
}
func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
var fs []IncludeByNameFunc
if len(opts.IncludeFiles) > 0 {

1
internal/fs/const_unix.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package fs

1
internal/fs/const_windows.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

1
internal/fs/ea_windows.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

1
internal/fs/ea_windows_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

1
internal/fs/file_unix.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package fs

1
internal/fs/fs_local_vss_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

1
internal/fs/mknod_unix.go
View file

@ -1,5 +1,4 @@
//go:build !freebsd && !windows
// +build !freebsd,!windows
package fs

1
internal/fs/node_freebsd.go
View file

@ -1,5 +1,4 @@
//go:build freebsd
// +build freebsd
package fs

1
internal/fs/node_noxattr.go
View file

@ -1,5 +1,4 @@
//go:build aix || dragonfly || openbsd
// +build aix dragonfly openbsd
package fs

1
internal/fs/node_unix.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package fs

1
internal/fs/node_unix_test.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package fs

171
internal/fs/node_windows_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs
@ -56,6 +55,176 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir string, file
compareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath)
}
// TestRestoreSecurityDescriptorInheritance checks that the DACL protection (inheritance)
// flags are correctly restored. This is the mechanism that preserves the `IsInherited`
// property on individual ACEs.
func TestRestoreSecurityDescriptorInheritance(t *testing.T) {
// This test requires admin privileges to manipulate ACLs effectively.
isAdmin, err := isAdmin()
test.OK(t, err)
if !isAdmin {
t.Skip("Skipping inheritance test, requires admin privileges")
}
tempDir := t.TempDir()
// 1. Create a parent/child directory structure.
parentDir := filepath.Join(tempDir, "parent")
err = os.Mkdir(parentDir, 0755)
test.OK(t, err)
childDir := filepath.Join(parentDir, "child")
err = os.Mkdir(childDir, 0755)
test.OK(t, err)
// 2. Set inheritable permissions on the parent.
// We will give the "Users" group inheritable read access.
users, err := windows.StringToSid("S-1-5-32-545") // BUILTIN\Users
test.OK(t, err)
// Create an EXPLICIT_ACCESS structure for the new ACE.
explicitAccess := windows.EXPLICIT_ACCESS{
AccessPermissions: windows.GENERIC_READ,
AccessMode: windows.GRANT_ACCESS,
Inheritance: windows.OBJECT_INHERIT_ACE | windows.CONTAINER_INHERIT_ACE,
Trustee: windows.TRUSTEE{
TrusteeForm: windows.TRUSTEE_IS_SID,
TrusteeType: windows.TRUSTEE_IS_GROUP,
TrusteeValue: windows.TrusteeValueFromSID(users),
},
}
// Create a new DACL from the entry.
dacl, err := windows.ACLFromEntries([]windows.EXPLICIT_ACCESS{explicitAccess}, nil)
test.OK(t, err)
// Apply this new DACL to the parent, marking it as unprotected so it can be inherited.
err = windows.SetNamedSecurityInfo(
parentDir,
windows.SE_FILE_OBJECT,
windows.DACL_SECURITY_INFORMATION|windows.UNPROTECTED_DACL_SECURITY_INFORMATION,
nil, nil, dacl, nil,
)
test.OK(t, errors.Wrapf(err, "failed to set inheritable ACL on parent dir"))
// 3. Get the Security Descriptor of the child, which should now have inherited the ACE.
sdBytesOriginal, err := getSecurityDescriptor(childDir)
test.OK(t, err)
// Sanity check: verify the original child SD is NOT protected from inheritance.
sdOriginal, err := securityDescriptorBytesToStruct(*sdBytesOriginal)
test.OK(t, err)
control, _, err := sdOriginal.Control()
test.OK(t, err)
test.Assert(t, control&windows.SE_DACL_PROTECTED == 0, "Pre-condition failed: child directory should have inheritance enabled")
// 4. Create a restic node for the child directory.
genericAttrs, err := data.WindowsAttrsToGenericAttributes(data.WindowsAttributes{SecurityDescriptor: sdBytesOriginal})
test.OK(t, err)
childNode := getNode("child-restored", "dir", genericAttrs)
// 5. Restore the node to a new location.
restoreDir := filepath.Join(tempDir, "restore")
err = os.Mkdir(restoreDir, 0755)
test.OK(t, err)
restoredPath, _ := restoreAndGetNode(t, restoreDir, &childNode, false)
// 6. Get the Security Descriptor of the restored child directory.
sdBytesRestored, err := getSecurityDescriptor(restoredPath)
test.OK(t, err)
// 7. Compare the control flags of the original and restored SDs.
sdRestored, err := securityDescriptorBytesToStruct(*sdBytesRestored)
test.OK(t, err)
controlRestored, _, err := sdRestored.Control()
test.OK(t, err)
// The core of the test: Ensure the restored DACL protection flag matches the original.
originalIsProtected := (control & windows.SE_DACL_PROTECTED) != 0
restoredIsProtected := (controlRestored & windows.SE_DACL_PROTECTED) != 0
test.Equals(t, originalIsProtected, restoredIsProtected, "DACL protection flag was not restored correctly. Inheritance state is wrong.")
}
// TestRestoreSecurityDescriptorInheritanceLowPrivilege tests that the low-privilege restore
// path (setNamedSecurityInfoLow) correctly handles inheritance flags. This test doesn't require
// admin privileges and focuses on DACL restoration only.
func TestRestoreSecurityDescriptorInheritanceLowPrivilege(t *testing.T) {
tempDir := t.TempDir()
// 1. Create a test directory
testDir := filepath.Join(tempDir, "testdir")
err := os.Mkdir(testDir, 0755)
test.OK(t, err)
// 2. Get its security descriptor (which will have some default ACL)
sdBytesOriginal, err := getSecurityDescriptor(testDir)
test.OK(t, err)
// Verify we can get the control flags
sdOriginal, err := securityDescriptorBytesToStruct(*sdBytesOriginal)
test.OK(t, err)
controlOriginal, _, err := sdOriginal.Control()
test.OK(t, err)
// 3. Test both protected and unprotected scenarios by modifying the control flags
testCases := []struct {
name string
shouldBeProtected bool
}{
{"unprotected_inheritance_enabled", false},
{"protected_inheritance_disabled", true},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Create a copy of the SD bytes to modify
sdBytesTest := make([]byte, len(*sdBytesOriginal))
copy(sdBytesTest, *sdBytesOriginal)
// Modify the control flags to simulate protected/unprotected DACL
sdTest, err := securityDescriptorBytesToStruct(sdBytesTest)
test.OK(t, err)
// Get the DACL from the test SD
dacl, _, err := sdTest.DACL()
test.OK(t, err)
// Determine which control flag to use based on test case
var controlToUse windows.SECURITY_DESCRIPTOR_CONTROL
if tc.shouldBeProtected {
controlToUse = controlOriginal | windows.SE_DACL_PROTECTED
} else {
controlToUse = controlOriginal &^ windows.SE_DACL_PROTECTED
}
// 4. Call setNamedSecurityInfoLow directly to test the low-privilege path
restoreTarget := filepath.Join(tempDir, "restore_"+tc.name)
err = os.Mkdir(restoreTarget, 0755)
test.OK(t, err)
// This directly tests the low-privilege restore function
err = setNamedSecurityInfoLow(restoreTarget, dacl, controlToUse)
test.OK(t, err)
// 5. Get the security descriptor of the restored directory
sdBytesRestored, err := getSecurityDescriptor(restoreTarget)
test.OK(t, err)
// 6. Verify that the control flags were correctly restored
sdRestored, err := securityDescriptorBytesToStruct(*sdBytesRestored)
test.OK(t, err)
controlRestored, _, err := sdRestored.Control()
test.OK(t, err) // Check if the protection flag matches what we requested
restoredIsProtected := (controlRestored & windows.SE_DACL_PROTECTED) != 0
if tc.shouldBeProtected != restoredIsProtected {
t.Errorf("DACL protection flag was not restored correctly in low-privilege path. Expected protected=%v, got protected=%v",
tc.shouldBeProtected, restoredIsProtected)
}
})
}
}
func getNode(name string, fileType data.NodeType, genericAttributes map[data.GenericAttributeType]json.RawMessage) data.Node {
return data.Node{
Name: name,

1
internal/fs/node_xattr.go
View file

@ -1,5 +1,4 @@
//go:build darwin || freebsd || netbsd || linux || solaris
// +build darwin freebsd netbsd linux solaris
package fs

1
internal/fs/node_xattr_all_test.go
View file

@ -1,5 +1,4 @@
//go:build darwin || freebsd || netbsd || linux || solaris || windows
// +build darwin freebsd netbsd linux solaris windows
package fs

1
internal/fs/node_xattr_test.go
View file

@ -1,5 +1,4 @@
//go:build darwin || freebsd || netbsd || linux || solaris
// +build darwin freebsd netbsd linux solaris
package fs

1
internal/fs/preallocate_other.go
View file

@ -1,5 +1,4 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package fs

1
internal/fs/priv.go
View file

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
package fs

1
internal/fs/priv_windows.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

1
internal/fs/priv_windows_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

58
internal/fs/sd_windows.go
View file

@ -12,14 +12,17 @@ import (
var lowerPrivileges atomic.Bool
// Flags for backup and restore with admin permissions
var highSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.PROTECTED_SACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_SACL_SECURITY_INFORMATION
// Flags for backup with admin permissions. Includes protection flags for GET operations.
var highBackupSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.PROTECTED_SACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_SACL_SECURITY_INFORMATION
// Flags for restore with admin permissions. Base flags without protection flags for SET operations.
var highRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION
// Flags for backup without admin permissions. If there are no admin permissions, only the current user's owner, group and DACL will be backed up.
var lowBackupSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION
// Flags for restore without admin permissions. If there are no admin permissions, only the DACL from the SD can be restored and owner and group will be set based on the current user.
var lowRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.DACL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION
var lowRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.DACL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION
// getSecurityDescriptor takes the path of the file and returns the SecurityDescriptor for the file.
// This needs admin permissions or SeBackupPrivilege for getting the full SD.
@ -95,15 +98,21 @@ func setSecurityDescriptor(filePath string, securityDescriptor *[]byte) error {
sacl = nil
}
// Get the control flags from the original security descriptor
control, _, err := sd.Control()
if err != nil {
// This is unlikely to fail if the sd is valid, but handle it.
return fmt.Errorf("could not get security descriptor control flags: %w", err)
}
// store original value to avoid unrelated changes in the error check
useLowerPrivileges := lowerPrivileges.Load()
if useLowerPrivileges {
err = setNamedSecurityInfoLow(filePath, dacl)
err = setNamedSecurityInfoLow(filePath, dacl, control)
} else {
err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl)
err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl, control)
// See corresponding fallback in getSecurityDescriptor for an explanation
if err != nil && isAccessDeniedError(err) {
err = setNamedSecurityInfoLow(filePath, dacl)
err = setNamedSecurityInfoLow(filePath, dacl, control)
}
}
@ -121,7 +130,7 @@ func setSecurityDescriptor(filePath string, securityDescriptor *[]byte) error {
// getNamedSecurityInfoHigh gets the higher level SecurityDescriptor which requires admin permissions.
func getNamedSecurityInfoHigh(filePath string) (*windows.SECURITY_DESCRIPTOR, error) {
return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags)
return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highBackupSecurityFlags)
}
// getNamedSecurityInfoLow gets the lower level SecurityDescriptor which requires no admin permissions.
@ -130,13 +139,40 @@ func getNamedSecurityInfoLow(filePath string) (*windows.SECURITY_DESCRIPTOR, err
}
// setNamedSecurityInfoHigh sets the higher level SecurityDescriptor which requires admin permissions.
func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL) error {
return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags, owner, group, dacl, sacl)
func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL, control windows.SECURITY_DESCRIPTOR_CONTROL) error {
securityInfo := highRestoreSecurityFlags
// Check if the original DACL was protected from inheritance and add the correct flag.
if control&windows.SE_DACL_PROTECTED != 0 {
securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION
} else {
// Explicitly state that it is NOT protected. This ensures inheritance is re-enabled correctly.
securityInfo |= windows.UNPROTECTED_DACL_SECURITY_INFORMATION
}
// Do the same for the SACL for completeness.
if control&windows.SE_SACL_PROTECTED != 0 {
securityInfo |= windows.PROTECTED_SACL_SECURITY_INFORMATION
} else {
securityInfo |= windows.UNPROTECTED_SACL_SECURITY_INFORMATION
}
return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, securityInfo, owner, group, dacl, sacl)
}
// setNamedSecurityInfoLow sets the lower level SecurityDescriptor which requires no admin permissions.
func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL) error {
return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, lowRestoreSecurityFlags, nil, nil, dacl, nil)
func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL, control windows.SECURITY_DESCRIPTOR_CONTROL) error {
securityInfo := lowRestoreSecurityFlags
// Check if the original DACL was protected from inheritance and add the correct flag.
if control&windows.SE_DACL_PROTECTED != 0 {
securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION
} else {
// Explicitly state that it is NOT protected. This ensures inheritance is re-enabled correctly.
securityInfo |= windows.UNPROTECTED_DACL_SECURITY_INFORMATION
}
return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, securityInfo, nil, nil, dacl, nil)
}
// isHandlePrivilegeNotHeldError checks if the error is ERROR_PRIVILEGE_NOT_HELD

1
internal/fs/sd_windows_test.go
View file

@ -1,5 +1,4 @@
//go:build windows
// +build windows
package fs

Some files were not shown because too many files have changed in this diff Show more