mirror of
https://github.com/restic/restic.git
synced 2026-05-28 04:35:41 -04:00
Merge branch 'restic:master' into master
This commit is contained in:
Jean-Yves LENHOF
GitHub
commit
5be8042e32
20 changed files with 471 additions and 61 deletions
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
|
|
@ -36,7 +36,7 @@ Please always follow these steps:
|
|||
- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
|
||||
-->
|
||||
|
||||
- [ ] I have added tests for all code changes.
|
||||
- [ ] I have added tests for all code changes, see [writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests)
|
||||
- [ ] I have added documentation for relevant changes (in the manual).
|
||||
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
|
||||
- [ ] I'm done! This pull request is ready for review.
|
||||
|
|
|
|||
|
|
@ -202,6 +202,9 @@ we'll be glad to assist. Having a PR with failing integration tests is nothing
|
|||
to be ashamed of. In contrast, that happens regularly for all of us. That's
|
||||
what the tests are there for.
|
||||
|
||||
More details of how to structure tests can be found here at
|
||||
[writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests).
|
||||
|
||||
Git Commits
|
||||
-----------
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,20 @@
|
|||
# The first line must start with Bugfix:, Enhancement: or Change:,
|
||||
# including the colon. Use present tense and the imperative mood. Remove
|
||||
# lines starting with '#' from this template.
|
||||
Enhancement: Allow custom bar in the foo command
|
||||
# including the colon. 'Change:' is for breaking changes only.
|
||||
# Documentation-only changes do not get a changelog entry.
|
||||
# Include the affected command in the summary if relevant.
|
||||
#
|
||||
# Use present tense and the imperative mood. Remove lines starting
|
||||
# with '#' from this template.
|
||||
Enhancement: Allow custom bar in the `foo` command
|
||||
|
||||
# Describe the problem in the past tense, the new behavior in the present
|
||||
# tense. Mention the affected commands, backends, operating systems, etc.
|
||||
# If the problem description just says that a feature was missing, then
|
||||
# only explain the new behavior.
|
||||
# Focus on user-facing behavior, not the implementation.
|
||||
# only explain the new behavior. Aim for a short and concise description.
|
||||
# Use "Restic now ..." instead of "We have changed ...".
|
||||
#
|
||||
# Focus on user-facing behavior, not the implementation. The description should
|
||||
# be understandable for a regular user without knowledge of the implementation.
|
||||
|
||||
Restic foo always used the system-wide bar when deciding how to frob an
|
||||
item in the `baz` backend. It now permits selecting the bar with `--bar`
|
||||
|
|
|
|||
7
changelog/unreleased/issue-5280
Normal file
7
changelog/unreleased/issue-5280
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
Bugfix: `restic find` now checks for correct ordering of time related options
|
||||
|
||||
`restic find` now immediately fails with an error if both `--oldest` and `--newest` are specified
|
||||
and `--oldest` is a timestamp after `--newest`.
|
||||
|
||||
https://github.com/restic/restic/issues/5280
|
||||
https://github.com/restic/restic/pull/5310
|
||||
6
changelog/unreleased/pull-5664
Normal file
6
changelog/unreleased/pull-5664
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Bugfix: restic find --pack <tree-pack> did not produce output for tree packs
|
||||
|
||||
`restic find --pack` now produces output for a tree related packfile.
|
||||
|
||||
https://github.com/restic/restic/issues/5280
|
||||
https://github.com/restic/restic/pull/5664
|
||||
9
changelog/unreleased/pull-5718
Normal file
9
changelog/unreleased/pull-5718
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Enhancement: stricter early mountpoint validation in `mount`
|
||||
|
||||
`restic mount` accepted parameters that would lead to a FUSE mount operation
|
||||
failing after having done computationally intensive work to prepare the mount.
|
||||
The `mountpoint` argument supplied must now refer to the name of a directory
|
||||
that the current user can access and write to, otherwise `restic mount` will
|
||||
exit with an error before interacting with the repository.
|
||||
|
||||
https://github.com/restic/restic/pull/5718
|
||||
|
|
@ -450,6 +450,9 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
|||
if f.blobIDs == nil {
|
||||
f.blobIDs = make(map[string]struct{})
|
||||
}
|
||||
if f.treeIDs == nil {
|
||||
f.treeIDs = make(map[string]struct{})
|
||||
}
|
||||
|
||||
debug.Log("Looking for packs...")
|
||||
err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
|
|
@ -470,7 +473,14 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
|||
return err
|
||||
}
|
||||
for _, b := range blobs {
|
||||
f.blobIDs[b.ID.String()] = struct{}{}
|
||||
switch b.Type {
|
||||
case restic.DataBlob:
|
||||
f.blobIDs[b.ID.String()] = struct{}{}
|
||||
case restic.TreeBlob:
|
||||
f.treeIDs[b.ID.String()] = struct{}{}
|
||||
default:
|
||||
panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String()))
|
||||
}
|
||||
}
|
||||
// Stop searching when all packs have been found
|
||||
if len(packIDs) == 0 {
|
||||
|
|
@ -557,7 +567,7 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) {
|
|||
|
||||
blobs := f.repo.LookupBlob(t, rid)
|
||||
if len(blobs) == 0 {
|
||||
f.printer.S("Object %s not found in the index", rid.Str())
|
||||
f.printer.S("Object %s with type %s not found in the index", t.String(), rid.Str())
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -608,6 +618,10 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
|
|||
}
|
||||
}
|
||||
|
||||
if !pat.newest.IsZero() && !pat.oldest.IsZero() && pat.oldest.After(pat.newest) {
|
||||
return errors.Fatal("--oldest must specify a time before --newest")
|
||||
}
|
||||
|
||||
// Check at most only one kind of IDs is provided: currently we
|
||||
// can't mix types
|
||||
if (opts.BlobID && opts.TreeID) ||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,16 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/global"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
)
|
||||
|
||||
func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts global.Options, pattern string) []byte {
|
||||
|
|
@ -132,3 +136,141 @@ func TestFindSorting(t *testing.T) {
|
|||
rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1")
|
||||
rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2")
|
||||
}
|
||||
|
||||
func TestFindInvalidTimeRange(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
err := runFind(context.TODO(), FindOptions{Oldest: "2026-01-01", Newest: "2020-01-01"}, env.gopts, []string{"quack"}, env.gopts.Term)
|
||||
rtest.Assert(t, err != nil && err.Error() == "Fatal: --oldest must specify a time before --newest",
|
||||
"unexpected error message: %v", err)
|
||||
}
|
||||
|
||||
// JsonOutput is the struct `restic find --json` produces
|
||||
type JSONOutput struct {
|
||||
ObjectType string `json:"object_type"`
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"`
|
||||
ParentTree string `json:"parent_tree,omitempty"`
|
||||
SnapshotID string `json:"snapshot"`
|
||||
Time time.Time `json:"time,omitempty"`
|
||||
}
|
||||
|
||||
func TestFindPackfile(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
testSetupBackupData(t, env)
|
||||
|
||||
// backup
|
||||
backupPath := env.testdata + "/0/0/9"
|
||||
testRunBackup(t, "", []string{backupPath}, BackupOptions{}, env.gopts)
|
||||
sn1 := testListSnapshots(t, env.gopts, 1)[0]
|
||||
|
||||
// do all the testing wrapped inside withTermStatus()
|
||||
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
|
||||
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
|
||||
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
|
||||
rtest.OK(t, err)
|
||||
defer unlock()
|
||||
|
||||
// load master index
|
||||
rtest.OK(t, repo.LoadIndex(ctx, printer))
|
||||
|
||||
packID := restic.ID{}
|
||||
done := false
|
||||
err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
|
||||
if !done && pb.Type == restic.TreeBlob {
|
||||
packID = pb.PackID
|
||||
done = true
|
||||
}
|
||||
})
|
||||
|
||||
rtest.OK(t, err)
|
||||
rtest.Assert(t, !packID.IsNull(), "expected a tree packfile ID")
|
||||
findOptions := FindOptions{PackID: true}
|
||||
results := testRunFind(t, true, findOptions, env.gopts, packID.String())
|
||||
|
||||
// get the json records
|
||||
jsonResult := []JSONOutput{}
|
||||
rtest.OK(t, json.Unmarshal(results, &jsonResult))
|
||||
rtest.Assert(t, len(jsonResult) > 0, "expected at least one tree record in the packfile")
|
||||
|
||||
// look at the last record
|
||||
lastIndex := len(jsonResult) - 1
|
||||
record := jsonResult[lastIndex]
|
||||
rtest.Assert(t, record.ObjectType == "tree" && record.SnapshotID == sn1.String(),
|
||||
"expected a tree record with known snapshot id, but got type=%s and snapID=%s instead of %s",
|
||||
record.ObjectType, record.SnapshotID, sn1.String())
|
||||
backupPath = filepath.ToSlash(backupPath)[2:] // take the offending drive mapping away
|
||||
rtest.Assert(t, strings.Contains(record.Path, backupPath), "expected %q as part of %q", backupPath, record.Path)
|
||||
|
||||
return nil
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
|
||||
func TestFindPackID(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
testSetupBackupData(t, env)
|
||||
|
||||
dir009 := filepath.Join(env.testdata, "0", "0", "9")
|
||||
dirEntries, err := os.ReadDir(dir009)
|
||||
rtest.OK(t, err)
|
||||
numberOfFiles := len(dirEntries)
|
||||
|
||||
// backup
|
||||
testRunBackup(t, "", []string{dir009}, BackupOptions{}, env.gopts)
|
||||
sn1 := testListSnapshots(t, env.gopts, 1)[0]
|
||||
|
||||
// extract packfile ID from repository index
|
||||
dataPackID := restic.ID{}
|
||||
treePackID := restic.ID{}
|
||||
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
|
||||
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
|
||||
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
|
||||
rtest.OK(t, err)
|
||||
defer unlock()
|
||||
|
||||
// load Index
|
||||
rtest.OK(t, repo.LoadIndex(ctx, nil))
|
||||
// go through all index entries and collect data and tree packfile(s)
|
||||
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
|
||||
switch blob.Type {
|
||||
case restic.DataBlob:
|
||||
dataPackID = blob.PackID
|
||||
case restic.TreeBlob:
|
||||
treePackID = blob.PackID
|
||||
}
|
||||
}))
|
||||
return nil
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
|
||||
// look for data packfile
|
||||
rtest.Assert(t, !dataPackID.IsNull(), "expected to find data packfile in repo")
|
||||
packID := dataPackID.String()
|
||||
out := testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
|
||||
|
||||
findRes := []JSONOutput{}
|
||||
rtest.OK(t, json.Unmarshal(out, &findRes))
|
||||
rtest.Assert(t, len(findRes) == numberOfFiles, "expected %d entries for this packfile, got %d",
|
||||
numberOfFiles, len(findRes))
|
||||
|
||||
// look for tree packfile
|
||||
rtest.Assert(t, !treePackID.IsNull(), "expected to find tree packfile in repo")
|
||||
packID = treePackID.String()
|
||||
out = testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
|
||||
|
||||
findRes = []JSONOutput{}
|
||||
rtest.OK(t, json.Unmarshal(out, &findRes))
|
||||
record := findRes[len(findRes)-1]
|
||||
|
||||
rtest.Equals(t, record.ObjectType, "tree")
|
||||
rtest.Equals(t, record.SnapshotID, sn1.String())
|
||||
// windows path are messy, so we get rid of the messy bits at the start
|
||||
// exp: "/C/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
|
||||
// got: "C:/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
|
||||
rtest.Equals(t, filepath.ToSlash(record.Path)[2:], filepath.ToSlash(dir009)[2:])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
|
@ -35,8 +36,8 @@ func newMountCommand(globalOptions *global.Options) *cobra.Command {
|
|||
Use: "mount [flags] mountpoint",
|
||||
Short: "Mount the repository",
|
||||
Long: `
|
||||
The "mount" command mounts the repository via fuse to a directory. This is a
|
||||
read-only mount.
|
||||
The "mount" command mounts the repository via fuse over a writeable directory.
|
||||
The repository will be mounted read-only.
|
||||
|
||||
Snapshot Directories
|
||||
====================
|
||||
|
|
@ -133,9 +134,19 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
|
|||
|
||||
// Check the existence of the mount point at the earliest stage to
|
||||
// prevent unnecessary computations while opening the repository.
|
||||
if _, err := os.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
|
||||
stat, err := os.Stat(mountpoint)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
printer.P("Mountpoint %s doesn't exist", mountpoint)
|
||||
return err
|
||||
return errors.Fatal("invalid mountpoint")
|
||||
} else if !stat.IsDir() {
|
||||
printer.P("Mountpoint %s is not a directory", mountpoint)
|
||||
return errors.Fatal("invalid mountpoint")
|
||||
}
|
||||
|
||||
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
|
||||
if err != nil {
|
||||
printer.P("Mountpoint %s is not writeable or not excutable", mountpoint)
|
||||
return errors.Fatal("inaccessible mountpoint")
|
||||
}
|
||||
|
||||
debug.Log("start mount")
|
||||
|
|
|
|||
|
|
@ -366,7 +366,7 @@ const (
|
|||
|
||||
func statsDebug(ctx context.Context, repo restic.Repository, printer progress.Printer) error {
|
||||
printer.E("Collecting size statistics\n\n")
|
||||
for _, t := range []restic.FileType{restic.KeyFile, restic.LockFile, restic.IndexFile, restic.PackFile} {
|
||||
for _, t := range []restic.FileType{restic.KeyFile, restic.LockFile, restic.IndexFile, restic.SnapshotFile, restic.PackFile} {
|
||||
hist, err := statsDebugFileType(ctx, repo, t)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ func main() {
|
|||
exitMessage = fmt.Sprintf("%+v", err)
|
||||
|
||||
if logBuffer.Len() > 0 {
|
||||
exitMessage += "also, the following messages were logged by a library:\n"
|
||||
exitMessage += " also, the following messages were logged by a library:\n"
|
||||
sc := bufio.NewScanner(logBuffer)
|
||||
for sc.Scan() {
|
||||
exitMessage += fmt.Sprintln(sc.Text())
|
||||
|
|
|
|||
|
|
@ -160,36 +160,16 @@ On openSUSE (leap 15.0 and greater, and tumbleweed), you can install restic usin
|
|||
|
||||
# zypper install restic
|
||||
|
||||
RHEL & CentOS
|
||||
=============
|
||||
RHEL & CentOS Stream
|
||||
====================
|
||||
|
||||
For RHEL / CentOS Stream 8 & 9 restic can be installed from the EPEL repository:
|
||||
For supported RHEL / CentOS Stream releases, restic can be installed from the EPEL repository:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ dnf install epel-release
|
||||
$ dnf install restic
|
||||
|
||||
For RHEL7/CentOS there is a copr repository available, you can try the following:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ yum install yum-plugin-copr
|
||||
$ yum copr enable copart/restic
|
||||
$ yum install restic
|
||||
|
||||
If that doesn't work, you can try adding the repository directly, for CentOS6 use:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-6/copart-restic-epel-6.repo
|
||||
|
||||
For CentOS7 use:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-7/copart-restic-epel-7.repo
|
||||
|
||||
Solus
|
||||
=====
|
||||
|
||||
|
|
|
|||
|
|
@ -48,9 +48,23 @@ some of the data was duplicate and restic was able to efficiently reduce it.
|
|||
The data compression also managed to compress the data down to 1.103 GiB.
|
||||
|
||||
If you don't pass the ``--verbose`` option, restic will print less data. You'll
|
||||
still get a nice live status display. Be aware that the live status shows the
|
||||
processed files and not the transferred data. Transferred volume might be lower
|
||||
(due to de-duplication) or higher.
|
||||
still get a nice live status display that looks like this:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
[0:34] 2.96% 867 files 5.046 GiB, total 307867 files 170.438 GiB, 0 errors ETA 18:35
|
||||
|
||||
The progress bar shows, from left to right: elapsed time, progress in percent,
|
||||
the number of files already processed and their size on the local filesystem,
|
||||
followed by the total expected file count and size for the whole backup (these
|
||||
totals come from the initial scan used for progress estimation, which can be
|
||||
disabled with ``--no-scan``). Next is a counter for the number of files that
|
||||
caused errors (these are logged above the progress bar), and finally an
|
||||
estimated time of completion. The file paths displayed below the progress bar
|
||||
are the files currently being read by restic.
|
||||
|
||||
Be aware that the live status shows the processed files and not the transferred
|
||||
data. Transferred volume might be lower (due to de-duplication) or higher.
|
||||
|
||||
On Windows, the ``--use-fs-snapshot`` option will use Windows' Volume Shadow Copy
|
||||
Service (VSS) when creating backups. Restic will transparently create a VSS
|
||||
|
|
|
|||
|
|
@ -542,6 +542,59 @@ a file size value the following command may be used:
|
|||
$ restic -r /srv/restic-repo check --read-data-subset=50M
|
||||
$ restic -r /srv/restic-repo check --read-data-subset=10G
|
||||
|
||||
Finding things in the repository
|
||||
================================
|
||||
|
||||
The ``restic find`` command searches for files or directories stored
|
||||
in the repository.
|
||||
|
||||
find files and directories
|
||||
--------------------------
|
||||
|
||||
If you want to find files or directories in the repository, you either specific filename(s)
|
||||
or a pattern which represents filename(s).
|
||||
The use of file patterns is described in :ref:`backup-excluding-files`.
|
||||
Here is an example:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo find "0/**/7"
|
||||
Found matching entries in snapshot 774ebacd from 2026-01-16 09:01:17
|
||||
/srv/restic-repo/restic/testdata/0/0/9/7
|
||||
|
||||
Another interesting feature of the ``find`` command is the ability to search for
|
||||
files and directories which have an ``inode`` modification time in a given
|
||||
time interval, by using the options ``--oldest`` and ``--newest``.
|
||||
You don't have to give both the options, a half open interval is perfectly acceptable.
|
||||
The following example searches for files which have a modification date in the year 2025.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo find --oldest 2025-01-01 --newest "2025-12-31 23:59:59" "*.txt"
|
||||
Found matching entries in snapshot dd90f84d from 2026-01-17 17:26:41
|
||||
/srv/restic-repo/restic/testdata/0/for_cmd_ls/file1.txt
|
||||
/srv/restic-repo/restic/testdata/0/for_cmd_ls/file2.txt
|
||||
|
||||
All these commands work in ``--json`` mode as well, for output details for the
|
||||
various options please refer to :ref:`find`.
|
||||
|
||||
find blobs, trees or packfiles
|
||||
------------------------------
|
||||
|
||||
The other options of the ``find`` command are devoted to finding blobs, trees and packfiles.
|
||||
These are typically not used by the normal user, but can help debugging a problem
|
||||
with restic. See :ref:`troubleshooting` for a more automated way to repair repositories.
|
||||
|
||||
Just one quick example: if you are looking for specific data blob(s), you can issue the command:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r /srv/restic-repo find --blob fcd9ec0c
|
||||
Found blob fcd9ec0c99f7992c184666e3040831b919f3375157bd563a2b65cde1c6789847
|
||||
... in file /srv/restic-repo/restic/testdata/0/0/9/60
|
||||
(tree 6409bed28d08898b849ecc4fdf338cdb0d67358619c99e6f6c3b402b1895baf8)
|
||||
... in snapshot 774ebacd (2026-01-16 09:01:17)
|
||||
|
||||
|
||||
Upgrading the repository format version
|
||||
=======================================
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ most cases. For high-latency backends it can be beneficial to increase the numbe
|
|||
connections. Please be aware that this increases the resource consumption of restic and
|
||||
that a too high connection count *will degrade performance*. This can also result in longer
|
||||
upload times for single temporary packs, which can lead to more disk wear on SSDs (see
|
||||
:ref:`Pack Size`).
|
||||
:ref:`pack_size`).
|
||||
|
||||
|
||||
CPU Usage
|
||||
|
|
@ -87,7 +87,7 @@ by reading more files in parallel. You can specify the concurrency of file reads
|
|||
the ``backup`` command.
|
||||
|
||||
|
||||
.. Pack Size:
|
||||
.. _pack_size:
|
||||
|
||||
Pack Size
|
||||
=========
|
||||
|
|
|
|||
|
|
@ -161,6 +161,8 @@ a more specific description.
|
|||
| 130 | Restic was interrupted using SIGINT or SIGSTOP |
|
||||
+-----+----------------------------------------------------+
|
||||
|
||||
.. _JSON output:
|
||||
|
||||
JSON output
|
||||
***********
|
||||
|
||||
|
|
@ -439,6 +441,7 @@ DiffStat object
|
|||
| ``bytes`` | Number of bytes | uint64 |
|
||||
+----------------+-------------------------------------------+--------+
|
||||
|
||||
.. _find:
|
||||
|
||||
find
|
||||
----
|
||||
|
|
@ -446,9 +449,8 @@ find
|
|||
The ``find`` command outputs a single JSON document containing an array of JSON
|
||||
objects with matches for your search term. These matches are organized by snapshot.
|
||||
|
||||
If the ``--blob`` or ``--tree`` option is passed, then the output is an array of
|
||||
`Blob objects`_.
|
||||
|
||||
If the ``--blob``, ``--tree`` or ``--pack`` option is passed, then the output is
|
||||
an array of `Blob objects`_.
|
||||
|
||||
+--------------+-----------------------------------+--------------------+
|
||||
| ``hits`` | Number of matches in the snapshot | uint64 |
|
||||
|
|
|
|||
|
|
@ -313,14 +313,56 @@ switches to the ``restic`` user:
|
|||
|
||||
# setpriv --no-new-privs --reuid=$(id -u restic) --regid=$(id -g restic) --init-groups --reset-env --inh-caps +DAC_READ_SEARCH --ambient-caps +DAC_READ_SEARCH restic backup --exclude={/dev,/media,/mnt,/proc,/run,/sys,/tmp,/var/tmp} /
|
||||
|
||||
Note that when using a systemd unit to run restic, you can use
|
||||
``AmbientCapabilities=CAP_DAC_READ_SEARCH`` option to grant the capability to restic.
|
||||
Using ambient capabilities with systemd
|
||||
---------------------------------------
|
||||
|
||||
If you are running restic as a systemd service, you can use systemd's
|
||||
ambient capability feature to assign the necessary capability without
|
||||
modifying the restic binary itself. This is the preferred method,
|
||||
as it still works after binary updates.
|
||||
|
||||
Add the following directives to the ``[Service]`` section of your
|
||||
systemd unit file (e.g., ``/etc/systemd/system/restic.service``):
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[Service]
|
||||
# ... other directives
|
||||
DynamicUser=yes
|
||||
AmbientCapabilities=CAP_DAC_READ_SEARCH
|
||||
CapabilityBoundingSet=CAP_DAC_READ_SEARCH
|
||||
|
||||
Note the use of ``DynamicUser=yes``. This is an added bonus of using the systemd method
|
||||
as you do not need to create a ``restic`` user.
|
||||
|
||||
After editing the unit file, do not forget to reload systemd's configuration and restart
|
||||
the service:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
# systemctl daemon-reload
|
||||
|
||||
Using file capabilities
|
||||
=======================
|
||||
|
||||
Alternatively, the capability can be granted to a file. First we
|
||||
create a new user called ``restic`` that is going to create
|
||||
.. warning::
|
||||
|
||||
Granting ``CAP_DAC_READ_SEARCH`` to the restic binary allows any process
|
||||
executing that binary to bypass standard file permission checks for reading
|
||||
and directory traversal. In practice, anyone who can execute this binary can
|
||||
read most of the system, regardless of their user ID.
|
||||
|
||||
Ensure that only a dedicated backup user (and root) can execute the
|
||||
capability-enabled restic binary, and treat that account as highly privileged.
|
||||
|
||||
See: `capabilities(7) <https://man7.org/linux/man-pages/man7/capabilities.7.html>`_
|
||||
|
||||
Alternatively, the capability can be granted to a file. On every
|
||||
execution, the system will read the assigned capabilities and assign
|
||||
them to the process. This is less secure than using ambient capabilities
|
||||
as anyone who is able to execute the binary can make use of the capability.
|
||||
|
||||
First we create a new user called ``restic`` that is going to create
|
||||
the backups:
|
||||
|
||||
.. code-block:: console
|
||||
|
|
@ -352,12 +394,10 @@ attribute, interpret it and assign capabilities accordingly.
|
|||
|
||||
# setcap cap_dac_read_search=+ep /home/restic/bin/restic
|
||||
|
||||
.. important:: The capabilities of the ``setcap`` command only applies to this
|
||||
.. important:: The capabilities of the ``setcap`` command only apply to this
|
||||
specific copy of the restic binary. If you run ``restic self-update`` or
|
||||
in any other way replace or update the binary, the capabilities you added
|
||||
above will not be in effect for the new binary. To mitigate this, simply
|
||||
run the ``setcap`` command again, to make sure that the new binary has the
|
||||
same and intended capabilities.
|
||||
will be lost, and you must run the ``setcap`` command again.
|
||||
|
||||
From now on the user ``restic`` can run restic to backup the whole
|
||||
system.
|
||||
|
|
|
|||
|
|
@ -105,6 +105,129 @@ project <https://dave.cheney.net/2016/03/12/suggestions-for-contributing-to-an-o
|
|||
A few issues have been tagged with the label ``help wanted``, you can
|
||||
start looking at `those <https://github.com/restic/restic/labels/help%3A%20wanted>`_.
|
||||
|
||||
|
||||
*************
|
||||
Writing tests
|
||||
*************
|
||||
|
||||
In case you want or need to create tests for an enhancement or a new feature of restic,
|
||||
here is a brief description of how to write tests.
|
||||
|
||||
Tests are typically falling into two categories: functional tests (unit tests) and integration tests.
|
||||
Functional tests will verify the correct workings of a function or a set of functions.
|
||||
See more on integration tests below.
|
||||
|
||||
The restic test package located in ``internal/test``, here named ``rtest``,
|
||||
provides the following very basic test functions:
|
||||
::
|
||||
|
||||
rtest.Equals(t, a, b, msg) compares two values, fails test if a != b, optional ``msg`` string
|
||||
for detailed message
|
||||
rtest.Assert(t, a == b, "msg", more variables a, b) checks for a condition to be true,
|
||||
<msg> is a format string to represent the values of <a and b>
|
||||
rtest.OK(t, err) expects err to be ``nil``, otherwise fails test
|
||||
rtest.OKs(t, errs) expects a slice of errs to be ``nil``
|
||||
|
||||
|
||||
Functional tests
|
||||
================
|
||||
|
||||
The packages in ``internal/...`` often provide ``Test*(...)`` functions and structs or
|
||||
have a dedicated test package like ``internal/backend/test``.
|
||||
A good starting points are also the `testing.go` files that exist in several places.
|
||||
Functional tests are stored in the same directory as their function, e.g.
|
||||
``internal/<sub-component>/<function>_test.go``.
|
||||
|
||||
Tests in ``internal`` that need a full repository should just create one using the
|
||||
memory backend by calling ``repository.TestRepository(t)``.
|
||||
|
||||
``checker.TestCheckRepo()`` can be used to verify the repository integrity.
|
||||
|
||||
In general, in-memory operation should be preferred over creating temporary files.
|
||||
If necessary, temporary files can be stored in ``t.TempDir()``. However, in most cases test code
|
||||
only requires a few of the methods provided by a ``Repository``.
|
||||
Then some helper like ``data.TestTreeMap`` can be used or just create a basic mock yourself.
|
||||
|
||||
For backends the test suite in ``internal/backend/test`` is mandatory.
|
||||
|
||||
To temporarily enable feature flags in tests, use
|
||||
``defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()``.
|
||||
Such tests must not run in parallel as this changes global state.
|
||||
|
||||
|
||||
Integration tests
|
||||
=================
|
||||
|
||||
The classical helpers for integration tests are, amongst others:
|
||||
|
||||
- ``env, cleanup := withTestEnvironment(t)``: build an environment for tests
|
||||
- ``datafile := testSetupBackupData(t, env)``: initialize a repo, unpack standard backup tree structure
|
||||
- ``testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)``: backup all of the standard tree structure
|
||||
- ``testListSnapshots(t, env.gopts, <n>)``: check that there are <n> snapshots in the repository
|
||||
- ``testRunCheck(t, env.gopts)``: check that the repository is sound and happy
|
||||
- the above mentioned ``rtest.OK()``, ``rtest.Equals()``, ``rtest.Assert()`` helpers
|
||||
- ``withCaptureStdout()`` and ``withTermStatus()`` wrappers: both functions are found in ``cmd/restic/integration_helpers_test.go`` for creating an enviroment where one can analyze the output created by the ``testRunXXX()`` command, particularly when checking JSON output
|
||||
|
||||
Integration tests test the overall workings of a command. Integration tests are used for commands and
|
||||
are stored in the same directory ``cmd/restic``. The recommended naming convention is
|
||||
``cmd_<command>_integration_test.go``.
|
||||
See the ``cmd/restic/*_integration_test.go`` files for further details.
|
||||
A lot of the base helpers are found in ``cmd/restic/integration_helpers_test.go``.
|
||||
|
||||
This is a typical setting for an integration test:
|
||||
|
||||
- run a ``backup``, compare number of files backup with the expected number of files
|
||||
- run a ``backup``, run the ``ls`` command with a ``sort`` option and compare actual output with the expected output.
|
||||
|
||||
For all backup related functions there is a directory tree which can be used for a
|
||||
default backup, to be found at ``cmd/restic/testdata/backup-data.tar.gz``.
|
||||
In this compressed archive you will find files, hardlinked files,
|
||||
symlinked files, an empty directory and a simple directory structure which is good for testing purposes.
|
||||
|
||||
Commands that require a ``progress.Printer`` should either be wrapped in ``withTermStatus`` or ``withCaptureStdout``.
|
||||
If you want to analyze JSON output, you use ``withCaptureStdout()``.
|
||||
It returns the generated output in a ``*bytes.Buffer``.
|
||||
JSON output can be unmarshalled to produce the approriate go structures; see
|
||||
``cmd/restic/cmd_find_integration_test.go`` as an example.
|
||||
|
||||
Example: this is a typical setup for a backup / find scenario
|
||||
::
|
||||
|
||||
import (
|
||||
... // all your other imports here
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
// setup test
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
// init repository and expand compressed archive into a tree structure
|
||||
testSetupBackupData(t, env)
|
||||
|
||||
// run one backup
|
||||
opts := BackupOptions{}
|
||||
testRunBackup(t, env.testdata+"/0", []string{"."}, opts, env.gopts)
|
||||
|
||||
// make sure we have exactly one snapshot
|
||||
testListSnapshots(t, env.gopts, 1)
|
||||
|
||||
// run command ``restic XXX``
|
||||
// notice that we use the existing wrapper 'testRunXXXX', here 'testRunFind'.
|
||||
// whenever possible use the existing wrapper or modify an existing wrapper
|
||||
// to suit your extra needs.
|
||||
// any remotely complex assertion should be extracted into a reusable helper function.
|
||||
// ``testRunFind()`` uses ``withCaptureStdout()`` to capture output text (in ``results``)
|
||||
results := testRunFind(t, false, FindOptions{}, env.gopts, "testfile")
|
||||
|
||||
// there is always a ``\n`` at the end of the output!
|
||||
lines := strings.Split(string(results), "\n")
|
||||
|
||||
// make sure that we have correct output
|
||||
rtest.Assert(t, len(lines) == 2, "expected one file, found (%v) in repo", len(lines)-1)
|
||||
|
||||
|
||||
********
|
||||
Security
|
||||
********
|
||||
|
|
|
|||
|
|
@ -1280,7 +1280,7 @@ func (b *packBlobIterator) Next() (packBlobValue, error) {
|
|||
nonce, ciphertext := buf[:b.key.NonceSize()], buf[b.key.NonceSize():]
|
||||
plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("decrypting blob %v from %v failed: %w", h, b.packID.Str(), err)
|
||||
err = fmt.Errorf("decrypting blob %v from pack %v failed: %w", h, b.packID.String(), err)
|
||||
}
|
||||
if err == nil && entry.IsCompressed() {
|
||||
// DecodeAll will allocate a slice if it is not large enough since it
|
||||
|
|
@ -1288,16 +1288,16 @@ func (b *packBlobIterator) Next() (packBlobValue, error) {
|
|||
b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0])
|
||||
plaintext = b.decode
|
||||
if err != nil {
|
||||
err = fmt.Errorf("decompressing blob %v from %v failed: %w", h, b.packID.Str(), err)
|
||||
err = fmt.Errorf("decompressing blob %v from pack %v failed: %w", h, b.packID.String(), err)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
id := restic.Hash(plaintext)
|
||||
if !id.Equal(entry.ID) {
|
||||
debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v",
|
||||
h.Type, h.ID, b.packID.Str(), id)
|
||||
err = fmt.Errorf("read blob %v from %v: wrong data returned, hash is %v",
|
||||
h, b.packID.Str(), id)
|
||||
debug.Log("read blob %v/%v from pack %v: wrong data returned, hash is %v",
|
||||
h.Type, h.ID, b.packID.String(), id)
|
||||
err = fmt.Errorf("read blob %v from pack %v: wrong data returned, hash is %v",
|
||||
h, b.packID.String(), id)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -104,8 +104,8 @@ func (b *TextProgress) CompleteItem(messageType, item string, s archiver.ItemSta
|
|||
item, d.Seconds(), ui.FormatBytes(s.DataSize),
|
||||
ui.FormatBytes(s.DataSizeInRepo), ui.FormatBytes(s.TreeSizeInRepo))
|
||||
case "file new":
|
||||
b.VV("new %v, saved in %.3fs (%v added)", item,
|
||||
d.Seconds(), ui.FormatBytes(s.DataSize))
|
||||
b.VV("new %v, saved in %.3fs (%v added, %v stored)", item,
|
||||
d.Seconds(), ui.FormatBytes(s.DataSize), ui.FormatBytes(s.DataSizeInRepo))
|
||||
case "file unchanged":
|
||||
b.VV("unchanged %v", item)
|
||||
case "file modified":
|
||||
|
|
|
|||
Loading…
Reference in a new issue