diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9faeb3a9d..032c74c76 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -36,7 +36,7 @@ Please always follow these steps: - Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits). --> -- [ ] I have added tests for all code changes. +- [ ] I have added tests for all code changes, see [writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests) - [ ] I have added documentation for relevant changes (in the manual). - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)). - [ ] I'm done! This pull request is ready for review. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc278fa3a..d1900fa16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -202,6 +202,9 @@ we'll be glad to assist. Having a PR with failing integration tests is nothing to be ashamed of. In contrast, that happens regularly for all of us. That's what the tests are there for. +More details of how to structure tests can be found here at +[writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests). + Git Commits ----------- diff --git a/changelog/TEMPLATE b/changelog/TEMPLATE index 7d6065e04..f67d9fe00 100644 --- a/changelog/TEMPLATE +++ b/changelog/TEMPLATE @@ -1,14 +1,20 @@ # The first line must start with Bugfix:, Enhancement: or Change:, -# including the colon. Use present tense and the imperative mood. Remove -# lines starting with '#' from this template. -Enhancement: Allow custom bar in the foo command +# including the colon. 'Change:' is for breaking changes only. +# Documentation-only changes do not get a changelog entry. +# Include the affected command in the summary if relevant. +# +# Use present tense and the imperative mood. Remove lines starting +# with '#' from this template. +Enhancement: Allow custom bar in the `foo` command # Describe the problem in the past tense, the new behavior in the present # tense. Mention the affected commands, backends, operating systems, etc. # If the problem description just says that a feature was missing, then -# only explain the new behavior. -# Focus on user-facing behavior, not the implementation. +# only explain the new behavior. Aim for a short and concise description. # Use "Restic now ..." instead of "We have changed ...". +# +# Focus on user-facing behavior, not the implementation. The description should +# be understandable for a regular user without knowledge of the implementation. Restic foo always used the system-wide bar when deciding how to frob an item in the `baz` backend. It now permits selecting the bar with `--bar` diff --git a/changelog/unreleased/issue-5280 b/changelog/unreleased/issue-5280 new file mode 100644 index 000000000..9714e3c43 --- /dev/null +++ b/changelog/unreleased/issue-5280 @@ -0,0 +1,7 @@ +Bugfix: `restic find` now checks for correct ordering of time related options + +`restic find` now immediately fails with an error if both `--oldest` and `--newest` are specified +and `--oldest` is a timestamp after `--newest`. + +https://github.com/restic/restic/issues/5280 +https://github.com/restic/restic/pull/5310 diff --git a/changelog/unreleased/pull-5664 b/changelog/unreleased/pull-5664 new file mode 100644 index 000000000..3e480f0d3 --- /dev/null +++ b/changelog/unreleased/pull-5664 @@ -0,0 +1,6 @@ +Bugfix: restic find --pack did not produce output for tree packs + +`restic find --pack` now produces output for a tree related packfile. + +https://github.com/restic/restic/issues/5280 +https://github.com/restic/restic/pull/5664 diff --git a/changelog/unreleased/pull-5718 b/changelog/unreleased/pull-5718 new file mode 100644 index 000000000..f349bb744 --- /dev/null +++ b/changelog/unreleased/pull-5718 @@ -0,0 +1,9 @@ +Enhancement: stricter early mountpoint validation in `mount` + +`restic mount` accepted parameters that would lead to a FUSE mount operation +failing after having done computationally intensive work to prepare the mount. +The `mountpoint` argument supplied must now refer to the name of a directory +that the current user can access and write to, otherwise `restic mount` will +exit with an error before interacting with the repository. + +https://github.com/restic/restic/pull/5718 diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 0ae7e315d..ae7782f85 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -450,6 +450,9 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { if f.blobIDs == nil { f.blobIDs = make(map[string]struct{}) } + if f.treeIDs == nil { + f.treeIDs = make(map[string]struct{}) + } debug.Log("Looking for packs...") err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { @@ -470,7 +473,14 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { return err } for _, b := range blobs { - f.blobIDs[b.ID.String()] = struct{}{} + switch b.Type { + case restic.DataBlob: + f.blobIDs[b.ID.String()] = struct{}{} + case restic.TreeBlob: + f.treeIDs[b.ID.String()] = struct{}{} + default: + panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String())) + } } // Stop searching when all packs have been found if len(packIDs) == 0 { @@ -557,7 +567,7 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) { blobs := f.repo.LookupBlob(t, rid) if len(blobs) == 0 { - f.printer.S("Object %s not found in the index", rid.Str()) + f.printer.S("Object %s with type %s not found in the index", t.String(), rid.Str()) return } @@ -608,6 +618,10 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [ } } + if !pat.newest.IsZero() && !pat.oldest.IsZero() && pat.oldest.After(pat.newest) { + return errors.Fatal("--oldest must specify a time before --newest") + } + // Check at most only one kind of IDs is provided: currently we // can't mix types if (opts.BlobID && opts.TreeID) || diff --git a/cmd/restic/cmd_find_integration_test.go b/cmd/restic/cmd_find_integration_test.go index 360620952..f46d63cc5 100644 --- a/cmd/restic/cmd_find_integration_test.go +++ b/cmd/restic/cmd_find_integration_test.go @@ -3,12 +3,16 @@ package main import ( "context" "encoding/json" + "os" + "path/filepath" "strings" "testing" "time" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts global.Options, pattern string) []byte { @@ -132,3 +136,141 @@ func TestFindSorting(t *testing.T) { rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1") rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2") } + +func TestFindInvalidTimeRange(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + err := runFind(context.TODO(), FindOptions{Oldest: "2026-01-01", Newest: "2020-01-01"}, env.gopts, []string{"quack"}, env.gopts.Term) + rtest.Assert(t, err != nil && err.Error() == "Fatal: --oldest must specify a time before --newest", + "unexpected error message: %v", err) +} + +// JsonOutput is the struct `restic find --json` produces +type JSONOutput struct { + ObjectType string `json:"object_type"` + ID string `json:"id"` + Path string `json:"path"` + ParentTree string `json:"parent_tree,omitempty"` + SnapshotID string `json:"snapshot"` + Time time.Time `json:"time,omitempty"` +} + +func TestFindPackfile(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + + // backup + backupPath := env.testdata + "/0/0/9" + testRunBackup(t, "", []string{backupPath}, BackupOptions{}, env.gopts) + sn1 := testListSnapshots(t, env.gopts, 1)[0] + + // do all the testing wrapped inside withTermStatus() + err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error { + printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term) + _, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) + rtest.OK(t, err) + defer unlock() + + // load master index + rtest.OK(t, repo.LoadIndex(ctx, printer)) + + packID := restic.ID{} + done := false + err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) { + if !done && pb.Type == restic.TreeBlob { + packID = pb.PackID + done = true + } + }) + + rtest.OK(t, err) + rtest.Assert(t, !packID.IsNull(), "expected a tree packfile ID") + findOptions := FindOptions{PackID: true} + results := testRunFind(t, true, findOptions, env.gopts, packID.String()) + + // get the json records + jsonResult := []JSONOutput{} + rtest.OK(t, json.Unmarshal(results, &jsonResult)) + rtest.Assert(t, len(jsonResult) > 0, "expected at least one tree record in the packfile") + + // look at the last record + lastIndex := len(jsonResult) - 1 + record := jsonResult[lastIndex] + rtest.Assert(t, record.ObjectType == "tree" && record.SnapshotID == sn1.String(), + "expected a tree record with known snapshot id, but got type=%s and snapID=%s instead of %s", + record.ObjectType, record.SnapshotID, sn1.String()) + backupPath = filepath.ToSlash(backupPath)[2:] // take the offending drive mapping away + rtest.Assert(t, strings.Contains(record.Path, backupPath), "expected %q as part of %q", backupPath, record.Path) + + return nil + }) + rtest.OK(t, err) +} + +func TestFindPackID(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testSetupBackupData(t, env) + + dir009 := filepath.Join(env.testdata, "0", "0", "9") + dirEntries, err := os.ReadDir(dir009) + rtest.OK(t, err) + numberOfFiles := len(dirEntries) + + // backup + testRunBackup(t, "", []string{dir009}, BackupOptions{}, env.gopts) + sn1 := testListSnapshots(t, env.gopts, 1)[0] + + // extract packfile ID from repository index + dataPackID := restic.ID{} + treePackID := restic.ID{} + err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error { + printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term) + _, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer) + rtest.OK(t, err) + defer unlock() + + // load Index + rtest.OK(t, repo.LoadIndex(ctx, nil)) + // go through all index entries and collect data and tree packfile(s) + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { + switch blob.Type { + case restic.DataBlob: + dataPackID = blob.PackID + case restic.TreeBlob: + treePackID = blob.PackID + } + })) + return nil + }) + rtest.OK(t, err) + + // look for data packfile + rtest.Assert(t, !dataPackID.IsNull(), "expected to find data packfile in repo") + packID := dataPackID.String() + out := testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID) + + findRes := []JSONOutput{} + rtest.OK(t, json.Unmarshal(out, &findRes)) + rtest.Assert(t, len(findRes) == numberOfFiles, "expected %d entries for this packfile, got %d", + numberOfFiles, len(findRes)) + + // look for tree packfile + rtest.Assert(t, !treePackID.IsNull(), "expected to find tree packfile in repo") + packID = treePackID.String() + out = testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID) + + findRes = []JSONOutput{} + rtest.OK(t, json.Unmarshal(out, &findRes)) + record := findRes[len(findRes)-1] + + rtest.Equals(t, record.ObjectType, "tree") + rtest.Equals(t, record.SnapshotID, sn1.String()) + // windows path are messy, so we get rid of the messy bits at the start + // exp: "/C/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9" + // got: "C:/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9" + rtest.Equals(t, filepath.ToSlash(record.Path)[2:], filepath.ToSlash(dir009)[2:]) +} diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 202109fa6..eb86cbada 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/sys/unix" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/debug" @@ -35,8 +36,8 @@ func newMountCommand(globalOptions *global.Options) *cobra.Command { Use: "mount [flags] mountpoint", Short: "Mount the repository", Long: ` -The "mount" command mounts the repository via fuse to a directory. This is a -read-only mount. +The "mount" command mounts the repository via fuse over a writeable directory. +The repository will be mounted read-only. Snapshot Directories ==================== @@ -133,9 +134,19 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args // Check the existence of the mount point at the earliest stage to // prevent unnecessary computations while opening the repository. - if _, err := os.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { + stat, err := os.Stat(mountpoint) + if errors.Is(err, os.ErrNotExist) { printer.P("Mountpoint %s doesn't exist", mountpoint) - return err + return errors.Fatal("invalid mountpoint") + } else if !stat.IsDir() { + printer.P("Mountpoint %s is not a directory", mountpoint) + return errors.Fatal("invalid mountpoint") + } + + err = unix.Access(mountpoint, unix.W_OK|unix.X_OK) + if err != nil { + printer.P("Mountpoint %s is not writeable or not excutable", mountpoint) + return errors.Fatal("inaccessible mountpoint") } debug.Log("start mount") diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 490806e2a..8578a481c 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -366,7 +366,7 @@ const ( func statsDebug(ctx context.Context, repo restic.Repository, printer progress.Printer) error { printer.E("Collecting size statistics\n\n") - for _, t := range []restic.FileType{restic.KeyFile, restic.LockFile, restic.IndexFile, restic.PackFile} { + for _, t := range []restic.FileType{restic.KeyFile, restic.LockFile, restic.IndexFile, restic.SnapshotFile, restic.PackFile} { hist, err := statsDebugFileType(ctx, repo, t) if err != nil { return err diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 6ed2811f3..619eee642 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -210,7 +210,7 @@ func main() { exitMessage = fmt.Sprintf("%+v", err) if logBuffer.Len() > 0 { - exitMessage += "also, the following messages were logged by a library:\n" + exitMessage += " also, the following messages were logged by a library:\n" sc := bufio.NewScanner(logBuffer) for sc.Scan() { exitMessage += fmt.Sprintln(sc.Text()) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 6d6c1b197..70627702a 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -160,36 +160,16 @@ On openSUSE (leap 15.0 and greater, and tumbleweed), you can install restic usin # zypper install restic -RHEL & CentOS -============= +RHEL & CentOS Stream +==================== -For RHEL / CentOS Stream 8 & 9 restic can be installed from the EPEL repository: +For supported RHEL / CentOS Stream releases, restic can be installed from the EPEL repository: .. code-block:: console $ dnf install epel-release $ dnf install restic -For RHEL7/CentOS there is a copr repository available, you can try the following: - -.. code-block:: console - - $ yum install yum-plugin-copr - $ yum copr enable copart/restic - $ yum install restic - -If that doesn't work, you can try adding the repository directly, for CentOS6 use: - -.. code-block:: console - - $ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-6/copart-restic-epel-6.repo - -For CentOS7 use: - -.. code-block:: console - - $ yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-7/copart-restic-epel-7.repo - Solus ===== diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 9504a223d..614d58ee0 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -48,9 +48,23 @@ some of the data was duplicate and restic was able to efficiently reduce it. The data compression also managed to compress the data down to 1.103 GiB. If you don't pass the ``--verbose`` option, restic will print less data. You'll -still get a nice live status display. Be aware that the live status shows the -processed files and not the transferred data. Transferred volume might be lower -(due to de-duplication) or higher. +still get a nice live status display that looks like this: + +.. code-block:: console + + [0:34] 2.96% 867 files 5.046 GiB, total 307867 files 170.438 GiB, 0 errors ETA 18:35 + +The progress bar shows, from left to right: elapsed time, progress in percent, +the number of files already processed and their size on the local filesystem, +followed by the total expected file count and size for the whole backup (these +totals come from the initial scan used for progress estimation, which can be +disabled with ``--no-scan``). Next is a counter for the number of files that +caused errors (these are logged above the progress bar), and finally an +estimated time of completion. The file paths displayed below the progress bar +are the files currently being read by restic. + +Be aware that the live status shows the processed files and not the transferred +data. Transferred volume might be lower (due to de-duplication) or higher. On Windows, the ``--use-fs-snapshot`` option will use Windows' Volume Shadow Copy Service (VSS) when creating backups. Restic will transparently create a VSS diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index ba263fd31..f03674383 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -542,6 +542,59 @@ a file size value the following command may be used: $ restic -r /srv/restic-repo check --read-data-subset=50M $ restic -r /srv/restic-repo check --read-data-subset=10G +Finding things in the repository +================================ + +The ``restic find`` command searches for files or directories stored +in the repository. + +find files and directories +-------------------------- + +If you want to find files or directories in the repository, you either specific filename(s) +or a pattern which represents filename(s). +The use of file patterns is described in :ref:`backup-excluding-files`. +Here is an example: + +.. code-block:: console + + $ restic -r /srv/restic-repo find "0/**/7" + Found matching entries in snapshot 774ebacd from 2026-01-16 09:01:17 + /srv/restic-repo/restic/testdata/0/0/9/7 + +Another interesting feature of the ``find`` command is the ability to search for +files and directories which have an ``inode`` modification time in a given +time interval, by using the options ``--oldest`` and ``--newest``. +You don't have to give both the options, a half open interval is perfectly acceptable. +The following example searches for files which have a modification date in the year 2025. + +.. code-block:: console + + $ restic -r /srv/restic-repo find --oldest 2025-01-01 --newest "2025-12-31 23:59:59" "*.txt" + Found matching entries in snapshot dd90f84d from 2026-01-17 17:26:41 + /srv/restic-repo/restic/testdata/0/for_cmd_ls/file1.txt + /srv/restic-repo/restic/testdata/0/for_cmd_ls/file2.txt + +All these commands work in ``--json`` mode as well, for output details for the +various options please refer to :ref:`find`. + +find blobs, trees or packfiles +------------------------------ + +The other options of the ``find`` command are devoted to finding blobs, trees and packfiles. +These are typically not used by the normal user, but can help debugging a problem +with restic. See :ref:`troubleshooting` for a more automated way to repair repositories. + +Just one quick example: if you are looking for specific data blob(s), you can issue the command: + +.. code-block:: console + + $ restic -r /srv/restic-repo find --blob fcd9ec0c + Found blob fcd9ec0c99f7992c184666e3040831b919f3375157bd563a2b65cde1c6789847 + ... in file /srv/restic-repo/restic/testdata/0/0/9/60 + (tree 6409bed28d08898b849ecc4fdf338cdb0d67358619c99e6f6c3b402b1895baf8) + ... in snapshot 774ebacd (2026-01-16 09:01:17) + Upgrading the repository format version ======================================= diff --git a/doc/047_tuning_parameters.rst b/doc/047_tuning_parameters.rst index eba958211..1cd6b9329 100644 --- a/doc/047_tuning_parameters.rst +++ b/doc/047_tuning_parameters.rst @@ -41,7 +41,7 @@ most cases. For high-latency backends it can be beneficial to increase the numbe connections. Please be aware that this increases the resource consumption of restic and that a too high connection count *will degrade performance*. This can also result in longer upload times for single temporary packs, which can lead to more disk wear on SSDs (see -:ref:`Pack Size`). +:ref:`pack_size`). CPU Usage @@ -87,7 +87,7 @@ by reading more files in parallel. You can specify the concurrency of file reads the ``backup`` command. -.. Pack Size: +.. _pack_size: Pack Size ========= diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index a24e597dd..f0c225d84 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -161,6 +161,8 @@ a more specific description. | 130 | Restic was interrupted using SIGINT or SIGSTOP | +-----+----------------------------------------------------+ +.. _JSON output: + JSON output *********** @@ -439,6 +441,7 @@ DiffStat object | ``bytes`` | Number of bytes | uint64 | +----------------+-------------------------------------------+--------+ +.. _find: find ---- @@ -446,9 +449,8 @@ find The ``find`` command outputs a single JSON document containing an array of JSON objects with matches for your search term. These matches are organized by snapshot. -If the ``--blob`` or ``--tree`` option is passed, then the output is an array of -`Blob objects`_. - +If the ``--blob``, ``--tree`` or ``--pack`` option is passed, then the output is +an array of `Blob objects`_. +--------------+-----------------------------------+--------------------+ | ``hits`` | Number of matches in the snapshot | uint64 | diff --git a/doc/080_examples.rst b/doc/080_examples.rst index d79fe1adb..f8a61884e 100644 --- a/doc/080_examples.rst +++ b/doc/080_examples.rst @@ -313,14 +313,56 @@ switches to the ``restic`` user: # setpriv --no-new-privs --reuid=$(id -u restic) --regid=$(id -g restic) --init-groups --reset-env --inh-caps +DAC_READ_SEARCH --ambient-caps +DAC_READ_SEARCH restic backup --exclude={/dev,/media,/mnt,/proc,/run,/sys,/tmp,/var/tmp} / -Note that when using a systemd unit to run restic, you can use -``AmbientCapabilities=CAP_DAC_READ_SEARCH`` option to grant the capability to restic. +Using ambient capabilities with systemd +--------------------------------------- + +If you are running restic as a systemd service, you can use systemd's +ambient capability feature to assign the necessary capability without +modifying the restic binary itself. This is the preferred method, +as it still works after binary updates. + +Add the following directives to the ``[Service]`` section of your +systemd unit file (e.g., ``/etc/systemd/system/restic.service``): + +.. code-block:: ini + + [Service] + # ... other directives + DynamicUser=yes + AmbientCapabilities=CAP_DAC_READ_SEARCH + CapabilityBoundingSet=CAP_DAC_READ_SEARCH + +Note the use of ``DynamicUser=yes``. This is an added bonus of using the systemd method +as you do not need to create a ``restic`` user. + +After editing the unit file, do not forget to reload systemd's configuration and restart +the service: + +.. code-block:: console + + # systemctl daemon-reload Using file capabilities ======================= -Alternatively, the capability can be granted to a file. First we -create a new user called ``restic`` that is going to create +.. warning:: + + Granting ``CAP_DAC_READ_SEARCH`` to the restic binary allows any process + executing that binary to bypass standard file permission checks for reading + and directory traversal. In practice, anyone who can execute this binary can + read most of the system, regardless of their user ID. + + Ensure that only a dedicated backup user (and root) can execute the + capability-enabled restic binary, and treat that account as highly privileged. + + See: `capabilities(7) `_ + +Alternatively, the capability can be granted to a file. On every +execution, the system will read the assigned capabilities and assign +them to the process. This is less secure than using ambient capabilities +as anyone who is able to execute the binary can make use of the capability. + +First we create a new user called ``restic`` that is going to create the backups: .. code-block:: console @@ -352,12 +394,10 @@ attribute, interpret it and assign capabilities accordingly. # setcap cap_dac_read_search=+ep /home/restic/bin/restic -.. important:: The capabilities of the ``setcap`` command only applies to this +.. important:: The capabilities of the ``setcap`` command only apply to this specific copy of the restic binary. If you run ``restic self-update`` or in any other way replace or update the binary, the capabilities you added - above will not be in effect for the new binary. To mitigate this, simply - run the ``setcap`` command again, to make sure that the new binary has the - same and intended capabilities. + will be lost, and you must run the ``setcap`` command again. From now on the user ``restic`` can run restic to backup the whole system. diff --git a/doc/090_participating.rst b/doc/090_participating.rst index 2a3964cac..7e84327bc 100644 --- a/doc/090_participating.rst +++ b/doc/090_participating.rst @@ -105,6 +105,129 @@ project `_. + +************* +Writing tests +************* + +In case you want or need to create tests for an enhancement or a new feature of restic, +here is a brief description of how to write tests. + +Tests are typically falling into two categories: functional tests (unit tests) and integration tests. +Functional tests will verify the correct workings of a function or a set of functions. +See more on integration tests below. + +The restic test package located in ``internal/test``, here named ``rtest``, +provides the following very basic test functions: +:: + + rtest.Equals(t, a, b, msg) compares two values, fails test if a != b, optional ``msg`` string + for detailed message + rtest.Assert(t, a == b, "msg", more variables a, b) checks for a condition to be true, + is a format string to represent the values of + rtest.OK(t, err) expects err to be ``nil``, otherwise fails test + rtest.OKs(t, errs) expects a slice of errs to be ``nil`` + + +Functional tests +================ + +The packages in ``internal/...`` often provide ``Test*(...)`` functions and structs or +have a dedicated test package like ``internal/backend/test``. +A good starting points are also the `testing.go` files that exist in several places. +Functional tests are stored in the same directory as their function, e.g. +``internal//_test.go``. + +Tests in ``internal`` that need a full repository should just create one using the +memory backend by calling ``repository.TestRepository(t)``. + +``checker.TestCheckRepo()`` can be used to verify the repository integrity. + +In general, in-memory operation should be preferred over creating temporary files. +If necessary, temporary files can be stored in ``t.TempDir()``. However, in most cases test code +only requires a few of the methods provided by a ``Repository``. +Then some helper like ``data.TestTreeMap`` can be used or just create a basic mock yourself. + +For backends the test suite in ``internal/backend/test`` is mandatory. + +To temporarily enable feature flags in tests, use +``defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()``. +Such tests must not run in parallel as this changes global state. + + +Integration tests +================= + +The classical helpers for integration tests are, amongst others: + +- ``env, cleanup := withTestEnvironment(t)``: build an environment for tests +- ``datafile := testSetupBackupData(t, env)``: initialize a repo, unpack standard backup tree structure +- ``testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)``: backup all of the standard tree structure +- ``testListSnapshots(t, env.gopts, )``: check that there are snapshots in the repository +- ``testRunCheck(t, env.gopts)``: check that the repository is sound and happy +- the above mentioned ``rtest.OK()``, ``rtest.Equals()``, ``rtest.Assert()`` helpers +- ``withCaptureStdout()`` and ``withTermStatus()`` wrappers: both functions are found in ``cmd/restic/integration_helpers_test.go`` for creating an enviroment where one can analyze the output created by the ``testRunXXX()`` command, particularly when checking JSON output + +Integration tests test the overall workings of a command. Integration tests are used for commands and +are stored in the same directory ``cmd/restic``. The recommended naming convention is +``cmd__integration_test.go``. +See the ``cmd/restic/*_integration_test.go`` files for further details. +A lot of the base helpers are found in ``cmd/restic/integration_helpers_test.go``. + +This is a typical setting for an integration test: + +- run a ``backup``, compare number of files backup with the expected number of files +- run a ``backup``, run the ``ls`` command with a ``sort`` option and compare actual output with the expected output. + +For all backup related functions there is a directory tree which can be used for a +default backup, to be found at ``cmd/restic/testdata/backup-data.tar.gz``. +In this compressed archive you will find files, hardlinked files, +symlinked files, an empty directory and a simple directory structure which is good for testing purposes. + +Commands that require a ``progress.Printer`` should either be wrapped in ``withTermStatus`` or ``withCaptureStdout``. +If you want to analyze JSON output, you use ``withCaptureStdout()``. +It returns the generated output in a ``*bytes.Buffer``. +JSON output can be unmarshalled to produce the approriate go structures; see +``cmd/restic/cmd_find_integration_test.go`` as an example. + +Example: this is a typical setup for a backup / find scenario +:: + + import ( + ... // all your other imports here + + rtest "github.com/restic/restic/internal/test" + ) + + // setup test + env, cleanup := withTestEnvironment(t) + defer cleanup() + + // init repository and expand compressed archive into a tree structure + testSetupBackupData(t, env) + + // run one backup + opts := BackupOptions{} + testRunBackup(t, env.testdata+"/0", []string{"."}, opts, env.gopts) + + // make sure we have exactly one snapshot + testListSnapshots(t, env.gopts, 1) + + // run command ``restic XXX`` + // notice that we use the existing wrapper 'testRunXXXX', here 'testRunFind'. + // whenever possible use the existing wrapper or modify an existing wrapper + // to suit your extra needs. + // any remotely complex assertion should be extracted into a reusable helper function. + // ``testRunFind()`` uses ``withCaptureStdout()`` to capture output text (in ``results``) + results := testRunFind(t, false, FindOptions{}, env.gopts, "testfile") + + // there is always a ``\n`` at the end of the output! + lines := strings.Split(string(results), "\n") + + // make sure that we have correct output + rtest.Assert(t, len(lines) == 2, "expected one file, found (%v) in repo", len(lines)-1) + + ******** Security ******** diff --git a/internal/repository/repository.go b/internal/repository/repository.go index e7a1b8c17..f1704291b 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -1280,7 +1280,7 @@ func (b *packBlobIterator) Next() (packBlobValue, error) { nonce, ciphertext := buf[:b.key.NonceSize()], buf[b.key.NonceSize():] plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil) if err != nil { - err = fmt.Errorf("decrypting blob %v from %v failed: %w", h, b.packID.Str(), err) + err = fmt.Errorf("decrypting blob %v from pack %v failed: %w", h, b.packID.String(), err) } if err == nil && entry.IsCompressed() { // DecodeAll will allocate a slice if it is not large enough since it @@ -1288,16 +1288,16 @@ func (b *packBlobIterator) Next() (packBlobValue, error) { b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0]) plaintext = b.decode if err != nil { - err = fmt.Errorf("decompressing blob %v from %v failed: %w", h, b.packID.Str(), err) + err = fmt.Errorf("decompressing blob %v from pack %v failed: %w", h, b.packID.String(), err) } } if err == nil { id := restic.Hash(plaintext) if !id.Equal(entry.ID) { - debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", - h.Type, h.ID, b.packID.Str(), id) - err = fmt.Errorf("read blob %v from %v: wrong data returned, hash is %v", - h, b.packID.Str(), id) + debug.Log("read blob %v/%v from pack %v: wrong data returned, hash is %v", + h.Type, h.ID, b.packID.String(), id) + err = fmt.Errorf("read blob %v from pack %v: wrong data returned, hash is %v", + h, b.packID.String(), id) } } diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 7085893fe..8b416da7d 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -104,8 +104,8 @@ func (b *TextProgress) CompleteItem(messageType, item string, s archiver.ItemSta item, d.Seconds(), ui.FormatBytes(s.DataSize), ui.FormatBytes(s.DataSizeInRepo), ui.FormatBytes(s.TreeSizeInRepo)) case "file new": - b.VV("new %v, saved in %.3fs (%v added)", item, - d.Seconds(), ui.FormatBytes(s.DataSize)) + b.VV("new %v, saved in %.3fs (%v added, %v stored)", item, + d.Seconds(), ui.FormatBytes(s.DataSize), ui.FormatBytes(s.DataSizeInRepo)) case "file unchanged": b.VV("unchanged %v", item) case "file modified":