diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index 9fb2e2463..80e1a5892 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -107,7 +107,6 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error bo := backoff.NewExponentialBackOff() bo.MaxElapsedTime = be.MaxElapsedTime - if feature.Flag.Enabled(feature.BackendErrorRedesign) { bo.InitialInterval = 1 * time.Second bo.Multiplier = 2 @@ -135,14 +134,25 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error err := retryNotifyErrorWithSuccess( func() error { err := f() + + // Running out of local disk space is not a transient backend + // failure. For example, `restic check` can fail while writing to a + // temporary cache directory. Retrying only repeats the same local + // disk-space error. + if isNoSpaceError(err) { + return backoff.Permanent(err) + } + // don't retry permanent errors as those very likely cannot be fixed by retrying // TODO remove IsNotExist(err) special cases when removing the feature flag if feature.Flag.Enabled(feature.BackendErrorRedesign) && !errors.Is(err, &backoff.PermanentError{}) && be.Backend.IsPermanentError(err) { permanentErrorAttempts-- } + if permanentErrorAttempts <= 0 { return backoff.Permanent(err) } + return err }, backoff.WithContext(b, ctx), diff --git a/internal/backend/retry/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go index e0849249e..020e2a96f 100644 --- a/internal/backend/retry/backend_retry_test.go +++ b/internal/backend/retry/backend_retry_test.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "io" + "os" "strings" "testing" "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" @@ -283,6 +285,31 @@ func TestBackendLoadRetry(t *testing.T) { test.Equals(t, 2, attempt) } +func TestBackendLoadNoSpaceNotRetried(t *testing.T) { + noSpaceErr := &os.PathError{ + Op: "write", + Path: "/tmp/restic-check-cache/data/tmp-123", + Err: errNoSpace, + } + + attempt := 0 + be := mock.NewBackend() + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + attempt++ + return nil, noSpaceErr + } + + TestFastRetries(t) + retryBackend := New(be, time.Second, nil, nil) + + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) error { + return nil + }) + + test.Equals(t, noSpaceErr, err) + test.Equals(t, 1, attempt) +} + func testBackendLoadNotExists(t *testing.T, hasFlakyErrors bool) { // load should not retry if the error matches IsNotExist notFound := errors.New("not found") diff --git a/internal/backend/retry/no_soace.go b/internal/backend/retry/no_soace.go new file mode 100644 index 000000000..8528021b1 --- /dev/null +++ b/internal/backend/retry/no_soace.go @@ -0,0 +1,14 @@ +//go:build !windows + +package retry + +import ( + "errors" + "syscall" +) + +var errNoSpace = syscall.ENOSPC + +func isNoSpaceError(err error) bool { + return errors.Is(err, errNoSpace) +} diff --git a/internal/backend/retry/no_space_windows.go b/internal/backend/retry/no_space_windows.go new file mode 100644 index 000000000..7087ad6e5 --- /dev/null +++ b/internal/backend/retry/no_space_windows.go @@ -0,0 +1,13 @@ +package retry + +import ( + "errors" + "syscall" +) + +var errNoSpace = syscall.ERROR_DISK_FULL + +func isNoSpaceError(err error) bool { + return errors.Is(err, errNoSpace) || + errors.Is(err, syscall.ERROR_HANDLE_DISK_FULL) +}