From 15e90b7a4caaba59c551d7ba624dbc68a919285f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Mar 2025 12:41:07 +0100 Subject: [PATCH 1/8] test go 1.24 and drop support for 1.22 --- .github/workflows/tests.yml | 12 ++++++------ build.go | 8 +------- changelog/unreleased/pull-4938 | 12 +++++++----- doc/020_installation.rst | 2 +- go.mod | 6 +++++- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 647cb5de7..a06dd0a35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - latest_go: "1.23.x" + latest_go: "1.24.x" GO111MODULE: on jobs: @@ -23,29 +23,29 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.23.x + go: 1.24.x os: windows-latest - job_name: macOS - go: 1.23.x + go: 1.24.x os: macOS-latest test_fuse: false - job_name: Linux - go: 1.23.x + go: 1.24.x os: ubuntu-latest test_cloud_backends: true test_fuse: true check_changelog: true - job_name: Linux (race) - go: 1.23.x + go: 1.24.x os: ubuntu-latest test_fuse: true test_opts: "-race" - job_name: Linux - go: 1.22.x + go: 1.23.x os: ubuntu-latest test_fuse: true diff --git a/build.go b/build.go index 32eefb7c0..0f4e80c91 100644 --- a/build.go +++ b/build.go @@ -58,7 +58,7 @@ var config = Config{ Main: "./cmd/restic", // package name for the main package DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used Tests: []string{"./..."}, // tests to run - MinVersion: GoVersion{Major: 1, Minor: 22, Patch: 0}, // minimum Go version supported + MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported } // Config configures the build. @@ -382,12 +382,6 @@ func main() { } } - solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0} - if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) { - fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion) - os.Exit(1) - } - verbosePrintf("detected Go version %v\n", goVersion) preserveSymbols := false diff --git a/changelog/unreleased/pull-4938 b/changelog/unreleased/pull-4938 index d95bca16d..831997a4a 100644 --- a/changelog/unreleased/pull-4938 +++ b/changelog/unreleased/pull-4938 @@ -1,9 +1,11 @@ -Change: Update dependencies and require Go 1.22 or newer +Change: Update dependencies and require Go 1.23 or newer -We have updated all dependencies. Since some libraries require newer Go standard -library features, support for Go 1.19, 1.20 and 1.21 has been dropped, which means -that restic now requires at least Go 1.22 to build. +We have updated all dependencies. Since some libraries require newer Go +standard library features, support for Go 1.19, 1.20, 1.21 and 1.22 has been +dropped, which means that restic now requires at least Go 1.23 to build. -This also disables support for TLS versions older than TLS 1.2. +This also disables support for TLS versions older than TLS 1.2. On Windows, +restic now requires at least Windows 10 or Windows Server 2016. On macOS, +restic now requires at least macOS 11 Big Sur. https://github.com/restic/restic/pull/4938 diff --git a/doc/020_installation.rst b/doc/020_installation.rst index a53888221..a39ca896b 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -284,7 +284,7 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.22. Building restic may also work with older versions of Go, +Go version 1.23. Building restic may also work with older versions of Go, but that's not supported. See the `Getting started `__ guide of the Go project for instructions how to install Go. diff --git a/go.mod b/go.mod index f4823c674..f94924f4d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,10 @@ module github.com/restic/restic -go 1.22 +go 1.23 + +// keep the old behavior for reparse points on windows until handling reparse points has been improved in restic +// https://forum.restic.net/t/windows-junction-backup-with-go1-23-or-later/8940 +godebug winsymlink=0 require ( cloud.google.com/go/storage v1.43.0 From c36970074d9610f8f2ab7ecf2956d87ee8f84f0e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Mar 2025 12:41:20 +0100 Subject: [PATCH 2/8] CI: bump golangci version --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a06dd0a35..ea6f0175d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -254,7 +254,7 @@ jobs: uses: golangci/golangci-lint-action@v6 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.63.4 + version: v1.64.8 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get From 4640b3c41a6f8ec9239ef71be00b9384f0fb7f26 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 15:18:21 +0100 Subject: [PATCH 3/8] backend/test: parallelize slow tests --- internal/backend/test/tests.go | 234 ++++++++++++++++++--------------- 1 file changed, 125 insertions(+), 109 deletions(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index f9dc0a17e..97bca5b97 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -10,11 +10,13 @@ import ( "os" "reflect" "sort" + "sync" "testing" "time" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" "github.com/restic/restic/internal/test" @@ -276,16 +278,26 @@ func (s *Suite[C]) TestList(t *testing.T) { } list1 := make(map[restic.ID]int64) + var m sync.Mutex + wg, ctx := errgroup.WithContext(context.TODO()) for i := 0; i < numTestFiles; i++ { data := test.Random(random.Int(), random.Intn(100)+55) - id := restic.Hash(data) - h := backend.Handle{Type: backend.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) - if err != nil { - t.Fatal(err) - } - list1[id] = int64(len(data)) + wg.Go(func() error { + id := restic.Hash(data) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(ctx, h, backend.NewByteReader(data, b.Hasher())) + + m.Lock() + defer m.Unlock() + list1[id] = int64(len(data)) + return err + }) + } + + err = wg.Wait() + if err != nil { + t.Fatal(err) } t.Logf("wrote %v files", len(list1)) @@ -777,125 +789,129 @@ func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, // TestBackend tests all functions of the backend. func (s *Suite[C]) TestBackend(t *testing.T) { - b := s.open(t) - defer s.close(t, b) - - test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") - test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error") - for _, tpe := range []backend.FileType{ backend.PackFile, backend.KeyFile, backend.LockFile, backend.SnapshotFile, backend.IndexFile, } { - // detect non-existing files - for _, ts := range testStrings { - id, err := restic.ParseID(ts.id) - test.OK(t, err) + t.Run(tpe.String(), func(t *testing.T) { + t.Parallel() - // test if blob is already in repository - h := backend.Handle{Type: tpe, Name: id.String()} - ret, err := beTest(context.TODO(), b, h) - test.OK(t, err) - test.Assert(t, !ret, "blob was found to exist before creating") + b := s.open(t) + defer s.close(t, b) - // try to stat a not existing blob - _, err = b.Stat(context.TODO(), h) - test.Assert(t, err != nil, "blob data could be extracted before creation") - test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err) - test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err) + test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") + test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error") - // try to read not existing blob - err = testLoad(b, h) - test.Assert(t, err != nil, "blob could be read before creation") - test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err) - test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err) + // detect non-existing files + for _, ts := range testStrings { + id, err := restic.ParseID(ts.id) + test.OK(t, err) - // try to get string out, should fail - ret, err = beTest(context.TODO(), b, h) - test.OK(t, err) - test.Assert(t, !ret, "id %q was found (but should not have)", ts.id) - } + // test if blob is already in repository + h := backend.Handle{Type: tpe, Name: id.String()} + ret, err := beTest(context.TODO(), b, h) + test.OK(t, err) + test.Assert(t, !ret, "blob was found to exist before creating") - // add files - for _, ts := range testStrings { - store(t, b, tpe, []byte(ts.data)) + // try to stat a not existing blob + _, err = b.Stat(context.TODO(), h) + test.Assert(t, err != nil, "blob data could be extracted before creation") + test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err) - // test Load() + // try to read not existing blob + err = testLoad(b, h) + test.Assert(t, err != nil, "blob could be read before creation") + test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err) + + // try to get string out, should fail + ret, err = beTest(context.TODO(), b, h) + test.OK(t, err) + test.Assert(t, !ret, "id %q was found (but should not have)", ts.id) + } + + // add files + for _, ts := range testStrings { + store(t, b, tpe, []byte(ts.data)) + + // test Load() + h := backend.Handle{Type: tpe, Name: ts.id} + buf, err := LoadAll(context.TODO(), b, h) + test.OK(t, err) + test.Equals(t, ts.data, string(buf)) + + // try to read it out with an offset and a length + start := 1 + end := len(ts.data) - 2 + length := end - start + + buf2 := make([]byte, length) + var n int + err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) { + n, ierr = io.ReadFull(rd, buf2) + return ierr + }) + test.OK(t, err) + test.OK(t, err) + test.Equals(t, len(buf2), n) + test.Equals(t, ts.data[start:end], string(buf2)) + } + + // test adding the first file again + ts := testStrings[0] h := backend.Handle{Type: tpe, Name: ts.id} - buf, err := LoadAll(context.TODO(), b, h) - test.OK(t, err) - test.Equals(t, ts.data, string(buf)) - // try to read it out with an offset and a length - start := 1 - end := len(ts.data) - 2 - length := end - start - - buf2 := make([]byte, length) - var n int - err = b.Load(context.TODO(), h, len(buf2), int64(start), func(rd io.Reader) (ierr error) { - n, ierr = io.ReadFull(rd, buf2) - return ierr - }) - test.OK(t, err) - test.OK(t, err) - test.Equals(t, len(buf2), n) - test.Equals(t, ts.data[start:end], string(buf2)) - } - - // test adding the first file again - ts := testStrings[0] - h := backend.Handle{Type: tpe, Name: ts.id} - - // remove and recreate - err := s.delayedRemove(t, b, h) - test.OK(t, err) - - // test that the blob is gone - ok, err := beTest(context.TODO(), b, h) - test.OK(t, err) - test.Assert(t, !ok, "removed blob still present") - - // create blob - err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher())) - test.OK(t, err) - - // list items - IDs := restic.IDs{} - - for _, ts := range testStrings { - id, err := restic.ParseID(ts.id) - test.OK(t, err) - IDs = append(IDs, id) - } - - list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval) - if len(IDs) != len(list) { - t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list)) - } - - sort.Sort(IDs) - sort.Sort(list) - - if !reflect.DeepEqual(IDs, list) { - t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) - } - - var handles []backend.Handle - for _, ts := range testStrings { - id, err := restic.ParseID(ts.id) + // remove and recreate + err := s.delayedRemove(t, b, h) test.OK(t, err) - h := backend.Handle{Type: tpe, Name: id.String()} - - found, err := beTest(context.TODO(), b, h) + // test that the blob is gone + ok, err := beTest(context.TODO(), b, h) test.OK(t, err) - test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id)) + test.Assert(t, !ok, "removed blob still present") - handles = append(handles, h) - } + // create blob + err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher())) + test.OK(t, err) - test.OK(t, s.delayedRemove(t, b, handles...)) + // list items + IDs := restic.IDs{} + + for _, ts := range testStrings { + id, err := restic.ParseID(ts.id) + test.OK(t, err) + IDs = append(IDs, id) + } + + list := delayedList(t, b, tpe, len(IDs), s.WaitForDelayedRemoval) + if len(IDs) != len(list) { + t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list)) + } + + sort.Sort(IDs) + sort.Sort(list) + + if !reflect.DeepEqual(IDs, list) { + t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) + } + + var handles []backend.Handle + for _, ts := range testStrings { + id, err := restic.ParseID(ts.id) + test.OK(t, err) + + h := backend.Handle{Type: tpe, Name: id.String()} + + found, err := beTest(context.TODO(), b, h) + test.OK(t, err) + test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id)) + + handles = append(handles, h) + } + + test.OK(t, s.delayedRemove(t, b, handles...)) + }) } } From 17b585f7c76ae5d651b86b1553c2f0c9d9dc895b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 16:14:48 +0100 Subject: [PATCH 4/8] backend/test: partially parallelize delayedRemove --- internal/backend/test/tests.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 97bca5b97..9c178cb16 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -725,14 +725,19 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...ba // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. + wg, ctx := errgroup.WithContext(context.TODO()) for _, h := range handles { - err := be.Remove(context.TODO(), h) - if s.ErrorHandler != nil { - err = s.ErrorHandler(t, be, err) - } - if err != nil { + wg.Go(func() error { + err := be.Remove(ctx, h) + if s.ErrorHandler != nil { + err = s.ErrorHandler(t, be, err) + } return err - } + }) + } + err := wg.Wait() + if err != nil { + return err } for _, h := range handles { From 2e58561ad6bc2d1da3e2612a049bfa14d3bb6cd6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 16:15:08 +0100 Subject: [PATCH 5/8] backend/test: remove redundant test call to the backend --- internal/backend/test/tests.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 9c178cb16..6f6ed007e 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -816,7 +816,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { h := backend.Handle{Type: tpe, Name: id.String()} ret, err := beTest(context.TODO(), b, h) test.OK(t, err) - test.Assert(t, !ret, "blob was found to exist before creating") + test.Assert(t, !ret, "id %q was found (but should not have)", ts.id) // try to stat a not existing blob _, err = b.Stat(context.TODO(), h) @@ -829,11 +829,6 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, err != nil, "blob could be read before creation") test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err) test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err) - - // try to get string out, should fail - ret, err = beTest(context.TODO(), b, h) - test.OK(t, err) - test.Assert(t, !ret, "id %q was found (but should not have)", ts.id) } // add files From 4350b95d27e9efa1b4db423b6b786274b9ee209e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 16:16:47 +0100 Subject: [PATCH 6/8] backend/test: fix delayedRemoval timeout handling The timeout for all blobs starts to run after the delete calls have been issue. Thus, use the same start time for all blobs instead of individual timeouts. --- internal/backend/test/tests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 6f6ed007e..b3deba8da 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -740,8 +740,8 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...ba return err } + start := time.Now() for _, h := range handles { - start := time.Now() attempt := 0 var found bool var err error From c99c76ada83e0f7497bec36ab67a2fe96643a8c0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 16:18:47 +0100 Subject: [PATCH 7/8] backend/test: increase parallelism to run all TestBackend tests in parallel --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ea6f0175d..bd05da36b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -185,7 +185,7 @@ jobs: # prepare credentials for Google Cloud Storage tests in a temp file export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX) echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - go test -cover -parallel 4 ./internal/backend/... + go test -cover -parallel 5 ./internal/backend/... # only run cloud backend tests for pull requests from and pushes to our # own repo, otherwise the secrets are not available @@ -204,7 +204,6 @@ jobs: cross_compile: strategy: - matrix: # run cross-compile in three batches parallel so the overall tests run faster subset: From 06535e62c177a4f8bf675fb76913e215c587e820 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 23 Mar 2025 16:28:49 +0100 Subject: [PATCH 8/8] CI: increase timeout to work around slow cloud backend --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bd05da36b..4da17a047 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -185,7 +185,7 @@ jobs: # prepare credentials for Google Cloud Storage tests in a temp file export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX) echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - go test -cover -parallel 5 ./internal/backend/... + go test -cover -parallel 5 -timeout 15m ./internal/backend/... # only run cloud backend tests for pull requests from and pushes to our # own repo, otherwise the secrets are not available