diff --git a/changelog/unreleased/pull-21800 b/changelog/unreleased/pull-21800 new file mode 100644 index 000000000..36776bdb8 --- /dev/null +++ b/changelog/unreleased/pull-21800 @@ -0,0 +1,7 @@ +Enhancement: Add WebDAV backend + +Adds a WebDAV backend that is similar to the REST backend but supports +RFC 4918 compatible WebDAV servers. On-disk repository layout is the +same as the local backend. Uses the `webdav:` prefix for URLs. + +https://github.com/restic/restic/pull/21800 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index dc1fbdb8d..20b54dad1 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -235,6 +235,43 @@ simultaneously. .. _Amazon S3: +WebDAV Server +************* + +Restic can backup data to a server that supports the WebDAV standard. +This works similarly to the REST server, documented above. Once the +server is configured, accessing it is achieved by changing the URL +scheme like this: + +.. code-block:: console + + $ restic -r webdav:http://host:8000/ init + +Depending on your WebDAV server setup, you can use HTTPS protocol, +password protection, multiple repositories or any combination of those +features. The TCP/IP port is also configurable. Here are some more +examples: + +.. code-block:: console + + $ restic -r webdav:https://host:8000/ init + $ restic -r webdav:https://user:pass@host:8000/ init + $ restic -r webdav:https://user:pass@host:8000/my_backup_repo/ init + +The server username and password can be specified using environment +variables as well: + +.. code-block:: console + + $ export RESTIC_WEBDAV_USERNAME= + $ export RESTIC_WEBDAV_PASSWORD= + +WebDAV follows the same TLS validation rules as the REST server. + +WebDAV server uses exactly the same directory structure as local +backend, so you should be able to access it both locally and via HTTP, +even simultaneously. + Amazon S3 ********* diff --git a/internal/backend/all/all.go b/internal/backend/all/all.go index c71acb00d..2713318ea 100644 --- a/internal/backend/all/all.go +++ b/internal/backend/all/all.go @@ -11,6 +11,7 @@ import ( "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/swift" + "github.com/restic/restic/internal/backend/webdav" ) func Backends() *location.Registry { @@ -21,6 +22,7 @@ func Backends() *location.Registry { backends.Register(local.NewFactory()) backends.Register(rclone.NewFactory()) backends.Register(rest.NewFactory()) + backends.Register(webdav.NewFactory()) backends.Register(s3.NewFactory()) backends.Register(sftp.NewFactory()) backends.Register(swift.NewFactory()) diff --git a/internal/backend/webdav/config.go b/internal/backend/webdav/config.go new file mode 100644 index 000000000..d5f9ceb33 --- /dev/null +++ b/internal/backend/webdav/config.go @@ -0,0 +1,90 @@ +package webdav + +import ( + "net/url" + "os" + "strings" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/options" +) + +// Config contains all configuration necessary to connect to a WebDAV server. +type Config struct { + URL *url.URL + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` +} + +func init() { + options.Register("webdav", Config{}) +} + +// NewConfig returns a new Config with the default values filled in. +func NewConfig() Config { + return Config{ + Connections: 5, + } +} + +// ParseConfig parses the string s and extracts the WebDAV server URL. +func ParseConfig(s string) (*Config, error) { + if !strings.HasPrefix(s, "webdav:") { + return nil, errors.New("invalid WebDAV backend specification") + } + + s = prepareURL(s) + + u, err := url.Parse(s) + if err != nil { + return nil, errors.WithStack(err) + } + + cfg := NewConfig() + cfg.URL = u + return &cfg, nil +} + +// StripPassword removes the password from the URL +// If the repository location cannot be parsed as a valid URL, it will be returned as is +// (it's because this function is used for logging errors) +func StripPassword(s string) string { + scheme := s[:7] + s = prepareURL(s) + + u, err := url.Parse(s) + if err != nil { + return scheme + s + } + + if _, set := u.User.Password(); !set { + return scheme + s + } + + // a password was set: we replace it with *** + return scheme + strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) +} + +func prepareURL(s string) string { + s = s[7:] + if !strings.HasSuffix(s, "/") { + s += "/" + } + return s +} + +var _ backend.ApplyEnvironmenter = &Config{} + +// ApplyEnvironment saves values from the environment to the config. +func (cfg *Config) ApplyEnvironment(prefix string) { + username := cfg.URL.User.Username() + _, pwdSet := cfg.URL.User.Password() + + // Only apply env variable values if neither username nor password are provided. + if username == "" && !pwdSet { + envName := os.Getenv(prefix + "RESTIC_WEBDAV_USERNAME") + envPwd := os.Getenv(prefix + "RESTIC_WEBDAV_PASSWORD") + + cfg.URL.User = url.UserPassword(envName, envPwd) + } +} diff --git a/internal/backend/webdav/config_test.go b/internal/backend/webdav/config_test.go new file mode 100644 index 000000000..7a26682a9 --- /dev/null +++ b/internal/backend/webdav/config_test.go @@ -0,0 +1,113 @@ +package webdav + +import ( + "net/url" + "testing" + + "github.com/restic/restic/internal/backend/test" +) + +func parseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + + return u +} + +var configTests = []test.ConfigTestData[Config]{ + { + S: "webdav:http://localhost:1234", + Cfg: Config{ + URL: parseURL("http://localhost:1234/"), + Connections: 5, + }, + }, + { + S: "webdav:http://localhost:1234/", + Cfg: Config{ + URL: parseURL("http://localhost:1234/"), + Connections: 5, + }, + }, + { + S: "webdav:http+unix:///tmp/webdav.socket:/my_backup_repo/", + Cfg: Config{ + URL: parseURL("http+unix:///tmp/webdav.socket:/my_backup_repo/"), + Connections: 5, + }, + }, +} + +func TestParseConfig(t *testing.T) { + test.ParseConfigTester(t, ParseConfig, configTests) +} + +var passwordTests = []struct { + input string + expected string +}{ + { + "webdav:", + "webdav:/", + }, + { + "webdav:localhost/", + "webdav:localhost/", + }, + { + "webdav::123/", + "webdav::123/", + }, + { + "webdav:http://", + "webdav:http://", + }, + { + "webdav:http://hostname.foo:1234/", + "webdav:http://hostname.foo:1234/", + }, + { + "webdav:http://user@hostname.foo:1234/", + "webdav:http://user@hostname.foo:1234/", + }, + { + "webdav:http://user:@hostname.foo:1234/", + "webdav:http://user:***@hostname.foo:1234/", + }, + { + "webdav:http://user:p@hostname.foo:1234/", + "webdav:http://user:***@hostname.foo:1234/", + }, + { + "webdav:http://user:pppppaaafhhfuuwiiehhthhghhdkjaoowpprooghjjjdhhwuuhgjsjhhfdjhruuhsjsdhhfhshhsppwufhhsjjsjs@hostname.foo:1234/", + "webdav:http://user:***@hostname.foo:1234/", + }, + { + "webdav:http://user:password@hostname", + "webdav:http://user:***@hostname/", + }, + { + "webdav:http://user:password@:123", + "webdav:http://user:***@:123/", + }, + { + "webdav:http://user:password@", + "webdav:http://user:***@/", + }, +} + +func TestStripPassword(t *testing.T) { + // Make sure that the factory uses the correct method + StripPassword := NewFactory().StripPassword + + for i, test := range passwordTests { + t.Run(test.input, func(t *testing.T) { + result := StripPassword(test.input) + if result != test.expected { + t.Errorf("test %d: expected '%s' but got '%s'", i, test.expected, result) + } + }) + } +} diff --git a/internal/backend/webdav/webdav.go b/internal/backend/webdav/webdav.go new file mode 100644 index 000000000..ae35b2ea3 --- /dev/null +++ b/internal/backend/webdav/webdav.go @@ -0,0 +1,467 @@ +package webdav + +import ( + "context" + "encoding/xml" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "path" + "slices" + "strconv" + "strings" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" +) + +// make sure the WebDAV backend implements backend.Backend +var _ backend.Backend = &Backend{} + +// Backend uses the WebDAV protocol to access data stored on a server. +type Backend struct { + url *url.URL + connections uint + client http.Client + layout.Layout +} + +// davError is returned whenever the server returns a non-successful HTTP status. +type davError struct { + backend.Handle + StatusCode int + Status string +} + +func (e *davError) Error() string { + if e.StatusCode == http.StatusNotFound && e.Handle.Type.String() != "invalid" { + return fmt.Sprintf("%v does not exist", e.Handle) + } + return fmt.Sprintf("unexpected HTTP response (%v): %v", e.StatusCode, e.Status) +} + +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory("webdav", ParseConfig, StripPassword, Create, Open) +} + +// Open opens the WebDAV backend with the given config. +func Open(_ context.Context, cfg Config, rt http.RoundTripper, _ func(string, ...interface{})) (*Backend, error) { + // use url without trailing slash for layout + url := cfg.URL.String() + if url[len(url)-1] == '/' { + url = url[:len(url)-1] + } + + be := &Backend{ + url: cfg.URL, + client: http.Client{Transport: rt}, + Layout: layout.NewDefaultLayout(url, func(parts ...string) string { + p := make([]string, len(parts)) + copy(p, parts) + p[0] = "/" + return strings.TrimRight(parts[0], "/") + path.Join(p...) + }), + connections: cfg.Connections, + } + + return be, nil +} + +func drainAndClose(resp *http.Response) error { + _, err := io.Copy(io.Discard, resp.Body) + cerr := resp.Body.Close() + + // return first error + if err != nil { + return errors.Errorf("drain: %w", err) + } + return cerr +} + +func createPath(ctx context.Context, be *Backend, url string) error { + req, err := http.NewRequestWithContext(ctx, "MKCOL", url, nil) + if err != nil { + return errors.WithStack(err) + } + + resp, err := be.client.Do(req) + if err != nil { + return errors.Wrap(err, "Create") + } + + if err := drainAndClose(resp); err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return &davError{backend.Handle{}, resp.StatusCode, resp.Status} + } + + return nil +} + +// Create creates a new WebDAV on server configured in config. +// TODO: Change this to MKCOL /{path}; expect 201 Created +func Create(ctx context.Context, cfg Config, rt http.RoundTripper, errorLog func(string, ...interface{})) (*Backend, error) { + be, err := Open(ctx, cfg, rt, errorLog) + if err != nil { + return nil, err + } + + _, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile}) + if err == nil { + return nil, errors.New("config file already exists") + } + + // MKCOL isn't recursive so create the root of the repo first + if err := createPath(ctx, be, cfg.URL.String()); err != nil { + return nil, err + } + + for _, url := range be.Layout.Paths() { + if err := createPath(ctx, be, url); err != nil { + return nil, err + } + } + + return be, nil +} + +func (b *Backend) Properties() backend.Properties { + return backend.Properties{ + Connections: b.connections, + HasAtomicReplace: false, // WebDAV server can prevent overwriting + } +} + +// Hasher may return a hash function for calculating a content hash for the backend +func (b *Backend) Hasher() hash.Hash { + return nil +} + +// Save stores data in the backend at the handle. +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // make sure that client.Post() cannot close the reader by wrapping it + req, err := http.NewRequestWithContext(ctx, + http.MethodPut, b.Filename(h), io.NopCloser(rd)) + if err != nil { + return errors.WithStack(err) + } + req.GetBody = func() (io.ReadCloser, error) { + if err := rd.Rewind(); err != nil { + return nil, err + } + return io.NopCloser(rd), nil + } + req.Header.Set("Content-Type", "application/octet-stream") + + // explicitly set the content length, this prevents chunked encoding and + // let's the server know what's coming. + req.ContentLength = rd.Length() + + resp, err := b.client.Do(req) + if err != nil { + return errors.WithStack(err) + } + + if err := drainAndClose(resp); err != nil { + return err + } + + if !slices.Contains( + []int{http.StatusOK, http.StatusCreated, http.StatusAccepted}, + resp.StatusCode, + ) { + return &davError{h, resp.StatusCode, resp.Status} + } + + return nil +} + +// IsNotExist returns true if the error was caused by a non-existing file. +func (b *Backend) IsNotExist(err error) bool { + var e *davError + return errors.As(err, &e) && e.StatusCode == http.StatusNotFound +} + +func (b *Backend) IsPermanentError(err error) bool { + if b.IsNotExist(err) { + return true + } + + var rerr *davError + if errors.As(err, &rerr) { + if rerr.StatusCode == http.StatusRequestedRangeNotSatisfiable || rerr.StatusCode == http.StatusUnauthorized || rerr.StatusCode == http.StatusForbidden || rerr.StatusCode == http.StatusInsufficientStorage { + return true + } + } + + return false +} + +// Load runs fn with a reader that yields the contents of the file at h at the +// given offset. +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + r, err := b.openReader(ctx, h, length, offset) + if err != nil { + return err + } + err = fn(r) + if err != nil { + _ = r.Close() // ignore error here + return err + } + + // Note: readerat.ReadAt() (the fn) uses io.ReadFull() that doesn't + // wait for EOF after reading body. Due to HTTP/2 stream multiplexing + // and goroutine timings the EOF frame arrives from server (eg. rclone) + // with a delay after reading body. Immediate close might trigger + // HTTP/2 stream reset resulting in the *stream closed* error on server, + // so we wait for EOF before closing body. + var buf [1]byte + _, err = r.Read(buf[:]) + if err == io.EOF { + err = nil + } + + if e := r.Close(); err == nil { + err = e + } + return err +} + +func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.Filename(h), nil) + if err != nil { + return nil, errors.WithStack(err) + } + + byteRange := fmt.Sprintf("bytes=%d-", offset) + if length > 0 { + byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) + } + req.Header.Set("Range", byteRange) + + resp, err := b.client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "client.Do") + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + _ = drainAndClose(resp) + return nil, &davError{h, resp.StatusCode, resp.Status} + } + + if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) { + return nil, &davError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"} + } + + return resp.Body, nil +} + +// Stat returns information about a blob. +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) + if err != nil { + return backend.FileInfo{}, errors.WithStack(err) + } + + resp, err := b.client.Do(req) + if err != nil { + return backend.FileInfo{}, errors.WithStack(err) + } + + if err = drainAndClose(resp); err != nil { + return backend.FileInfo{}, err + } + + if resp.StatusCode != http.StatusOK { + return backend.FileInfo{}, &davError{h, resp.StatusCode, resp.Status} + } + + if resp.ContentLength < 0 { + return backend.FileInfo{}, errors.New("negative content length") + } + + bi := backend.FileInfo{ + Size: resp.ContentLength, + Name: h.Name, + } + + return bi, nil +} + +// Remove removes the blob with the given name and type. +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, b.Filename(h), nil) + if err != nil { + return errors.WithStack(err) + } + + resp, err := b.client.Do(req) + if err != nil { + return errors.Wrap(err, "client.Do") + } + + if err = drainAndClose(resp); err != nil { + return err + } + + if !slices.Contains( + []int{http.StatusOK, http.StatusAccepted, http.StatusNoContent}, + resp.StatusCode, + ) { + return &davError{h, resp.StatusCode, resp.Status} + } + + return nil +} + +type props struct { + Status string `xml:"DAV: status"` + Name string `xml:"DAV: prop>displayname,omitempty"` + Type xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` + Size string `xml:"DAV: prop>getcontentlength,omitempty"` +} + +type propfindresponse struct { + Href string `xml:"DAV: href"` + Props []props `xml:"DAV: propstat"` +} + +func parsePropfind(data io.Reader, parse func(resp *propfindresponse) error) error { + decoder := xml.NewDecoder(data) + for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() { + switch se := t.(type) { + case xml.StartElement: + if se.Name.Local == "response" { + var response propfindresponse + if e := decoder.DecodeElement(&response, &se); e == nil { + if err := parse(&response); err != nil { + return err + } + } + } + } + } + return nil +} + +func getProps(r *propfindresponse, status string) *props { + for _, prop := range r.Props { + if strings.Contains(prop.Status, status) { + return &prop + } + } + return nil +} + +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { + url := b.Dirname(backend.Handle{Type: t}) + if !strings.HasSuffix(url, "/") { + url += "/" + } + + payload := strings.NewReader(` + + + + + + `) + + req, err := http.NewRequestWithContext(ctx, "PROPFIND", url, payload) + if err != nil { + return errors.WithStack(err) + } + req.Header.Set("Accept", "application/xml,text/xml") + req.Header.Set("Accept-Charset", "utf-8") + req.Header.Set("Content-Type", "application/xml;charset=UTF-8") + req.Header.Set("Accept-Encoding", "") // Don't allow compressed response + + resp, err := b.client.Do(req) + if err != nil { + return errors.Wrap(err, "List") + } + + // ignore missing directories + if resp.StatusCode == http.StatusNotFound { + return drainAndClose(resp) + } + + if resp.StatusCode != http.StatusMultiStatus { + _ = drainAndClose(resp) + return &davError{backend.Handle{Type: t}, resp.StatusCode, resp.Status} + } + + err = parsePropfind(resp.Body, func(r *propfindresponse) error { + // Tests expect context cancellation to cause failure in a very + // specific way, but often this backend will have read the entire + // response and be in the middle of decoding it at that time. Check + // context cancellation and return the correct error if the context + // was cancelled. + select { + case <-ctx.Done(): + if err := ctx.Err(); err != nil { + return err + } else { + return context.Canceled + } + default: + } + + if p := getProps(r, "200 OK"); p != nil { + // Skip folders + if p.Type.Local == "collection" { + return nil + } + size, err := strconv.ParseInt(p.Size, 10, 64) + if err != nil { + return err + } + file := backend.FileInfo{ + Name: p.Name, + Size: size, + } + return fn(file) + } + return nil + }) + if err != nil { + return err + } + + if cerr := drainAndClose(resp); cerr != nil && err == nil { + err = cerr + } + return err +} + +// Close closes all open files. +func (b *Backend) Close() error { + // this does not need to do anything, all open files are closed within the + // same function. + return nil +} + +// Delete removes all data in the backend. +func (b *Backend) Delete(ctx context.Context) error { + return util.DefaultDelete(ctx, b) +} + +// Warmup not implemented +func (b *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (b *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/webdav/webdav_test.go b/internal/backend/webdav/webdav_test.go new file mode 100644 index 000000000..06ba6ea29 --- /dev/null +++ b/internal/backend/webdav/webdav_test.go @@ -0,0 +1,106 @@ +package webdav_test + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "os" + "testing" + "time" + + "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/backend/webdav" + rtest "github.com/restic/restic/internal/test" + gowebdav "golang.org/x/net/webdav" +) + +func runWebDAVServer(ctx context.Context, t testing.TB) (*url.URL, func()) { + ctx, cancel := context.WithCancel(ctx) + + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + + address := ln.Addr().(*net.TCPAddr).AddrPort().String() + url, err := url.Parse(fmt.Sprintf("http://%s/restic-test/", address)) + if err != nil { + t.Fatal(err) + } + + server := &http.Server{ + Addr: address, + Handler: &gowebdav.Handler{ + FileSystem: gowebdav.NewMemFS(), + LockSystem: gowebdav.NewMemLS(), + }, + } + + go func() { + server.Serve(ln) + }() + + go func() { + downCtx, downCancel := context.WithTimeout(context.Background(), time.Second) + defer downCancel() + + <-ctx.Done() + server.Shutdown(downCtx) + }() + + return url, cancel +} + +func newTestSuite(url *url.URL, minimalData bool) *test.Suite[webdav.Config] { + return &test.Suite[webdav.Config]{ + MinimalData: minimalData, + Factory: webdav.NewFactory(), + NewConfig: func() (*webdav.Config, error) { + cfg := webdav.NewConfig() + cfg.URL = url + return &cfg, nil + }, + } +} + +func TestBackendWebDAV(t *testing.T) { + defer func() { + if t.Skipped() { + rtest.SkipDisallowed(t, "restic/backend/webdav.TestBackendWebDAV") + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + serverURL, cleanup := runWebDAVServer(ctx, t) + defer cleanup() + + newTestSuite(serverURL, false).RunTests(t) +} + +func TestBackendWebDAVExternalServer(t *testing.T) { + repostr := os.Getenv("RESTIC_TEST_WEBDAV_REPOSITORY") + if repostr == "" { + t.Skipf("environment variable %v not set", "RESTIC_TEST_WEBDAV_REPOSITORY") + } + + cfg, err := webdav.ParseConfig(repostr) + if err != nil { + t.Fatal(err) + } + + newTestSuite(cfg.URL, true).RunTests(t) +} + +func BenchmarkBackendWebDAV(t *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + serverURL, cleanup := runWebDAVServer(ctx, t) + defer cleanup() + + newTestSuite(serverURL, false).RunBenchmarks(t) +}