Merge pull request #5449 from provokateurin/restore-ownership-by-name

feat(internal/fs/node): Restore ownership by name

changelog/unreleased/issue-3572

@@ -0,0 +1,9 @@
+Enhancement: Support restoring ownership by name on UNIX systems
+
+Restic restore used to restore file ownership on UNIX systems by UID and GID.
+It now allows restoring the file ownership by user name and group name with `--ownership-by-name`.
+This allows restoring snapshots on a system where the UID/GID are not the same as they were on the system where the snapshot was created.
+However it does not include support for POSIX ACLs, which are still restored by their numeric value.
+
+https://github.com/restic/restic/issues/3572
+https://github.com/restic/restic/pull/5449

cmd/restic/cmd_restore.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"path/filepath"
+	"runtime"
 	"time"
 
 	"github.com/restic/restic/internal/data"
@@ -35,6 +36,8 @@ repository.
 To only restore a specific subfolder, you can use the "snapshotID:subfolder"
 syntax, where "subfolder" is a path within the snapshot.
 
+POSIX ACLs are always restored by their numeric value, while file ownership can optionally be restored by name instead of numeric value.
+
 EXIT STATUS
 ===========
 
@@ -69,6 +72,7 @@ type RestoreOptions struct {
 	Delete              bool
 	ExcludeXattrPattern []string
 	IncludeXattrPattern []string
+	OwnershipByName     bool
 }
 
 func (opts *RestoreOptions) AddFlags(f *pflag.FlagSet) {
@@ -86,6 +90,9 @@ func (opts *RestoreOptions) AddFlags(f *pflag.FlagSet) {
 	f.BoolVar(&opts.Verify, "verify", false, "verify restored files content")
 	f.Var(&opts.Overwrite, "overwrite", "overwrite behavior, one of (always|if-changed|if-newer|never)")
 	f.BoolVar(&opts.Delete, "delete", false, "delete files from target directory if they do not exist in snapshot. Use '--dry-run -vv' to check what would be deleted")
+	if runtime.GOOS != "windows" {
+		f.BoolVar(&opts.OwnershipByName, "ownership-by-name", false, "restore file ownership by user name and group name (except POSIX ACLs)")
+	}
 }
 
 func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
@@ -165,11 +172,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
 
 	progress := restoreui.NewProgress(printer, ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()))
 	res := restorer.NewRestorer(repo, sn, restorer.Options{
-		DryRun:    opts.DryRun,
-		Sparse:    opts.Sparse,
-		Progress:  progress,
-		Overwrite: opts.Overwrite,
-		Delete:    opts.Delete,
+		DryRun:          opts.DryRun,
+		Sparse:          opts.Sparse,
+		Progress:        progress,
+		Overwrite:       opts.Overwrite,
+		Delete:          opts.Delete,
+		OwnershipByName: opts.OwnershipByName,
 	})
 
 	totalErrors := 0

internal/fs/node.go

@@ -130,6 +130,39 @@ func lookupUsername(uid uint32) string {
 	return username
 }
 
+var (
+	userNameLookupCache      = make(map[string]uint32)
+	userNameLookupCacheMutex = sync.RWMutex{}
+)
+
+// Cached uid lookup by user name. Returns 0 when no id can be found.
+//
+//nolint:revive // captialization is correct as is
+func lookupUid(userName string) uint32 {
+	userNameLookupCacheMutex.RLock()
+	uid, ok := userNameLookupCache[userName]
+	userNameLookupCacheMutex.RUnlock()
+
+	if ok {
+		return uid
+	}
+
+	u, err := user.Lookup(userName)
+	if err == nil {
+		var s int
+		s, err = strconv.Atoi(u.Uid)
+		if err == nil {
+			uid = uint32(s)
+		}
+	}
+
+	userNameLookupCacheMutex.Lock()
+	userNameLookupCache[userName] = uid
+	userNameLookupCacheMutex.Unlock()
+
+	return uid
+}
+
 var (
 	gidLookupCache      = make(map[uint32]string)
 	gidLookupCacheMutex = sync.RWMutex{}
@@ -157,6 +190,37 @@ func lookupGroup(gid uint32) string {
 	return group
 }
 
+var (
+	groupNameLookupCache      = make(map[string]uint32)
+	groupNameLookupCacheMutex = sync.RWMutex{}
+)
+
+// Cached uid lookup by group name. Returns 0 when no id can be found.
+func lookupGid(groupName string) uint32 {
+	groupNameLookupCacheMutex.RLock()
+	gid, ok := groupNameLookupCache[groupName]
+	groupNameLookupCacheMutex.RUnlock()
+
+	if ok {
+		return gid
+	}
+
+	g, err := user.LookupGroup(groupName)
+	if err == nil {
+		var s int
+		s, err = strconv.Atoi(g.Gid)
+		if err == nil {
+			gid = uint32(s)
+		}
+	}
+
+	groupNameLookupCacheMutex.Lock()
+	groupNameLookupCache[groupName] = gid
+	groupNameLookupCacheMutex.Unlock()
+
+	return gid
+}
+
 // NodeCreateAt creates the node at the given path but does NOT restore node meta data.
 func NodeCreateAt(node *data.Node, path string) (err error) {
 	debug.Log("create node %v at %v", node.Name, path)
@@ -230,8 +294,8 @@ func mkfifo(path string, mode uint32) (err error) {
 }
 
 // NodeRestoreMetadata restores node metadata
-func NodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error {
-	err := nodeRestoreMetadata(node, path, warn, xattrSelectFilter)
+func NodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool, ownershipByName bool) error {
+	err := nodeRestoreMetadata(node, path, warn, xattrSelectFilter, ownershipByName)
 	if err != nil {
 		// It is common to have permission errors for folders like /home
 		// unless you're running as root, so ignore those.
@@ -246,10 +310,10 @@ func NodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xa
 	return err
 }
 
-func nodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error {
+func nodeRestoreMetadata(node *data.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool, ownershipByName bool) error {
 	var firsterr error
 
-	if err := lchown(path, int(node.UID), int(node.GID)); err != nil {
+	if err := lchown(path, node, ownershipByName); err != nil {
 		firsterr = errors.WithStack(err)
 	}
 

internal/fs/node_test.go

@@ -185,81 +185,83 @@ var nodeTests = []data.Node{
 }
 
 func TestNodeRestoreAt(t *testing.T) {
-	tempdir := t.TempDir()
+	for _, ownershipByName := range []bool{false, true} {
+		tempdir := t.TempDir()
 
-	for _, test := range nodeTests {
-		t.Run("", func(t *testing.T) {
-			var nodePath string
-			if test.ExtendedAttributes != nil {
-				if runtime.GOOS == "windows" {
-					// In windows extended attributes are case insensitive and windows returns
-					// the extended attributes in UPPER case.
-					// Update the tests to use UPPER case xattr names for windows.
-					extAttrArr := test.ExtendedAttributes
-					// Iterate through the array using pointers
-					for i := 0; i < len(extAttrArr); i++ {
-						extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name)
+		for _, test := range nodeTests {
+			t.Run("", func(t *testing.T) {
+				var nodePath string
+				if test.ExtendedAttributes != nil {
+					if runtime.GOOS == "windows" {
+						// In windows extended attributes are case insensitive and windows returns
+						// the extended attributes in UPPER case.
+						// Update the tests to use UPPER case xattr names for windows.
+						extAttrArr := test.ExtendedAttributes
+						// Iterate through the array using pointers
+						for i := 0; i < len(extAttrArr); i++ {
+							extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name)
+						}
 					}
+					for _, attr := range test.ExtendedAttributes {
+						if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" {
+							t.Skipf("attr %v only relevant on macOS", attr.Name)
+						}
+					}
+
+					// tempdir might be backed by a filesystem that does not support
+					// extended attributes
+					nodePath = test.Name
+					defer func() {
+						_ = os.Remove(nodePath)
+					}()
+				} else {
+					nodePath = filepath.Join(tempdir, test.Name)
 				}
-				for _, attr := range test.ExtendedAttributes {
-					if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" {
-						t.Skipf("attr %v only relevant on macOS", attr.Name)
+				rtest.OK(t, NodeCreateAt(&test, nodePath))
+				// Restore metadata, restoring all xattrs
+				rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) },
+					func(_ string) bool { return true }, ownershipByName))
+
+				fs := &Local{}
+				meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true)
+				rtest.OK(t, err)
+				n2, err := meta.ToNode(false, t.Logf)
+				rtest.OK(t, err)
+				n3, err := meta.ToNode(true, t.Logf)
+				rtest.OK(t, err)
+				rtest.OK(t, meta.Close())
+				rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3))
+
+				rtest.Assert(t, test.Name == n2.Name,
+					"%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name)
+				rtest.Assert(t, test.Type == n2.Type,
+					"%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type)
+				rtest.Assert(t, test.Size == n2.Size,
+					"%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size)
+
+				if runtime.GOOS != "windows" {
+					rtest.Assert(t, test.UID == n2.UID,
+						"%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID)
+					rtest.Assert(t, test.GID == n2.GID,
+						"%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID)
+					if test.Type != data.NodeTypeSymlink {
+						// On OpenBSD only root can set sticky bit (see sticky(8)).
+						if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" {
+							rtest.Assert(t, test.Mode == n2.Mode,
+								"%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode)
+						}
 					}
 				}
 
-				// tempdir might be backed by a filesystem that does not support
-				// extended attributes
-				nodePath = test.Name
-				defer func() {
-					_ = os.Remove(nodePath)
-				}()
-			} else {
-				nodePath = filepath.Join(tempdir, test.Name)
-			}
-			rtest.OK(t, NodeCreateAt(&test, nodePath))
-			// Restore metadata, restoring all xattrs
-			rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) },
-				func(_ string) bool { return true }))
-
-			fs := &Local{}
-			meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true)
-			rtest.OK(t, err)
-			n2, err := meta.ToNode(false, t.Logf)
-			rtest.OK(t, err)
-			n3, err := meta.ToNode(true, t.Logf)
-			rtest.OK(t, err)
-			rtest.OK(t, meta.Close())
-			rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3))
-
-			rtest.Assert(t, test.Name == n2.Name,
-				"%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name)
-			rtest.Assert(t, test.Type == n2.Type,
-				"%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type)
-			rtest.Assert(t, test.Size == n2.Size,
-				"%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size)
-
-			if runtime.GOOS != "windows" {
-				rtest.Assert(t, test.UID == n2.UID,
-					"%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID)
-				rtest.Assert(t, test.GID == n2.GID,
-					"%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID)
-				if test.Type != data.NodeTypeSymlink {
-					// On OpenBSD only root can set sticky bit (see sticky(8)).
-					if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" {
-						rtest.Assert(t, test.Mode == n2.Mode,
-							"%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode)
-					}
+				AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime)
+				AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime)
+				if len(n2.ExtendedAttributes) == 0 {
+					n2.ExtendedAttributes = nil
 				}
-			}
-
-			AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime)
-			AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime)
-			if len(n2.ExtendedAttributes) == 0 {
-				n2.ExtendedAttributes = nil
-			}
-			rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes),
-				"%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes)
-		})
+				rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes),
+					"%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes)
+			})
+		}
 	}
 }
 
@@ -295,6 +297,6 @@ func TestNodeRestoreMetadataError(t *testing.T) {
 
 	// This will fail because the target file does not exist
 	err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) },
-		func(_ string) bool { return true })
+		func(_ string) bool { return true }, false)
 	rtest.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason")
 }

internal/fs/node_unix.go

@@ -9,8 +9,17 @@ import (
 	"github.com/restic/restic/internal/data"
 )
 
-func lchown(name string, uid, gid int) error {
-	return os.Lchown(name, uid, gid)
+func lchown(name string, node *data.Node, lookupByName bool) error {
+	var uid, gid uint32
+	if lookupByName {
+		uid = lookupUid(node.User)
+		gid = lookupGid(node.Group)
+	} else {
+		uid = node.UID
+		gid = node.GID
+	}
+
+	return os.Lchown(name, int(uid), int(gid))
 }
 
 // nodeRestoreGenericAttributes is no-op.

internal/fs/node_unix_test.go

@@ -6,8 +6,10 @@ package fs
 import (
 	"io/fs"
 	"os"
+	"os/user"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -144,3 +146,42 @@ func TestMknodError(t *testing.T) {
 	rtest.Assert(t, errors.Is(err, os.ErrExist), "want ErrExist, got %q", err)
 	rtest.Assert(t, strings.Contains(err.Error(), d), "filename not in %q", err)
 }
+
+func TestLchown(t *testing.T) {
+	usr, err := user.Current()
+	rtest.OK(t, err)
+
+	uid, err := strconv.Atoi(usr.Uid)
+	rtest.OK(t, err)
+
+	gid, err := strconv.Atoi(usr.Gid)
+	rtest.OK(t, err)
+
+	d := t.TempDir()
+	f := d + "/test.txt"
+	err = os.WriteFile(f, []byte(""), 0o700)
+	rtest.OK(t, err)
+
+	t.Run("by UID/GID", func(t *testing.T) {
+		n := &data.Node{
+			UID: uint32(uid),
+			GID: uint32(gid),
+		}
+
+		err = lchown(f, n, false)
+		rtest.OK(t, err)
+	})
+
+	t.Run("by user name and group name", func(t *testing.T) {
+		group, err := user.LookupGroupId(strconv.Itoa(gid))
+		rtest.OK(t, err)
+
+		n := &data.Node{
+			User:  usr.Username,
+			Group: group.Name,
+		}
+
+		err = lchown(f, n, true)
+		rtest.OK(t, err)
+	})
+}

internal/fs/node_windows.go

@@ -38,7 +38,7 @@ func mknod(_ string, _ uint32, _ uint64) (err error) {
 }
 
 // Windows doesn't need lchown
-func lchown(_ string, _ int, _ int) (err error) {
+func lchown(_ string, _ *data.Node, _ bool) (err error) {
 	return nil
 }
 

internal/fs/node_windows_test.go

@@ -218,7 +218,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *data.Node, warnin
 			// If warning is not expected, this code should not get triggered.
 			test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg))
 		}
-	}, func(_ string) bool { return true })
+	}, func(_ string) bool { return true }, false)
 	test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath))
 
 	fs := &Local{}

internal/restorer/restorer.go

@@ -40,11 +40,12 @@ type Restorer struct {
 var restorerAbortOnAllErrors = func(_ string, err error) error { return err }
 
 type Options struct {
-	DryRun    bool
-	Sparse    bool
-	Progress  *restoreui.Progress
-	Overwrite OverwriteBehavior
-	Delete    bool
+	DryRun          bool
+	Sparse          bool
+	Progress        *restoreui.Progress
+	Overwrite       OverwriteBehavior
+	Delete          bool
+	OwnershipByName bool
 }
 
 type OverwriteBehavior int
@@ -293,7 +294,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *data.Node, target, location str
 		return nil
 	}
 	debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location)
-	err := fs.NodeRestoreMetadata(node, target, res.Warn, res.XattrSelectFilter)
+	err := fs.NodeRestoreMetadata(node, target, res.Warn, res.XattrSelectFilter, res.opts.OwnershipByName)
 	if err != nil {
 		debug.Log("node.RestoreMetadata(%s) error %v", target, err)
 	}