Add MountSensitiveWithFlags to the subpath Mounter interface

- Also adding an implementation for all the platforms

pkg/volume/util/subpath/subpath_linux.go

@@ -44,11 +44,11 @@ const (
 )
 
 type subpath struct {
-	mounter mount.Interface
+	mounter MountInterface
 }
 
 // New returns a subpath.Interface for the current system
-func New(mounter mount.Interface) Interface {
+func New(mounter MountInterface) Interface {
 	return &subpath{
 		mounter: mounter,
 	}

pkg/volume/util/subpath/subpath_mount.go

@@ -26,6 +26,9 @@ import (
 // MountInterface defines the set of methods to allow for mount operations on a system.
 type MountInterface interface {
 	mount.Interface
+
+	// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags
+	MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error
 }
 
 // Compile-time check to ensure all Mounter implementations satisfy

pkg/volume/util/subpath/subpath_mount_linux.go

@@ -19,23 +19,198 @@ limitations under the License.
 package subpath
 
 import (
-	"k8s.io/utils/mount"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"k8s.io/klog/v2"
+	mountutils "k8s.io/utils/mount"
+)
+
+const (
+	// Default mount command if mounter path is not specified.
+	defaultMountCommand = "mount"
+	// Log message where sensitive mount options were removed
+	sensitiveOptionsRemoved = "<masked>"
 )
 
 // Mounter provides the subpath implementation of mount.Interface
 // for the linux platform.  This implementation assumes that the
 // kubelet is running in the host's root mount namespace.
 type Mounter struct {
-	mount.Interface
+	mountutils.Interface
 	mounterPath string
+	withSystemd bool
 }
 
 // NewMounter returns a MountInterface for the current system.
 // It provides options to override the default mounter behavior.
 // mounterPath allows using an alternative to `/bin/mount` for mounting.
-func NewMounter(mounter mount.Interface, mounterPath string) MountInterface {
+func NewMounter(mounter mountutils.Interface, mounterPath string) MountInterface {
 	return &Mounter{
 		Interface:   mounter,
 		mounterPath: mounterPath,
+		withSystemd: detectSystemd(),
 	}
 }
+
+// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags
+func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error {
+	// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
+	// All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
+	mounterPath := ""
+	bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := mountutils.MakeBindOptsSensitive(options, sensitiveOptions)
+	if bind {
+		err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, mountFlags)
+		if err != nil {
+			return err
+		}
+		return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, mountFlags)
+	}
+	// The list of filesystems that require containerized mounter on GCI image cluster
+	fsTypesNeedMounter := map[string]struct{}{
+		"nfs":       {},
+		"glusterfs": {},
+		"ceph":      {},
+		"cifs":      {},
+	}
+	if _, ok := fsTypesNeedMounter[fstype]; ok {
+		mounterPath = mounter.mounterPath
+	}
+	return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, mountFlags)
+}
+
+// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
+// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material)
+// mountFlags are additional flags used in the mount command that are not related with fstype and mount options
+func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error {
+	mountArgs, mountArgsLogStr := makeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, mountFlags)
+	if len(mounterPath) > 0 {
+		mountArgs = append([]string{mountCmd}, mountArgs...)
+		mountArgsLogStr = mountCmd + " " + mountArgsLogStr
+		mountCmd = mounterPath
+	}
+
+	if mounter.withSystemd {
+		// Try to run mount via systemd-run --scope. This will escape the
+		// service where kubelet runs and any fuse daemons will be started in a
+		// specific scope. kubelet service than can be restarted without killing
+		// these fuse daemons.
+		//
+		// Complete command line (when mounterPath is not used):
+		// systemd-run --description=... --scope -- mount -t <type> <what> <where>
+		//
+		// Expected flow:
+		// * systemd-run creates a transient scope (=~ cgroup) and executes its
+		//   argument (/bin/mount) there.
+		// * mount does its job, forks a fuse daemon if necessary and finishes.
+		//   (systemd-run --scope finishes at this point, returning mount's exit
+		//   code and stdout/stderr - thats one of --scope benefits).
+		// * systemd keeps the fuse daemon running in the scope (i.e. in its own
+		//   cgroup) until the fuse daemon dies (another --scope benefit).
+		//   Kubelet service can be restarted and the fuse daemon survives.
+		// * When the fuse daemon dies (e.g. during unmount) systemd removes the
+		//   scope automatically.
+		//
+		// systemd-mount is not used because it's too new for older distros
+		// (CentOS 7, Debian Jessie).
+		mountCmd, mountArgs, mountArgsLogStr = mountutils.AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr)
+		// } else {
+		// No systemd-run on the host (or we failed to check it), assume kubelet
+		// does not run as a systemd service.
+		// No code here, mountCmd and mountArgs are already populated.
+	}
+
+	// Logging with sensitive mount options removed.
+	klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr)
+	command := exec.Command(mountCmd, mountArgs...)
+	output, err := command.CombinedOutput()
+	if err != nil {
+		klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output))
+		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s",
+			err, mountCmd, mountArgsLogStr, string(output))
+	}
+	return err
+}
+
+// detectSystemd returns true if OS runs with systemd as init. When not sure
+// (permission errors, ...), it returns false.
+// There may be different ways how to detect systemd, this one makes sure that
+// systemd-runs (needed by Mount()) works.
+func detectSystemd() bool {
+	if _, err := exec.LookPath("systemd-run"); err != nil {
+		klog.V(2).Infof("Detected OS without systemd")
+		return false
+	}
+	// Try to run systemd-run --scope /bin/true, that should be enough
+	// to make sure that systemd is really running and not just installed,
+	// which happens when running in a container with a systemd-based image
+	// but with different pid 1.
+	cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS")
+		klog.V(4).Infof("systemd-run failed with: %v", err)
+		klog.V(4).Infof("systemd-run output: %s", string(output))
+		return false
+	}
+	klog.V(2).Infof("Detected OS with systemd")
+	return true
+}
+
+// makeMountArgsSensitiveWithMountFlags makes the arguments to the mount(8) command.
+// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material)
+// mountFlags are additional mount flags that are not related with the fstype and mount options
+func makeMountArgsSensitiveWithMountFlags(source, target, fstype string, options []string, sensitiveOptions []string, mountFlags []string) (mountArgs []string, mountArgsLogStr string) {
+	// Build mount command as follows:
+	//   mount [$mountFlags] [-t $fstype] [-o $options] [$source] $target
+	mountArgs = []string{}
+	mountArgsLogStr = ""
+
+	mountArgs = append(mountArgs, mountFlags...)
+	mountArgsLogStr += strings.Join(mountFlags, " ")
+
+	if len(fstype) > 0 {
+		mountArgs = append(mountArgs, "-t", fstype)
+		mountArgsLogStr += strings.Join(mountArgs, " ")
+	}
+	if len(options) > 0 || len(sensitiveOptions) > 0 {
+		combinedOptions := []string{}
+		combinedOptions = append(combinedOptions, options...)
+		combinedOptions = append(combinedOptions, sensitiveOptions...)
+		mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ","))
+		// exclude sensitiveOptions from log string
+		mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions)
+	}
+	if len(source) > 0 {
+		mountArgs = append(mountArgs, source)
+		mountArgsLogStr += " " + source
+	}
+	mountArgs = append(mountArgs, target)
+	mountArgsLogStr += " " + target
+
+	return mountArgs, mountArgsLogStr
+}
+
+// sanitizedOptionsForLogging will return a comma separated string containing
+// options and sensitiveOptions. Each entry in sensitiveOptions will be
+// replaced with the string sensitiveOptionsRemoved
+// e.g. o1,o2,<masked>,<masked>
+func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string {
+	separator := ""
+	if len(options) > 0 && len(sensitiveOptions) > 0 {
+		separator = ","
+	}
+
+	sensitiveOptionsStart := ""
+	sensitiveOptionsEnd := ""
+	if len(sensitiveOptions) > 0 {
+		sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1)
+		sensitiveOptionsEnd = sensitiveOptionsRemoved
+	}
+
+	return strings.Join(options, ",") +
+		separator +
+		sensitiveOptionsStart +
+		sensitiveOptionsEnd
+}

pkg/volume/util/subpath/subpath_mount_unsupported.go

@@ -30,7 +30,7 @@ type Mounter struct {
 	mounterPath string
 }
 
-var errUnsupported = errors.New("utils/mount on this platform is not supported")
+var errUtilsMountUnsupported = errors.New("utils/mount on this platform is not supported")
 
 // NewMounter returns a MountInterface for the current system.
 // It provides options to override the default mounter behavior.
@@ -41,3 +41,8 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface {
 		mounterPath: mounterPath,
 	}
 }
+
+// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags
+func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error {
+	return errUtilsMountUnsupported
+}

pkg/volume/util/subpath/subpath_mount_windows.go

@@ -39,3 +39,9 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface {
 		mounterPath: mounterPath,
 	}
 }
+
+// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags but
+// because mountFlags are linux mount(8) flags this method is the same as MountSensitive() in Windows
+func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error {
+	return mounter.MountSensitive(source, target, fstype, options, sensitiveOptions)
+}