diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index b03800d2fc7..b2f50c20409 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -44,11 +44,11 @@ const ( ) type subpath struct { - mounter mount.Interface + mounter MountInterface } // New returns a subpath.Interface for the current system -func New(mounter mount.Interface) Interface { +func New(mounter MountInterface) Interface { return &subpath{ mounter: mounter, } diff --git a/pkg/volume/util/subpath/subpath_mount.go b/pkg/volume/util/subpath/subpath_mount.go index 76a230b97d9..a8ad8c228a1 100644 --- a/pkg/volume/util/subpath/subpath_mount.go +++ b/pkg/volume/util/subpath/subpath_mount.go @@ -26,6 +26,9 @@ import ( // MountInterface defines the set of methods to allow for mount operations on a system. type MountInterface interface { mount.Interface + + // MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags + MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error } // Compile-time check to ensure all Mounter implementations satisfy diff --git a/pkg/volume/util/subpath/subpath_mount_linux.go b/pkg/volume/util/subpath/subpath_mount_linux.go index 01cfe2c1a05..492b8d7e62f 100644 --- a/pkg/volume/util/subpath/subpath_mount_linux.go +++ b/pkg/volume/util/subpath/subpath_mount_linux.go @@ -19,23 +19,198 @@ limitations under the License. package subpath import ( - "k8s.io/utils/mount" + "fmt" + "os/exec" + "strings" + + "k8s.io/klog/v2" + mountutils "k8s.io/utils/mount" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" + // Log message where sensitive mount options were removed + sensitiveOptionsRemoved = "" ) // Mounter provides the subpath implementation of mount.Interface // for the linux platform. This implementation assumes that the // kubelet is running in the host's root mount namespace. type Mounter struct { - mount.Interface + mountutils.Interface mounterPath string + withSystemd bool } // NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. -func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { +func NewMounter(mounter mountutils.Interface, mounterPath string) MountInterface { return &Mounter{ Interface: mounter, mounterPath: mounterPath, + withSystemd: detectSystemd(), } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := mountutils.MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, mountFlags) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, mountFlags) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, mountFlags) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// mountFlags are additional flags used in the mount command that are not related with fstype and mount options +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + mountArgs, mountArgsLogStr := makeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, mountFlags) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountArgsLogStr = mountCmd + " " + mountArgsLogStr + mountCmd = mounterPath + } + + if mounter.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs, mountArgsLogStr = mountutils.AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) + // } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + // Logging with sensitive mount options removed. + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, mountArgsLogStr, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// makeMountArgsSensitiveWithMountFlags makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// mountFlags are additional mount flags that are not related with the fstype and mount options +func makeMountArgsSensitiveWithMountFlags(source, target, fstype string, options []string, sensitiveOptions []string, mountFlags []string) (mountArgs []string, mountArgsLogStr string) { + // Build mount command as follows: + // mount [$mountFlags] [-t $fstype] [-o $options] [$source] $target + mountArgs = []string{} + mountArgsLogStr = "" + + mountArgs = append(mountArgs, mountFlags...) + mountArgsLogStr += strings.Join(mountFlags, " ") + + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + mountArgsLogStr += strings.Join(mountArgs, " ") + } + if len(options) > 0 || len(sensitiveOptions) > 0 { + combinedOptions := []string{} + combinedOptions = append(combinedOptions, options...) + combinedOptions = append(combinedOptions, sensitiveOptions...) + mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) + // exclude sensitiveOptions from log string + mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + mountArgsLogStr += " " + source + } + mountArgs = append(mountArgs, target) + mountArgsLogStr += " " + target + + return mountArgs, mountArgsLogStr +} + +// sanitizedOptionsForLogging will return a comma separated string containing +// options and sensitiveOptions. Each entry in sensitiveOptions will be +// replaced with the string sensitiveOptionsRemoved +// e.g. o1,o2,, +func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { + separator := "" + if len(options) > 0 && len(sensitiveOptions) > 0 { + separator = "," + } + + sensitiveOptionsStart := "" + sensitiveOptionsEnd := "" + if len(sensitiveOptions) > 0 { + sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) + sensitiveOptionsEnd = sensitiveOptionsRemoved + } + + return strings.Join(options, ",") + + separator + + sensitiveOptionsStart + + sensitiveOptionsEnd +} diff --git a/pkg/volume/util/subpath/subpath_mount_unsupported.go b/pkg/volume/util/subpath/subpath_mount_unsupported.go index d15d619597d..16dcd053396 100644 --- a/pkg/volume/util/subpath/subpath_mount_unsupported.go +++ b/pkg/volume/util/subpath/subpath_mount_unsupported.go @@ -30,7 +30,7 @@ type Mounter struct { mounterPath string } -var errUnsupported = errors.New("utils/mount on this platform is not supported") +var errUtilsMountUnsupported = errors.New("utils/mount on this platform is not supported") // NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. @@ -41,3 +41,8 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { mounterPath: mounterPath, } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + return errUtilsMountUnsupported +} diff --git a/pkg/volume/util/subpath/subpath_mount_windows.go b/pkg/volume/util/subpath/subpath_mount_windows.go index a78eef0a287..b8feefc8c6a 100644 --- a/pkg/volume/util/subpath/subpath_mount_windows.go +++ b/pkg/volume/util/subpath/subpath_mount_windows.go @@ -39,3 +39,9 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { mounterPath: mounterPath, } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags but +// because mountFlags are linux mount(8) flags this method is the same as MountSensitive() in Windows +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + return mounter.MountSensitive(source, target, fstype, options, sensitiveOptions) +}