From 6351e9cd0d820336218ab494f777f88b9de29eb3 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 00:53:17 +0000 Subject: [PATCH 1/7] Duplicate vendor/k8s.io/utils/mount/* history in pkg/volume/util/subpath/subpath_* history. --- pkg/volume/util/subpath/subpath_mount.go | 370 ++++++++++++ .../util/subpath/subpath_mount_linux.go | 551 ++++++++++++++++++ .../util/subpath/subpath_mount_unsupported.go | 77 +++ .../util/subpath/subpath_mount_windows.go | 283 +++++++++ 4 files changed, 1281 insertions(+) create mode 100644 pkg/volume/util/subpath/subpath_mount.go create mode 100644 pkg/volume/util/subpath/subpath_mount_linux.go create mode 100644 pkg/volume/util/subpath/subpath_mount_unsupported.go create mode 100644 pkg/volume/util/subpath/subpath_mount_windows.go diff --git a/pkg/volume/util/subpath/subpath_mount.go b/pkg/volume/util/subpath/subpath_mount.go new file mode 100644 index 00000000000..14997a75c9b --- /dev/null +++ b/pkg/volume/util/subpath/subpath_mount.go @@ -0,0 +1,370 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have +// an alternate platform, we will need to abstract further. + +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + utilexec "k8s.io/utils/exec" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" + // Log message where sensitive mount options were removed + sensitiveOptionsRemoved = "" +) + +// Interface defines the set of methods to allow for mount operations on a system. +type Interface interface { + // Mount mounts source to target as fstype with given options. + // options MUST not contain sensitive material (like passwords). + Mount(source string, target string, fstype string, options []string) error + // MountSensitive is the same as Mount() but this method allows + // sensitiveOptions to be passed in a separate parameter from the normal + // mount options and ensures the sensitiveOptions are never logged. This + // method should be used by callers that pass sensitive material (like + // passwords) as mount options. + MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts directly from the OS is not guaranteed + // consistent (i.e. it could change between chunked reads). This is guaranteed + // to be consistent. + List() ([]MountPoint, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is not a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts and symbolic link. For callers that do not + // care about such situations, this is a faster alternative to calling List() + // and scanning that output. + IsLikelyNotMountPoint(file string) (bool, error) + // GetMountRefs finds all mount references to pathname, returning a slice of + // paths. Pathname can be a mountpoint path or a normal directory + // (for bind mount). On Linux, pathname is excluded from the slice. + // For example, if /dev/sdc was mounted at /path/a and /path/b, + // GetMountRefs("/path/a") would return ["/path/b"] + // GetMountRefs("/path/b") would return ["/path/a"] + // On Windows there is no way to query all mount points; as long as pathname is + // a valid mount, it will be returned. + GetMountRefs(pathname string) ([]string, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface. +var _ Interface = &Mounter{} + +// MountPoint represents a single line in /proc/mounts or /etc/fstab. +type MountPoint struct { // nolint: golint + Device string + Path string + Type string + Opts []string // Opts may contain sensitive mount options (like passwords) and MUST be treated as such (e.g. not logged). + Freq int + Pass int +} + +type MountErrorType string // nolint: golint + +const ( + FilesystemMismatch MountErrorType = "FilesystemMismatch" + HasFilesystemErrors MountErrorType = "HasFilesystemErrors" + UnformattedReadOnly MountErrorType = "UnformattedReadOnly" + FormatFailed MountErrorType = "FormatFailed" + GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed" + UnknownMountError MountErrorType = "UnknownMountError" +) + +type MountError struct { // nolint: golint + Type MountErrorType + Message string +} + +func (mountError MountError) String() string { + return mountError.Message +} + +func (mountError MountError) Error() string { + return mountError.Message +} + +func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error { + mountError := MountError{ + Type: mountErrorValue, + Message: fmt.Sprintf(format, args...), + } + return mountError +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec utilexec.Interface +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +// options MUST not contain sensitive material (like passwords). +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// FormatAndMountSensitive is the same as FormatAndMount but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions) +} + +// getMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +// Note that mountPath should be path after the evaluation of any symblolic links. +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + + // Finding the device mounted to mountPath. + diskDev := "" + for i := range mps { + if mountPath == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code. +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned. + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// IsNotMountPoint is more expensive than IsLikelyNotMountPoint. +// IsNotMountPoint detects bind mounts in linux. +// IsNotMountPoint enumerates all the mountpoints using List() and +// the list of mountpoints may be large, then it uses +// isMountPointMatch to evaluate whether the directory is a mountpoint. +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + // IsLikelyNotMountPoint provides a quick check + // to determine whether file IS A mountpoint. + notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil { + return notMnt, notMntErr + } + // identified as mountpoint, so return this fact. + if notMnt == false { + return notMnt, nil + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts. + resolvedFile, err := filepath.EvalSymlinks(file) + if err != nil { + return true, err + } + + // check all mountpoints since IsLikelyNotMountPoint + // is not reliable for some mountpoint types. + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return notMnt, mountPointsErr + } + for _, mp := range mountPoints { + if isMountPointMatch(mp, resolvedFile) { + notMnt = false + break + } + } + return notMnt, nil +} + +// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func MakeBindOpts(options []string) (bool, []string, []string) { + bind, bindOpts, bindRemountOpts, _ := MakeBindOptsSensitive(options, nil /* sensitiveOptions */) + return bind, bindOpts, bindRemountOpts +} + +// MakeBindOptsSensitive is the same as MakeBindOpts but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func MakeBindOptsSensitive(options []string, sensitiveOptions []string) (bool, []string, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bindRemountSensitiveOpts := []string{} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options, sensitiveOptions) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + break + case "remount": + break + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + for _, sensitiveOption := range sensitiveOptions { + switch sensitiveOption { + case "bind": + bind = true + break + case "remount": + break + default: + bindRemountSensitiveOpts = append(bindRemountSensitiveOpts, sensitiveOption) + } + } + + return bind, bindOpts, bindRemountOpts, bindRemountSensitiveOpts +} + +func checkForNetDev(options []string, sensitiveOptions []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + for _, sensitiveOption := range sensitiveOptions { + if sensitiveOption == "_netdev" { + return true + } + } + return false +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if StartsWithBackstep(rel) { + // Needed to escape the base path. + return false + } + return true +} + +// StartsWithBackstep checks if the given path starts with a backstep segment. +func StartsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} + +// sanitizedOptionsForLogging will return a comma separated string containing +// options and sensitiveOptions. Each entry in sensitiveOptions will be +// replaced with the string sensitiveOptionsRemoved +// e.g. o1,o2,, +func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { + separator := "" + if len(options) > 0 && len(sensitiveOptions) > 0 { + separator = "," + } + + sensitiveOptionsStart := "" + sensitiveOptionsEnd := "" + if len(sensitiveOptions) > 0 { + sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) + sensitiveOptionsEnd = sensitiveOptionsRemoved + } + + return strings.Join(options, ",") + + separator + + sensitiveOptionsStart + + sensitiveOptionsEnd +} diff --git a/pkg/volume/util/subpath/subpath_mount_linux.go b/pkg/volume/util/subpath/subpath_mount_linux.go new file mode 100644 index 00000000000..b7a443fdf6c --- /dev/null +++ b/pkg/volume/util/subpath/subpath_mount_linux.go @@ -0,0 +1,551 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog/v2" + utilexec "k8s.io/utils/exec" + utilio "k8s.io/utils/io" +) + +const ( + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return mounter.MountSensitive(source, target, fstype, options, nil) +} + +// MountSensitive is the same as Mount() but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string) error { + mountArgs, mountArgsLogStr := MakeMountArgsSensitive(source, target, fstype, options, sensitiveOptions) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountArgsLogStr = mountCmd + " " + mountArgsLogStr + mountCmd = mounterPath + } + + if mounter.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs, mountArgsLogStr = AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) + } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + // Logging with sensitive mount options removed. + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, mountArgsLogStr, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// MakeMountArgs makes the arguments to the mount(8) command. +// options MUST not contain sensitive material (like passwords). +func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) { + mountArgs, _ = MakeMountArgsSensitive(source, target, fstype, options, nil /* sensitiveOptions */) + return mountArgs +} + +// MakeMountArgsSensitive makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +func MakeMountArgsSensitive(source, target, fstype string, options []string, sensitiveOptions []string) (mountArgs []string, mountArgsLogStr string) { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs = []string{} + mountArgsLogStr = "" + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + mountArgsLogStr += strings.Join(mountArgs, " ") + } + if len(options) > 0 || len(sensitiveOptions) > 0 { + combinedOptions := []string{} + combinedOptions = append(combinedOptions, options...) + combinedOptions = append(combinedOptions, sensitiveOptions...) + mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) + // exclude sensitiveOptions from log string + mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + mountArgsLogStr += " " + source + } + mountArgs = append(mountArgs, target) + mountArgsLogStr += " " + target + + return mountArgs, mountArgsLogStr +} + +// AddSystemdScope adds "system-run --scope" to given command line +// If args contains sensitive material, use AddSystemdScopeSensitive to construct +// a safe to log string. +func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// AddSystemdScopeSensitive adds "system-run --scope" to given command line +// It also accepts takes a sanitized string containing mount arguments, mountArgsLogStr, +// and returns the string appended to the systemd command for logging. +func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []string, mountArgsLogStr string) (string, []string, string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...), strings.Join(systemdRunArgs, " ") + " " + mountArgsLogStr +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return ListProcMounts(procMountsPath) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// It also can not distinguish between mountpoints and symbolic links. +// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// is of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// GetMountRefs finds all mount references to pathname, returns a +// list of paths. Path could be a mountpoint or a normal +// directory (for bind mount). +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + pathExists, pathErr := PathExists(pathname) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr) + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return SearchMountPoints(realpath, procMountInfoPath) +} + +// checkAndRepairFileSystem checks and repairs filesystems using command fsck. +func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error { + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + return nil +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + if !readOnly { + // Check sensitiveOptions for ro + for _, option := range sensitiveOptions { + if option == "ro" { + readOnly = true + break + } + } + } + + options = append(options, "defaults") + mountErrorValue := UnknownMountError + + // Check if the disk is already formatted + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err) + } + + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if existingFormat == "" { + // Do not attempt to format the disk if mounting as readonly, return an error to reflect this. + if readOnly { + return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source) + } + + // Disk is unformatted so format it. + args := []string{source} + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput() + if err != nil { + // Do not log sensitiveOptions only options + sensitiveOptionsLog := sanitizedOptionsForLogging(options, sensitiveOptions) + detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, sensitiveOptionsLog, err, string(output)) + klog.Error(detailedErr) + return NewMountError(FormatFailed, detailedErr) + } + + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + } else { + if fstype != existingFormat { + // Verify that the disk is formatted with filesystem type we are expecting + mountErrorValue = FilesystemMismatch + klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype) + } + + if !readOnly { + // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw. + err := mounter.checkAndRepairFilesystem(source) + if err != nil { + return err + } + } + } + + // Mount the disk + klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target) + if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil { + return NewMountError(mountErrorValue, err.Error()) + } + + return nil +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Command("blkid", args...).CombinedOutput() + output := string(dataOut) + klog.V(4).Infof("Output: %q, err: %v", output, err) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// ListProcMounts is shared with NsEnterMounter +func ListProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + // Do not log line in case it contains sensitive Mount options + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d)", expectedNumFieldsPerLine, len(fields)) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +// SearchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// The source can be a mount point or a normal directory (bind mount). We +// didn't support device because there is no use case by now. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := ParseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + major := -1 + minor := -1 + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].ID + rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint)) + major = mis[i].Major + minor = mis[i].Minor + break + } + } + + if rootPath == "" || major == -1 || minor == -1 { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].ID == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor { + refs = append(refs, mis[i].MountPoint) + } + } + + return refs, nil +} diff --git a/pkg/volume/util/subpath/subpath_mount_unsupported.go b/pkg/volume/util/subpath/subpath_mount_unsupported.go new file mode 100644 index 00000000000..985edbe3d56 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_mount_unsupported.go @@ -0,0 +1,77 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" +) + +// Mounter implements mount.Interface for unsupported platforms +type Mounter struct { + mounterPath string +} + +var errUnsupported = errors.New("util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount always returns an error on unsupported platforms +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return errUnsupported +} + +// Mount always returns an error on unsupported platforms +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return errUnsupported +} + +// Unmount always returns an error on unsupported platforms +func (mounter *Mounter) Unmount(target string) error { + return errUnsupported +} + +// List always returns an error on unsupported platforms +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, errUnsupported +} + +// IsLikelyNotMountPoint always returns an error on unsupported platforms +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, errUnsupported +} + +// GetMountRefs always returns an error on unsupported platforms +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errUnsupported +} + +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, errUnsupported +} diff --git a/pkg/volume/util/subpath/subpath_mount_windows.go b/pkg/volume/util/subpath/subpath_mount_windows.go new file mode 100644 index 00000000000..0534649c062 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_mount_windows.go @@ -0,0 +1,283 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/klog/v2" + "k8s.io/utils/keymutex" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// acquire lock for smb mount +var getSMBMountMutex = keymutex.NewHashed(0) + +// Mount : mounts source to target with given options. +// currently only supports cifs(smb), bind mount(for disk) +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// MountSensitive is the same as Mount() but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + target = NormalizeWindowsPath(target) + sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions) + + if source == "tmpfs" { + klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + sanitizedOptionsForLogging, source, target, fstype) + bindSource := source + + if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind { + bindSource = NormalizeWindowsPath(source) + } else { + allOptions := []string{} + allOptions = append(allOptions, options...) + allOptions = append(allOptions, sensitiveOptions...) + if len(allOptions) < 2 { + klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + sanitizedOptionsForLogging, len(allOptions), source, target) + return nil + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging) + } + + // lock smb mount for the same source + getSMBMountMutex.LockKey(source) + defer getSMBMountMutex.UnlockKey(source) + + if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil { + if isSMBMappingExist(source) { + klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) + if output, err := removeSMBMapping(source); err != nil { + return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) + } + if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil { + return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) + } + } else { + return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) + } + } + } + + output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput() + if err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + klog.V(2).Infof("mklink source(%q) on target(%q) successfully, output: %q", bindSource, target, string(output)) + + return nil +} + +// do the SMB mount with username, password, remotepath +// return (output, error) +func newSMBMapping(username, password, remotepath string) (string, error) { + if username == "" || password == "" || remotepath == "" { + return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath) + } + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", username), + fmt.Sprintf("smbpassword=%s", password), + fmt.Sprintf("smbremotepath=%s", remotepath)) + + output, err := cmd.CombinedOutput() + return string(output), err +} + +// check whether remotepath is already mounted +func isSMBMappingExist(remotepath string) bool { + cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + _, err := cmd.CombinedOutput() + return err == nil +} + +// remove SMB mapping +func removeSMBMapping(remotepath string) (string, error) { + cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = NormalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + + if stat.Mode()&os.ModeSymlink != 0 { + return false, err + } + return true, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + windowsPath := NormalizeWindowsPath(pathname) + pathExists, pathErr := PathExists(windowsPath) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr) + } + return []string{pathname}, nil +} + +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + volumeIds, err := listVolumesOnDisk(source) + if err != nil { + return err + } + driverPath := volumeIds[0] + target = NormalizeWindowsPath(target) + output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput() + if err != nil { + klog.Errorf("mklink(%s, %s) failed: %v, output: %q", target, driverPath, err, string(output)) + return err + } + klog.V(2).Infof("formatAndMount disk(%s) fstype(%s) on(%s) with output(%s) successfully", driverPath, fstype, target, string(output)) + return nil +} + +// ListVolumesOnDisk - returns back list of volumes(volumeIDs) in the disk (requested in diskID). +func listVolumesOnDisk(diskID string) (volumeIDs []string, err error) { + cmd := fmt.Sprintf("(Get-Disk -DeviceId %s | Get-Partition | Get-Volume).UniqueId", diskID) + output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() + klog.V(4).Infof("listVolumesOnDisk id from %s: %s", diskID, string(output)) + if err != nil { + return []string{}, fmt.Errorf("error list volumes on disk. cmd: %s, output: %s, error: %v", cmd, string(output), err) + } + + volumeIds := strings.Split(strings.TrimSpace(string(output)), "\r\n") + return volumeIds, nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} From 8337e4764e1efe883b9481d8ffe1e265e44cfe0b Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 19:11:12 +0000 Subject: [PATCH 2/7] New subpath Mounter implements mount.Interface, update kubelet startup - The kubelet startup code will create a new Mounter instance that wraps the existing k8s.io/utils/mount instance - Provided implementations for all the platforms --- cmd/kubelet/app/server.go | 3 +- pkg/volume/util/subpath/subpath_linux.go | 2 +- pkg/volume/util/subpath/subpath_mount.go | 351 +----------- .../util/subpath/subpath_mount_linux.go | 526 +----------------- .../util/subpath/subpath_mount_unsupported.go | 52 +- .../util/subpath/subpath_mount_windows.go | 258 +-------- 6 files changed, 35 insertions(+), 1157 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 7baf008b5e7..9b7c8a8f065 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -363,7 +363,8 @@ func UnsecuredDependencies(s *options.KubeletServer, featureGate featuregate.Fea } mounter := mount.New(s.ExperimentalMounterPath) - subpather := subpath.New(mounter) + subpatherMounter := subpath.NewMounter(mounter, s.ExperimentalMounterPath) + subpather := subpath.New(subpatherMounter) hu := hostutil.NewHostUtil() var pluginRunner = exec.New() diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index 706e1f83da8..b03800d2fc7 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -160,7 +160,7 @@ func getSubpathBindTarget(subpath Subpath) string { return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) } -func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) { +func doBindSubPath(mounter MountInterface, subpath Subpath) (hostPath string, err error) { // Linux, kubelet runs on the host: // - safely open the subpath // - bind-mount /proc//fd/ to subpath target diff --git a/pkg/volume/util/subpath/subpath_mount.go b/pkg/volume/util/subpath/subpath_mount.go index 14997a75c9b..76a230b97d9 100644 --- a/pkg/volume/util/subpath/subpath_mount.go +++ b/pkg/volume/util/subpath/subpath_mount.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,354 +17,17 @@ limitations under the License. // TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have // an alternate platform, we will need to abstract further. -package mount +package subpath import ( - "fmt" - "os" - "path/filepath" - "strings" - - utilexec "k8s.io/utils/exec" + "k8s.io/utils/mount" ) -const ( - // Default mount command if mounter path is not specified. - defaultMountCommand = "mount" - // Log message where sensitive mount options were removed - sensitiveOptionsRemoved = "" -) - -// Interface defines the set of methods to allow for mount operations on a system. -type Interface interface { - // Mount mounts source to target as fstype with given options. - // options MUST not contain sensitive material (like passwords). - Mount(source string, target string, fstype string, options []string) error - // MountSensitive is the same as Mount() but this method allows - // sensitiveOptions to be passed in a separate parameter from the normal - // mount options and ensures the sensitiveOptions are never logged. This - // method should be used by callers that pass sensitive material (like - // passwords) as mount options. - MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error - // Unmount unmounts given target. - Unmount(target string) error - // List returns a list of all mounted filesystems. This can be large. - // On some platforms, reading mounts directly from the OS is not guaranteed - // consistent (i.e. it could change between chunked reads). This is guaranteed - // to be consistent. - List() ([]MountPoint, error) - // IsLikelyNotMountPoint uses heuristics to determine if a directory - // is not a mountpoint. - // It should return ErrNotExist when the directory does not exist. - // IsLikelyNotMountPoint does NOT properly detect all mountpoint types - // most notably linux bind mounts and symbolic link. For callers that do not - // care about such situations, this is a faster alternative to calling List() - // and scanning that output. - IsLikelyNotMountPoint(file string) (bool, error) - // GetMountRefs finds all mount references to pathname, returning a slice of - // paths. Pathname can be a mountpoint path or a normal directory - // (for bind mount). On Linux, pathname is excluded from the slice. - // For example, if /dev/sdc was mounted at /path/a and /path/b, - // GetMountRefs("/path/a") would return ["/path/b"] - // GetMountRefs("/path/b") would return ["/path/a"] - // On Windows there is no way to query all mount points; as long as pathname is - // a valid mount, it will be returned. - GetMountRefs(pathname string) ([]string, error) +// MountInterface defines the set of methods to allow for mount operations on a system. +type MountInterface interface { + mount.Interface } // Compile-time check to ensure all Mounter implementations satisfy // the mount interface. -var _ Interface = &Mounter{} - -// MountPoint represents a single line in /proc/mounts or /etc/fstab. -type MountPoint struct { // nolint: golint - Device string - Path string - Type string - Opts []string // Opts may contain sensitive mount options (like passwords) and MUST be treated as such (e.g. not logged). - Freq int - Pass int -} - -type MountErrorType string // nolint: golint - -const ( - FilesystemMismatch MountErrorType = "FilesystemMismatch" - HasFilesystemErrors MountErrorType = "HasFilesystemErrors" - UnformattedReadOnly MountErrorType = "UnformattedReadOnly" - FormatFailed MountErrorType = "FormatFailed" - GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed" - UnknownMountError MountErrorType = "UnknownMountError" -) - -type MountError struct { // nolint: golint - Type MountErrorType - Message string -} - -func (mountError MountError) String() string { - return mountError.Message -} - -func (mountError MountError) Error() string { - return mountError.Message -} - -func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error { - mountError := MountError{ - Type: mountErrorValue, - Message: fmt.Sprintf(format, args...), - } - return mountError -} - -// SafeFormatAndMount probes a device to see if it is formatted. -// Namely it checks to see if a file system is present. If so it -// mounts it otherwise the device is formatted first then mounted. -type SafeFormatAndMount struct { - Interface - Exec utilexec.Interface -} - -// FormatAndMount formats the given disk, if needed, and mounts it. -// That is if the disk is not formatted and it is not being mounted as -// read-only it will format it first then mount it. Otherwise, if the -// disk is already formatted or it is being mounted as read-only, it -// will be mounted without formatting. -// options MUST not contain sensitive material (like passwords). -func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { - return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) -} - -// FormatAndMountSensitive is the same as FormatAndMount but this method allows -// sensitiveOptions to be passed in a separate parameter from the normal mount -// options and ensures the sensitiveOptions are never logged. This method should -// be used by callers that pass sensitive material (like passwords) as mount -// options. -func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions) -} - -// getMountRefsByDev finds all references to the device provided -// by mountPath; returns a list of paths. -// Note that mountPath should be path after the evaluation of any symblolic links. -func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { - mps, err := mounter.List() - if err != nil { - return nil, err - } - - // Finding the device mounted to mountPath. - diskDev := "" - for i := range mps { - if mountPath == mps[i].Path { - diskDev = mps[i].Device - break - } - } - - // Find all references to the device. - var refs []string - for i := range mps { - if mps[i].Device == diskDev || mps[i].Device == mountPath { - if mps[i].Path != mountPath { - refs = append(refs, mps[i].Path) - } - } - } - return refs, nil -} - -// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts -// returns the device name, reference count, and error code. -func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { - mps, err := mounter.List() - if err != nil { - return "", 0, err - } - - // Find the device name. - // FIXME if multiple devices mounted on the same mount path, only the first one is returned. - device := "" - // If mountPath is symlink, need get its target path. - slTarget, err := filepath.EvalSymlinks(mountPath) - if err != nil { - slTarget = mountPath - } - for i := range mps { - if mps[i].Path == slTarget { - device = mps[i].Device - break - } - } - - // Find all references to the device. - refCount := 0 - for i := range mps { - if mps[i].Device == device { - refCount++ - } - } - return device, refCount, nil -} - -// IsNotMountPoint determines if a directory is a mountpoint. -// It should return ErrNotExist when the directory does not exist. -// IsNotMountPoint is more expensive than IsLikelyNotMountPoint. -// IsNotMountPoint detects bind mounts in linux. -// IsNotMountPoint enumerates all the mountpoints using List() and -// the list of mountpoints may be large, then it uses -// isMountPointMatch to evaluate whether the directory is a mountpoint. -func IsNotMountPoint(mounter Interface, file string) (bool, error) { - // IsLikelyNotMountPoint provides a quick check - // to determine whether file IS A mountpoint. - notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) - if notMntErr != nil && os.IsPermission(notMntErr) { - // We were not allowed to do the simple stat() check, e.g. on NFS with - // root_squash. Fall back to /proc/mounts check below. - notMnt = true - notMntErr = nil - } - if notMntErr != nil { - return notMnt, notMntErr - } - // identified as mountpoint, so return this fact. - if notMnt == false { - return notMnt, nil - } - - // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts. - resolvedFile, err := filepath.EvalSymlinks(file) - if err != nil { - return true, err - } - - // check all mountpoints since IsLikelyNotMountPoint - // is not reliable for some mountpoint types. - mountPoints, mountPointsErr := mounter.List() - if mountPointsErr != nil { - return notMnt, mountPointsErr - } - for _, mp := range mountPoints { - if isMountPointMatch(mp, resolvedFile) { - notMnt = false - break - } - } - return notMnt, nil -} - -// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to -// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. -// The list equals: -// options - 'bind' + 'remount' (no duplicate) -func MakeBindOpts(options []string) (bool, []string, []string) { - bind, bindOpts, bindRemountOpts, _ := MakeBindOptsSensitive(options, nil /* sensitiveOptions */) - return bind, bindOpts, bindRemountOpts -} - -// MakeBindOptsSensitive is the same as MakeBindOpts but this method allows -// sensitiveOptions to be passed in a separate parameter from the normal mount -// options and ensures the sensitiveOptions are never logged. This method should -// be used by callers that pass sensitive material (like passwords) as mount -// options. -func MakeBindOptsSensitive(options []string, sensitiveOptions []string) (bool, []string, []string, []string) { - // Because we have an FD opened on the subpath bind mount, the "bind" option - // needs to be included, otherwise the mount target will error as busy if you - // remount as readonly. - // - // As a consequence, all read only bind mounts will no longer change the underlying - // volume mount to be read only. - bindRemountOpts := []string{"bind", "remount"} - bindRemountSensitiveOpts := []string{} - bind := false - bindOpts := []string{"bind"} - - // _netdev is a userspace mount option and does not automatically get added when - // bind mount is created and hence we must carry it over. - if checkForNetDev(options, sensitiveOptions) { - bindOpts = append(bindOpts, "_netdev") - } - - for _, option := range options { - switch option { - case "bind": - bind = true - break - case "remount": - break - default: - bindRemountOpts = append(bindRemountOpts, option) - } - } - - for _, sensitiveOption := range sensitiveOptions { - switch sensitiveOption { - case "bind": - bind = true - break - case "remount": - break - default: - bindRemountSensitiveOpts = append(bindRemountSensitiveOpts, sensitiveOption) - } - } - - return bind, bindOpts, bindRemountOpts, bindRemountSensitiveOpts -} - -func checkForNetDev(options []string, sensitiveOptions []string) bool { - for _, option := range options { - if option == "_netdev" { - return true - } - } - for _, sensitiveOption := range sensitiveOptions { - if sensitiveOption == "_netdev" { - return true - } - } - return false -} - -// PathWithinBase checks if give path is within given base directory. -func PathWithinBase(fullPath, basePath string) bool { - rel, err := filepath.Rel(basePath, fullPath) - if err != nil { - return false - } - if StartsWithBackstep(rel) { - // Needed to escape the base path. - return false - } - return true -} - -// StartsWithBackstep checks if the given path starts with a backstep segment. -func StartsWithBackstep(rel string) bool { - // normalize to / and check for ../ - return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") -} - -// sanitizedOptionsForLogging will return a comma separated string containing -// options and sensitiveOptions. Each entry in sensitiveOptions will be -// replaced with the string sensitiveOptionsRemoved -// e.g. o1,o2,, -func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { - separator := "" - if len(options) > 0 && len(sensitiveOptions) > 0 { - separator = "," - } - - sensitiveOptionsStart := "" - sensitiveOptionsEnd := "" - if len(sensitiveOptions) > 0 { - sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) - sensitiveOptionsEnd = sensitiveOptionsRemoved - } - - return strings.Join(options, ",") + - separator + - sensitiveOptionsStart + - sensitiveOptionsEnd -} +var _ MountInterface = &Mounter{} diff --git a/pkg/volume/util/subpath/subpath_mount_linux.go b/pkg/volume/util/subpath/subpath_mount_linux.go index b7a443fdf6c..01cfe2c1a05 100644 --- a/pkg/volume/util/subpath/subpath_mount_linux.go +++ b/pkg/volume/util/subpath/subpath_mount_linux.go @@ -1,7 +1,7 @@ // +build linux /* -Copyright 2014 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,536 +16,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "syscall" - - "k8s.io/klog/v2" - utilexec "k8s.io/utils/exec" - utilio "k8s.io/utils/io" + "k8s.io/utils/mount" ) -const ( - // Number of fields per line in /proc/mounts as per the fstab man page. - expectedNumFieldsPerLine = 6 - // Location of the mount file to use - procMountsPath = "/proc/mounts" - // Location of the mountinfo file - procMountInfoPath = "/proc/self/mountinfo" - // 'fsck' found errors and corrected them - fsckErrorsCorrected = 1 - // 'fsck' found errors but exited without correcting them - fsckErrorsUncorrected = 4 -) - -// Mounter provides the default implementation of mount.Interface +// Mounter provides the subpath implementation of mount.Interface // for the linux platform. This implementation assumes that the // kubelet is running in the host's root mount namespace. type Mounter struct { + mount.Interface mounterPath string - withSystemd bool } -// New returns a mount.Interface for the current system. +// NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { +func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { return &Mounter{ + Interface: mounter, mounterPath: mounterPath, - withSystemd: detectSystemd(), } } - -// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must -// be an empty string in case it's not required, e.g. for remount, or for auto filesystem -// type, where kernel handles fstype for you. The mount 'options' is a list of options, -// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is -// required, call Mount with an empty string list or nil. -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - return mounter.MountSensitive(source, target, fstype, options, nil) -} - -// MountSensitive is the same as Mount() but this method allows -// sensitiveOptions to be passed in a separate parameter from the normal -// mount options and ensures the sensitiveOptions are never logged. This -// method should be used by callers that pass sensitive material (like -// passwords) as mount options. -func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. - // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. - mounterPath := "" - bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) - if bind { - err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive) - if err != nil { - return err - } - return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive) - } - // The list of filesystems that require containerized mounter on GCI image cluster - fsTypesNeedMounter := map[string]struct{}{ - "nfs": {}, - "glusterfs": {}, - "ceph": {}, - "cifs": {}, - } - if _, ok := fsTypesNeedMounter[fstype]; ok { - mounterPath = mounter.mounterPath - } - return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions) -} - -// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. -// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) -func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string) error { - mountArgs, mountArgsLogStr := MakeMountArgsSensitive(source, target, fstype, options, sensitiveOptions) - if len(mounterPath) > 0 { - mountArgs = append([]string{mountCmd}, mountArgs...) - mountArgsLogStr = mountCmd + " " + mountArgsLogStr - mountCmd = mounterPath - } - - if mounter.withSystemd { - // Try to run mount via systemd-run --scope. This will escape the - // service where kubelet runs and any fuse daemons will be started in a - // specific scope. kubelet service than can be restarted without killing - // these fuse daemons. - // - // Complete command line (when mounterPath is not used): - // systemd-run --description=... --scope -- mount -t - // - // Expected flow: - // * systemd-run creates a transient scope (=~ cgroup) and executes its - // argument (/bin/mount) there. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // (systemd-run --scope finishes at this point, returning mount's exit - // code and stdout/stderr - thats one of --scope benefits). - // * systemd keeps the fuse daemon running in the scope (i.e. in its own - // cgroup) until the fuse daemon dies (another --scope benefit). - // Kubelet service can be restarted and the fuse daemon survives. - // * When the fuse daemon dies (e.g. during unmount) systemd removes the - // scope automatically. - // - // systemd-mount is not used because it's too new for older distros - // (CentOS 7, Debian Jessie). - mountCmd, mountArgs, mountArgsLogStr = AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) - } else { - // No systemd-run on the host (or we failed to check it), assume kubelet - // does not run as a systemd service. - // No code here, mountCmd and mountArgs are already populated. - } - - // Logging with sensitive mount options removed. - klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) - command := exec.Command(mountCmd, mountArgs...) - output, err := command.CombinedOutput() - if err != nil { - klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) - return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", - err, mountCmd, mountArgsLogStr, string(output)) - } - return err -} - -// detectSystemd returns true if OS runs with systemd as init. When not sure -// (permission errors, ...), it returns false. -// There may be different ways how to detect systemd, this one makes sure that -// systemd-runs (needed by Mount()) works. -func detectSystemd() bool { - if _, err := exec.LookPath("systemd-run"); err != nil { - klog.V(2).Infof("Detected OS without systemd") - return false - } - // Try to run systemd-run --scope /bin/true, that should be enough - // to make sure that systemd is really running and not just installed, - // which happens when running in a container with a systemd-based image - // but with different pid 1. - cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") - output, err := cmd.CombinedOutput() - if err != nil { - klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") - klog.V(4).Infof("systemd-run failed with: %v", err) - klog.V(4).Infof("systemd-run output: %s", string(output)) - return false - } - klog.V(2).Infof("Detected OS with systemd") - return true -} - -// MakeMountArgs makes the arguments to the mount(8) command. -// options MUST not contain sensitive material (like passwords). -func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) { - mountArgs, _ = MakeMountArgsSensitive(source, target, fstype, options, nil /* sensitiveOptions */) - return mountArgs -} - -// MakeMountArgsSensitive makes the arguments to the mount(8) command. -// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) -func MakeMountArgsSensitive(source, target, fstype string, options []string, sensitiveOptions []string) (mountArgs []string, mountArgsLogStr string) { - // Build mount command as follows: - // mount [-t $fstype] [-o $options] [$source] $target - mountArgs = []string{} - mountArgsLogStr = "" - if len(fstype) > 0 { - mountArgs = append(mountArgs, "-t", fstype) - mountArgsLogStr += strings.Join(mountArgs, " ") - } - if len(options) > 0 || len(sensitiveOptions) > 0 { - combinedOptions := []string{} - combinedOptions = append(combinedOptions, options...) - combinedOptions = append(combinedOptions, sensitiveOptions...) - mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) - // exclude sensitiveOptions from log string - mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) - } - if len(source) > 0 { - mountArgs = append(mountArgs, source) - mountArgsLogStr += " " + source - } - mountArgs = append(mountArgs, target) - mountArgsLogStr += " " + target - - return mountArgs, mountArgsLogStr -} - -// AddSystemdScope adds "system-run --scope" to given command line -// If args contains sensitive material, use AddSystemdScopeSensitive to construct -// a safe to log string. -func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { - descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) - systemdRunArgs := []string{descriptionArg, "--scope", "--", command} - return systemdRunPath, append(systemdRunArgs, args...) -} - -// AddSystemdScopeSensitive adds "system-run --scope" to given command line -// It also accepts takes a sanitized string containing mount arguments, mountArgsLogStr, -// and returns the string appended to the systemd command for logging. -func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []string, mountArgsLogStr string) (string, []string, string) { - descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) - systemdRunArgs := []string{descriptionArg, "--scope", "--", command} - return systemdRunPath, append(systemdRunArgs, args...), strings.Join(systemdRunArgs, " ") + " " + mountArgsLogStr -} - -// Unmount unmounts the target. -func (mounter *Mounter) Unmount(target string) error { - klog.V(4).Infof("Unmounting %s", target) - command := exec.Command("umount", target) - output, err := command.CombinedOutput() - if err != nil { - return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) - } - return nil -} - -// List returns a list of all mounted filesystems. -func (*Mounter) List() ([]MountPoint, error) { - return ListProcMounts(procMountsPath) -} - -// IsLikelyNotMountPoint determines if a directory is not a mountpoint. -// It is fast but not necessarily ALWAYS correct. If the path is in fact -// a bind mount from one part of a mount to another it will not be detected. -// It also can not distinguish between mountpoints and symbolic links. -// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") -// will return true. When in fact /tmp/b is a mount point. If this situation -// is of interest to you, don't use this function... -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - stat, err := os.Stat(file) - if err != nil { - return true, err - } - rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) - if err != nil { - return true, err - } - // If the directory has a different device as parent, then it is a mountpoint. - if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { - return false, nil - } - - return true, nil -} - -// GetMountRefs finds all mount references to pathname, returns a -// list of paths. Path could be a mountpoint or a normal -// directory (for bind mount). -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - pathExists, pathErr := PathExists(pathname) - if !pathExists { - return []string{}, nil - } else if IsCorruptedMnt(pathErr) { - klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname) - return []string{}, nil - } else if pathErr != nil { - return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr) - } - realpath, err := filepath.EvalSymlinks(pathname) - if err != nil { - return nil, err - } - return SearchMountPoints(realpath, procMountInfoPath) -} - -// checkAndRepairFileSystem checks and repairs filesystems using command fsck. -func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error { - klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) - args := []string{"-a", source} - out, err := mounter.Exec.Command("fsck", args...).CombinedOutput() - if err != nil { - ee, isExitError := err.(utilexec.ExitError) - switch { - case err == utilexec.ErrExecutableNotFound: - klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") - case isExitError && ee.ExitStatus() == fsckErrorsCorrected: - klog.Infof("Device %s has errors which were corrected by fsck.", source) - case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: - return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out)) - case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: - klog.Infof("`fsck` error %s", string(out)) - } - } - return nil -} - -// formatAndMount uses unix utils to format and mount the given disk -func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - readOnly := false - for _, option := range options { - if option == "ro" { - readOnly = true - break - } - } - if !readOnly { - // Check sensitiveOptions for ro - for _, option := range sensitiveOptions { - if option == "ro" { - readOnly = true - break - } - } - } - - options = append(options, "defaults") - mountErrorValue := UnknownMountError - - // Check if the disk is already formatted - existingFormat, err := mounter.GetDiskFormat(source) - if err != nil { - return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err) - } - - // Use 'ext4' as the default - if len(fstype) == 0 { - fstype = "ext4" - } - - if existingFormat == "" { - // Do not attempt to format the disk if mounting as readonly, return an error to reflect this. - if readOnly { - return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source) - } - - // Disk is unformatted so format it. - args := []string{source} - if fstype == "ext4" || fstype == "ext3" { - args = []string{ - "-F", // Force flag - "-m0", // Zero blocks reserved for super-user - source, - } - } - - klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) - output, err := mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput() - if err != nil { - // Do not log sensitiveOptions only options - sensitiveOptionsLog := sanitizedOptionsForLogging(options, sensitiveOptions) - detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, sensitiveOptionsLog, err, string(output)) - klog.Error(detailedErr) - return NewMountError(FormatFailed, detailedErr) - } - - klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) - } else { - if fstype != existingFormat { - // Verify that the disk is formatted with filesystem type we are expecting - mountErrorValue = FilesystemMismatch - klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype) - } - - if !readOnly { - // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw. - err := mounter.checkAndRepairFilesystem(source) - if err != nil { - return err - } - } - } - - // Mount the disk - klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target) - if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil { - return NewMountError(mountErrorValue, err.Error()) - } - - return nil -} - -// GetDiskFormat uses 'blkid' to see if the given disk is unformatted -func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { - args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} - klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) - dataOut, err := mounter.Exec.Command("blkid", args...).CombinedOutput() - output := string(dataOut) - klog.V(4).Infof("Output: %q, err: %v", output, err) - - if err != nil { - if exit, ok := err.(utilexec.ExitError); ok { - if exit.ExitStatus() == 2 { - // Disk device is unformatted. - // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was - // not found, or no (specified) devices could be identified, an - // exit code of 2 is returned. - return "", nil - } - } - klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) - return "", err - } - - var fstype, pttype string - - lines := strings.Split(output, "\n") - for _, l := range lines { - if len(l) <= 0 { - // Ignore empty line. - continue - } - cs := strings.Split(l, "=") - if len(cs) != 2 { - return "", fmt.Errorf("blkid returns invalid output: %s", output) - } - // TYPE is filesystem type, and PTTYPE is partition table type, according - // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. - if cs[0] == "TYPE" { - fstype = cs[1] - } else if cs[0] == "PTTYPE" { - pttype = cs[1] - } - } - - if len(pttype) > 0 { - klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) - // Returns a special non-empty string as filesystem type, then kubelet - // will not format it. - return "unknown data, probably partitions", nil - } - - return fstype, nil -} - -// ListProcMounts is shared with NsEnterMounter -func ListProcMounts(mountFilePath string) ([]MountPoint, error) { - content, err := utilio.ConsistentRead(mountFilePath, maxListTries) - if err != nil { - return nil, err - } - return parseProcMounts(content) -} - -func parseProcMounts(content []byte) ([]MountPoint, error) { - out := []MountPoint{} - lines := strings.Split(string(content), "\n") - for _, line := range lines { - if line == "" { - // the last split() item is empty string following the last \n - continue - } - fields := strings.Fields(line) - if len(fields) != expectedNumFieldsPerLine { - // Do not log line in case it contains sensitive Mount options - return nil, fmt.Errorf("wrong number of fields (expected %d, got %d)", expectedNumFieldsPerLine, len(fields)) - } - - mp := MountPoint{ - Device: fields[0], - Path: fields[1], - Type: fields[2], - Opts: strings.Split(fields[3], ","), - } - - freq, err := strconv.Atoi(fields[4]) - if err != nil { - return nil, err - } - mp.Freq = freq - - pass, err := strconv.Atoi(fields[5]) - if err != nil { - return nil, err - } - mp.Pass = pass - - out = append(out, mp) - } - return out, nil -} - -// SearchMountPoints finds all mount references to the source, returns a list of -// mountpoints. -// The source can be a mount point or a normal directory (bind mount). We -// didn't support device because there is no use case by now. -// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, -// it's possible to mount a non-root path of a filesystem, so we need to use -// root path and major:minor to represent mount source uniquely. -// This implementation is shared between Linux and NsEnterMounter -func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) { - mis, err := ParseMountInfo(mountInfoPath) - if err != nil { - return nil, err - } - - mountID := 0 - rootPath := "" - major := -1 - minor := -1 - - // Finding the underlying root path and major:minor if possible. - // We need search in backward order because it's possible for later mounts - // to overlap earlier mounts. - for i := len(mis) - 1; i >= 0; i-- { - if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) { - // If it's a mount point or path under a mount point. - mountID = mis[i].ID - rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint)) - major = mis[i].Major - minor = mis[i].Minor - break - } - } - - if rootPath == "" || major == -1 || minor == -1 { - return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) - } - - var refs []string - for i := range mis { - if mis[i].ID == mountID { - // Ignore mount entry for mount source itself. - continue - } - if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor { - refs = append(refs, mis[i].MountPoint) - } - } - - return refs, nil -} diff --git a/pkg/volume/util/subpath/subpath_mount_unsupported.go b/pkg/volume/util/subpath/subpath_mount_unsupported.go index 985edbe3d56..d15d619597d 100644 --- a/pkg/volume/util/subpath/subpath_mount_unsupported.go +++ b/pkg/volume/util/subpath/subpath_mount_unsupported.go @@ -1,7 +1,7 @@ // +build !linux,!windows /* -Copyright 2014 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,62 +16,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( "errors" + + "k8s.io/utils/mount" ) // Mounter implements mount.Interface for unsupported platforms type Mounter struct { + mount.Interface mounterPath string } -var errUnsupported = errors.New("util/mount on this platform is not supported") +var errUnsupported = errors.New("utils/mount on this platform is not supported") -// New returns a mount.Interface for the current system. +// NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { +func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { return &Mounter{ + Interface: mounter, mounterPath: mounterPath, } } - -// Mount always returns an error on unsupported platforms -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - return errUnsupported -} - -// Mount always returns an error on unsupported platforms -func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - return errUnsupported -} - -// Unmount always returns an error on unsupported platforms -func (mounter *Mounter) Unmount(target string) error { - return errUnsupported -} - -// List always returns an error on unsupported platforms -func (mounter *Mounter) List() ([]MountPoint, error) { - return []MountPoint{}, errUnsupported -} - -// IsLikelyNotMountPoint always returns an error on unsupported platforms -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - return true, errUnsupported -} - -// GetMountRefs always returns an error on unsupported platforms -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - return nil, errUnsupported -} - -func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - return mounter.Interface.Mount(source, target, fstype, options) -} - -func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { - return true, errUnsupported -} diff --git a/pkg/volume/util/subpath/subpath_mount_windows.go b/pkg/volume/util/subpath/subpath_mount_windows.go index 0534649c062..a78eef0a287 100644 --- a/pkg/volume/util/subpath/subpath_mount_windows.go +++ b/pkg/volume/util/subpath/subpath_mount_windows.go @@ -1,7 +1,7 @@ // +build windows /* -Copyright 2017 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,268 +16,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "k8s.io/klog/v2" - "k8s.io/utils/keymutex" + "k8s.io/utils/mount" ) -// Mounter provides the default implementation of mount.Interface +// Mounter provides the subpath implementation of mount.Interface // for the windows platform. This implementation assumes that the // kubelet is running in the host's root mount namespace. type Mounter struct { + mount.Interface mounterPath string } -// New returns a mount.Interface for the current system. +// NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. -func New(mounterPath string) Interface { +func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { return &Mounter{ + Interface: mounter, mounterPath: mounterPath, } } - -// acquire lock for smb mount -var getSMBMountMutex = keymutex.NewHashed(0) - -// Mount : mounts source to target with given options. -// currently only supports cifs(smb), bind mount(for disk) -func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) -} - -// MountSensitive is the same as Mount() but this method allows -// sensitiveOptions to be passed in a separate parameter from the normal -// mount options and ensures the sensitiveOptions are never logged. This -// method should be used by callers that pass sensitive material (like -// passwords) as mount options. -func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - target = NormalizeWindowsPath(target) - sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions) - - if source == "tmpfs" { - klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging) - return os.MkdirAll(target, 0755) - } - - parentDir := filepath.Dir(target) - if err := os.MkdirAll(parentDir, 0755); err != nil { - return err - } - - klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", - sanitizedOptionsForLogging, source, target, fstype) - bindSource := source - - if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind { - bindSource = NormalizeWindowsPath(source) - } else { - allOptions := []string{} - allOptions = append(allOptions, options...) - allOptions = append(allOptions, sensitiveOptions...) - if len(allOptions) < 2 { - klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", - sanitizedOptionsForLogging, len(allOptions), source, target) - return nil - } - - // currently only cifs mount is supported - if strings.ToLower(fstype) != "cifs" { - return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging) - } - - // lock smb mount for the same source - getSMBMountMutex.LockKey(source) - defer getSMBMountMutex.UnlockKey(source) - - if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil { - if isSMBMappingExist(source) { - klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) - if output, err := removeSMBMapping(source); err != nil { - return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) - } - if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil { - return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) - } - } else { - return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) - } - } - } - - output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput() - if err != nil { - klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) - return err - } - klog.V(2).Infof("mklink source(%q) on target(%q) successfully, output: %q", bindSource, target, string(output)) - - return nil -} - -// do the SMB mount with username, password, remotepath -// return (output, error) -func newSMBMapping(username, password, remotepath string) (string, error) { - if username == "" || password == "" || remotepath == "" { - return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath) - } - - // use PowerShell Environment Variables to store user input string to prevent command line injection - // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 - cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + - `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + - `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` - cmd := exec.Command("powershell", "/c", cmdLine) - cmd.Env = append(os.Environ(), - fmt.Sprintf("smbuser=%s", username), - fmt.Sprintf("smbpassword=%s", password), - fmt.Sprintf("smbremotepath=%s", remotepath)) - - output, err := cmd.CombinedOutput() - return string(output), err -} - -// check whether remotepath is already mounted -func isSMBMappingExist(remotepath string) bool { - cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) - cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) - _, err := cmd.CombinedOutput() - return err == nil -} - -// remove SMB mapping -func removeSMBMapping(remotepath string) (string, error) { - cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) - cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) - output, err := cmd.CombinedOutput() - return string(output), err -} - -// Unmount unmounts the target. -func (mounter *Mounter) Unmount(target string) error { - klog.V(4).Infof("azureMount: Unmount target (%q)", target) - target = NormalizeWindowsPath(target) - if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { - klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) - return err - } - return nil -} - -// List returns a list of all mounted filesystems. todo -func (mounter *Mounter) List() ([]MountPoint, error) { - return []MountPoint{}, nil -} - -// IsLikelyNotMountPoint determines if a directory is not a mountpoint. -func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - stat, err := os.Lstat(file) - if err != nil { - return true, err - } - - if stat.Mode()&os.ModeSymlink != 0 { - return false, err - } - return true, nil -} - -// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows -func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - windowsPath := NormalizeWindowsPath(pathname) - pathExists, pathErr := PathExists(windowsPath) - if !pathExists { - return []string{}, nil - } else if IsCorruptedMnt(pathErr) { - klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath) - return []string{}, nil - } else if pathErr != nil { - return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr) - } - return []string{pathname}, nil -} - -func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - // Try to mount the disk - klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) - - if err := ValidateDiskNumber(source); err != nil { - klog.Errorf("diskMount: formatAndMount failed, err: %v", err) - return err - } - - if len(fstype) == 0 { - // Use 'NTFS' as the default - fstype = "NTFS" - } - - // format disk if it is unformatted(raw) - cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ - " | New-Partition -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) - if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil { - return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) - } - klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) - - volumeIds, err := listVolumesOnDisk(source) - if err != nil { - return err - } - driverPath := volumeIds[0] - target = NormalizeWindowsPath(target) - output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput() - if err != nil { - klog.Errorf("mklink(%s, %s) failed: %v, output: %q", target, driverPath, err, string(output)) - return err - } - klog.V(2).Infof("formatAndMount disk(%s) fstype(%s) on(%s) with output(%s) successfully", driverPath, fstype, target, string(output)) - return nil -} - -// ListVolumesOnDisk - returns back list of volumes(volumeIDs) in the disk (requested in diskID). -func listVolumesOnDisk(diskID string) (volumeIDs []string, err error) { - cmd := fmt.Sprintf("(Get-Disk -DeviceId %s | Get-Partition | Get-Volume).UniqueId", diskID) - output, err := exec.Command("powershell", "/c", cmd).CombinedOutput() - klog.V(4).Infof("listVolumesOnDisk id from %s: %s", diskID, string(output)) - if err != nil { - return []string{}, fmt.Errorf("error list volumes on disk. cmd: %s, output: %s, error: %v", cmd, string(output), err) - } - - volumeIds := strings.Split(strings.TrimSpace(string(output)), "\r\n") - return volumeIds, nil -} - -// getAllParentLinks walks all symbolic links and return all the parent targets recursively -func getAllParentLinks(path string) ([]string, error) { - const maxIter = 255 - links := []string{} - for { - links = append(links, path) - if len(links) > maxIter { - return links, fmt.Errorf("unexpected length of parent links: %v", links) - } - - fi, err := os.Lstat(path) - if err != nil { - return links, fmt.Errorf("Lstat: %v", err) - } - if fi.Mode()&os.ModeSymlink == 0 { - break - } - - path, err = os.Readlink(path) - if err != nil { - return links, fmt.Errorf("Readlink error: %v", err) - } - } - - return links, nil -} From 3e8a2c3c30f338ce0b60fc867ba8c7899c334e6e Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 21:12:51 +0000 Subject: [PATCH 3/7] Add MountSensitiveWithFlags to the subpath Mounter interface - Also adding an implementation for all the platforms --- pkg/volume/util/subpath/subpath_linux.go | 4 +- pkg/volume/util/subpath/subpath_mount.go | 3 + .../util/subpath/subpath_mount_linux.go | 181 +++++++++++++++++- .../util/subpath/subpath_mount_unsupported.go | 7 +- .../util/subpath/subpath_mount_windows.go | 6 + 5 files changed, 195 insertions(+), 6 deletions(-) diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index b03800d2fc7..b2f50c20409 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -44,11 +44,11 @@ const ( ) type subpath struct { - mounter mount.Interface + mounter MountInterface } // New returns a subpath.Interface for the current system -func New(mounter mount.Interface) Interface { +func New(mounter MountInterface) Interface { return &subpath{ mounter: mounter, } diff --git a/pkg/volume/util/subpath/subpath_mount.go b/pkg/volume/util/subpath/subpath_mount.go index 76a230b97d9..a8ad8c228a1 100644 --- a/pkg/volume/util/subpath/subpath_mount.go +++ b/pkg/volume/util/subpath/subpath_mount.go @@ -26,6 +26,9 @@ import ( // MountInterface defines the set of methods to allow for mount operations on a system. type MountInterface interface { mount.Interface + + // MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags + MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error } // Compile-time check to ensure all Mounter implementations satisfy diff --git a/pkg/volume/util/subpath/subpath_mount_linux.go b/pkg/volume/util/subpath/subpath_mount_linux.go index 01cfe2c1a05..492b8d7e62f 100644 --- a/pkg/volume/util/subpath/subpath_mount_linux.go +++ b/pkg/volume/util/subpath/subpath_mount_linux.go @@ -19,23 +19,198 @@ limitations under the License. package subpath import ( - "k8s.io/utils/mount" + "fmt" + "os/exec" + "strings" + + "k8s.io/klog/v2" + mountutils "k8s.io/utils/mount" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" + // Log message where sensitive mount options were removed + sensitiveOptionsRemoved = "" ) // Mounter provides the subpath implementation of mount.Interface // for the linux platform. This implementation assumes that the // kubelet is running in the host's root mount namespace. type Mounter struct { - mount.Interface + mountutils.Interface mounterPath string + withSystemd bool } // NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. -func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { +func NewMounter(mounter mountutils.Interface, mounterPath string) MountInterface { return &Mounter{ Interface: mounter, mounterPath: mounterPath, + withSystemd: detectSystemd(), } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := mountutils.MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, mountFlags) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, mountFlags) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, mountFlags) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// mountFlags are additional flags used in the mount command that are not related with fstype and mount options +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + mountArgs, mountArgsLogStr := makeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, mountFlags) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountArgsLogStr = mountCmd + " " + mountArgsLogStr + mountCmd = mounterPath + } + + if mounter.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs, mountArgsLogStr = mountutils.AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) + // } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + // Logging with sensitive mount options removed. + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, mountArgsLogStr, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// makeMountArgsSensitiveWithMountFlags makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// mountFlags are additional mount flags that are not related with the fstype and mount options +func makeMountArgsSensitiveWithMountFlags(source, target, fstype string, options []string, sensitiveOptions []string, mountFlags []string) (mountArgs []string, mountArgsLogStr string) { + // Build mount command as follows: + // mount [$mountFlags] [-t $fstype] [-o $options] [$source] $target + mountArgs = []string{} + mountArgsLogStr = "" + + mountArgs = append(mountArgs, mountFlags...) + mountArgsLogStr += strings.Join(mountFlags, " ") + + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + mountArgsLogStr += strings.Join(mountArgs, " ") + } + if len(options) > 0 || len(sensitiveOptions) > 0 { + combinedOptions := []string{} + combinedOptions = append(combinedOptions, options...) + combinedOptions = append(combinedOptions, sensitiveOptions...) + mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) + // exclude sensitiveOptions from log string + mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + mountArgsLogStr += " " + source + } + mountArgs = append(mountArgs, target) + mountArgsLogStr += " " + target + + return mountArgs, mountArgsLogStr +} + +// sanitizedOptionsForLogging will return a comma separated string containing +// options and sensitiveOptions. Each entry in sensitiveOptions will be +// replaced with the string sensitiveOptionsRemoved +// e.g. o1,o2,, +func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { + separator := "" + if len(options) > 0 && len(sensitiveOptions) > 0 { + separator = "," + } + + sensitiveOptionsStart := "" + sensitiveOptionsEnd := "" + if len(sensitiveOptions) > 0 { + sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) + sensitiveOptionsEnd = sensitiveOptionsRemoved + } + + return strings.Join(options, ",") + + separator + + sensitiveOptionsStart + + sensitiveOptionsEnd +} diff --git a/pkg/volume/util/subpath/subpath_mount_unsupported.go b/pkg/volume/util/subpath/subpath_mount_unsupported.go index d15d619597d..16dcd053396 100644 --- a/pkg/volume/util/subpath/subpath_mount_unsupported.go +++ b/pkg/volume/util/subpath/subpath_mount_unsupported.go @@ -30,7 +30,7 @@ type Mounter struct { mounterPath string } -var errUnsupported = errors.New("utils/mount on this platform is not supported") +var errUtilsMountUnsupported = errors.New("utils/mount on this platform is not supported") // NewMounter returns a MountInterface for the current system. // It provides options to override the default mounter behavior. @@ -41,3 +41,8 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { mounterPath: mounterPath, } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + return errUtilsMountUnsupported +} diff --git a/pkg/volume/util/subpath/subpath_mount_windows.go b/pkg/volume/util/subpath/subpath_mount_windows.go index a78eef0a287..b8feefc8c6a 100644 --- a/pkg/volume/util/subpath/subpath_mount_windows.go +++ b/pkg/volume/util/subpath/subpath_mount_windows.go @@ -39,3 +39,9 @@ func NewMounter(mounter mount.Interface, mounterPath string) MountInterface { mounterPath: mounterPath, } } + +// MountSensitiveWithFlags is the same as MountSensitive() with additional mount flags but +// because mountFlags are linux mount(8) flags this method is the same as MountSensitive() in Windows +func (mounter *Mounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + return mounter.MountSensitive(source, target, fstype, options, sensitiveOptions) +} From 513640908935794d3a512724b395be8fcc21839c Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 21:17:46 +0000 Subject: [PATCH 4/7] Duplicate vendor/k8s.io/utils/mount/fake_mounter.go history in pkg/volume/util/subpath/subpath_fake_mounter.go history. --- .../util/subpath/subpath_fake_mounter.go | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 pkg/volume/util/subpath/subpath_fake_mounter.go diff --git a/pkg/volume/util/subpath/subpath_fake_mounter.go b/pkg/volume/util/subpath/subpath_fake_mounter.go new file mode 100644 index 00000000000..f48c2badba6 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_fake_mounter.go @@ -0,0 +1,216 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "os" + "path/filepath" + "sync" + + "k8s.io/klog/v2" +) + +// FakeMounter implements mount.Interface for tests. +type FakeMounter struct { + MountPoints []MountPoint + log []FakeAction + // Error to return for a path when calling IsLikelyNotMountPoint + MountCheckErrors map[string]error + // Some tests run things in parallel, make sure the mounter does not produce + // any golang's DATA RACE warnings. + mutex sync.Mutex + UnmountFunc UnmountFunc +} + +// UnmountFunc is a function callback to be executed during the Unmount() call. +type UnmountFunc func(path string) error + +var _ Interface = &FakeMounter{} + +const ( + // FakeActionMount is the string for specifying mount as FakeAction.Action + FakeActionMount = "mount" + // FakeActionUnmount is the string for specifying unmount as FakeAction.Action + FakeActionUnmount = "unmount" +) + +// FakeAction objects are logged every time a fake mount or unmount is called. +type FakeAction struct { + Action string // "mount" or "unmount" + Target string // applies to both mount and unmount actions + Source string // applies only to "mount" actions + FSType string // applies only to "mount" actions +} + +// NewFakeMounter returns a FakeMounter struct that implements Interface and is +// suitable for testing purposes. +func NewFakeMounter(mps []MountPoint) *FakeMounter { + return &FakeMounter{ + MountPoints: mps, + } +} + +// ResetLog clears all the log entries in FakeMounter +func (f *FakeMounter) ResetLog() { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.log = []FakeAction{} +} + +// GetLog returns the slice of FakeActions taken by the mounter +func (f *FakeMounter) GetLog() []FakeAction { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.log +} + +// Mount records the mount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Mount(source string, target string, fstype string, options []string) error { + return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// Mount records the mount event and updates the in-memory mount points for FakeMounter +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (f *FakeMounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + opts := []string{} + + for _, option := range options { + // find 'bind' option + if option == "bind" { + // This is a bind-mount. In order to mimic linux behaviour, we must + // use the original device of the bind-mount as the real source. + // E.g. when mounted /dev/sda like this: + // $ mount /dev/sda /mnt/test + // $ mount -o bind /mnt/test /mnt/bound + // then /proc/mount contains: + // /dev/sda /mnt/test + // /dev/sda /mnt/bound + // (and not /mnt/test /mnt/bound) + // I.e. we must use /dev/sda as source instead of /mnt/test in the + // bind mount. + for _, mnt := range f.MountPoints { + if source == mnt.Path { + source = mnt.Device + break + } + } + } + // reuse MountPoint.Opts field to mark mount as readonly + opts = append(opts, option) + } + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: append(opts, sensitiveOptions...)}) + klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) + f.log = append(f.log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) + return nil +} + +// Unmount records the unmount event and updates the in-memory mount points for FakeMounter +func (f *FakeMounter) Unmount(target string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + // If target is a symlink, get its absolute path + absTarget, err := filepath.EvalSymlinks(target) + if err != nil { + absTarget = target + } + + newMountpoints := []MountPoint{} + for _, mp := range f.MountPoints { + if mp.Path == absTarget { + if f.UnmountFunc != nil { + err := f.UnmountFunc(absTarget) + if err != nil { + return err + } + } + klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) + // Don't copy it to newMountpoints + continue + } + newMountpoints = append(newMountpoints, MountPoint{Device: mp.Device, Path: mp.Path, Type: mp.Type}) + } + f.MountPoints = newMountpoints + f.log = append(f.log, FakeAction{Action: FakeActionUnmount, Target: absTarget}) + delete(f.MountCheckErrors, target) + return nil +} + +// List returns all the in-memory mountpoints for FakeMounter +func (f *FakeMounter) List() ([]MountPoint, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + return f.MountPoints, nil +} + +// IsLikelyNotMountPoint determines whether a path is a mountpoint by checking +// if the absolute path to file is in the in-memory mountpoints +func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + err := f.MountCheckErrors[file] + if err != nil { + return false, err + } + + _, err = os.Stat(file) + if err != nil { + return true, err + } + + // If file is a symlink, get its absolute path + absFile, err := filepath.EvalSymlinks(file) + if err != nil { + absFile = file + } + + for _, mp := range f.MountPoints { + if mp.Path == absFile { + klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) + return false, nil + } + } + klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) + return true, nil +} + +// GetMountRefs finds all mount references to the path, returns a +// list of paths. +func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + // Ignore error in FakeMounter, because we actually didn't create files. + realpath = pathname + } + return getMountRefsByDev(f, realpath) +} From ce5e9f6c8322802ed777a250f1c943c0192ac776 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 21:27:40 +0000 Subject: [PATCH 5/7] Create a FakeMounter for the subpath that wraps the mountutils FakeMounter --- .../util/subpath/subpath_fake_mounter.go | 195 ++---------------- pkg/volume/util/subpath/subpath_linux_test.go | 6 +- 2 files changed, 16 insertions(+), 185 deletions(-) diff --git a/pkg/volume/util/subpath/subpath_fake_mounter.go b/pkg/volume/util/subpath/subpath_fake_mounter.go index f48c2badba6..cf54209f5ff 100644 --- a/pkg/volume/util/subpath/subpath_fake_mounter.go +++ b/pkg/volume/util/subpath/subpath_fake_mounter.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,203 +14,34 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mount +package subpath import ( - "os" - "path/filepath" - "sync" - - "k8s.io/klog/v2" + mountutils "k8s.io/utils/mount" ) -// FakeMounter implements mount.Interface for tests. +// FakeMounter implements MountInterface for tests. type FakeMounter struct { - MountPoints []MountPoint - log []FakeAction - // Error to return for a path when calling IsLikelyNotMountPoint - MountCheckErrors map[string]error - // Some tests run things in parallel, make sure the mounter does not produce - // any golang's DATA RACE warnings. - mutex sync.Mutex - UnmountFunc UnmountFunc + *mountutils.FakeMounter } -// UnmountFunc is a function callback to be executed during the Unmount() call. -type UnmountFunc func(path string) error - -var _ Interface = &FakeMounter{} - -const ( - // FakeActionMount is the string for specifying mount as FakeAction.Action - FakeActionMount = "mount" - // FakeActionUnmount is the string for specifying unmount as FakeAction.Action - FakeActionUnmount = "unmount" -) - -// FakeAction objects are logged every time a fake mount or unmount is called. -type FakeAction struct { - Action string // "mount" or "unmount" - Target string // applies to both mount and unmount actions - Source string // applies only to "mount" actions - FSType string // applies only to "mount" actions -} +var _ MountInterface = &FakeMounter{} // NewFakeMounter returns a FakeMounter struct that implements Interface and is // suitable for testing purposes. -func NewFakeMounter(mps []MountPoint) *FakeMounter { +func NewFakeMounter(mps []mountutils.MountPoint) *FakeMounter { return &FakeMounter{ - MountPoints: mps, + FakeMounter: &mountutils.FakeMounter{ + MountPoints: mps, + }, } } -// ResetLog clears all the log entries in FakeMounter -func (f *FakeMounter) ResetLog() { - f.mutex.Lock() - defer f.mutex.Unlock() - - f.log = []FakeAction{} -} - -// GetLog returns the slice of FakeActions taken by the mounter -func (f *FakeMounter) GetLog() []FakeAction { - f.mutex.Lock() - defer f.mutex.Unlock() - - return f.log -} - -// Mount records the mount event and updates the in-memory mount points for FakeMounter -func (f *FakeMounter) Mount(source string, target string, fstype string, options []string) error { - return f.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) -} - -// Mount records the mount event and updates the in-memory mount points for FakeMounter +// MountSensitiveWithFlags records the mount event and updates the in-memory mount points for FakeMounter // sensitiveOptions to be passed in a separate parameter from the normal // mount options and ensures the sensitiveOptions are never logged. This // method should be used by callers that pass sensitive material (like // passwords) as mount options. -func (f *FakeMounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { - f.mutex.Lock() - defer f.mutex.Unlock() - - opts := []string{} - - for _, option := range options { - // find 'bind' option - if option == "bind" { - // This is a bind-mount. In order to mimic linux behaviour, we must - // use the original device of the bind-mount as the real source. - // E.g. when mounted /dev/sda like this: - // $ mount /dev/sda /mnt/test - // $ mount -o bind /mnt/test /mnt/bound - // then /proc/mount contains: - // /dev/sda /mnt/test - // /dev/sda /mnt/bound - // (and not /mnt/test /mnt/bound) - // I.e. we must use /dev/sda as source instead of /mnt/test in the - // bind mount. - for _, mnt := range f.MountPoints { - if source == mnt.Path { - source = mnt.Device - break - } - } - } - // reuse MountPoint.Opts field to mark mount as readonly - opts = append(opts, option) - } - - // If target is a symlink, get its absolute path - absTarget, err := filepath.EvalSymlinks(target) - if err != nil { - absTarget = target - } - f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: append(opts, sensitiveOptions...)}) - klog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget) - f.log = append(f.log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype}) - return nil -} - -// Unmount records the unmount event and updates the in-memory mount points for FakeMounter -func (f *FakeMounter) Unmount(target string) error { - f.mutex.Lock() - defer f.mutex.Unlock() - - // If target is a symlink, get its absolute path - absTarget, err := filepath.EvalSymlinks(target) - if err != nil { - absTarget = target - } - - newMountpoints := []MountPoint{} - for _, mp := range f.MountPoints { - if mp.Path == absTarget { - if f.UnmountFunc != nil { - err := f.UnmountFunc(absTarget) - if err != nil { - return err - } - } - klog.V(5).Infof("Fake mounter: unmounted %s from %s", mp.Device, absTarget) - // Don't copy it to newMountpoints - continue - } - newMountpoints = append(newMountpoints, MountPoint{Device: mp.Device, Path: mp.Path, Type: mp.Type}) - } - f.MountPoints = newMountpoints - f.log = append(f.log, FakeAction{Action: FakeActionUnmount, Target: absTarget}) - delete(f.MountCheckErrors, target) - return nil -} - -// List returns all the in-memory mountpoints for FakeMounter -func (f *FakeMounter) List() ([]MountPoint, error) { - f.mutex.Lock() - defer f.mutex.Unlock() - - return f.MountPoints, nil -} - -// IsLikelyNotMountPoint determines whether a path is a mountpoint by checking -// if the absolute path to file is in the in-memory mountpoints -func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { - f.mutex.Lock() - defer f.mutex.Unlock() - - err := f.MountCheckErrors[file] - if err != nil { - return false, err - } - - _, err = os.Stat(file) - if err != nil { - return true, err - } - - // If file is a symlink, get its absolute path - absFile, err := filepath.EvalSymlinks(file) - if err != nil { - absFile = file - } - - for _, mp := range f.MountPoints { - if mp.Path == absFile { - klog.V(5).Infof("isLikelyNotMountPoint for %s: mounted %s, false", file, mp.Path) - return false, nil - } - } - klog.V(5).Infof("isLikelyNotMountPoint for %s: true", file) - return true, nil -} - -// GetMountRefs finds all mount references to the path, returns a -// list of paths. -func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { - realpath, err := filepath.EvalSymlinks(pathname) - if err != nil { - // Ignore error in FakeMounter, because we actually didn't create files. - realpath = pathname - } - return getMountRefsByDev(f, realpath) +func (f *FakeMounter) MountSensitiveWithFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + return f.MountSensitive(source, target, fstype, options, sensitiveOptions) } diff --git a/pkg/volume/util/subpath/subpath_linux_test.go b/pkg/volume/util/subpath/subpath_linux_test.go index f57bea0a540..e07bb54d720 100644 --- a/pkg/volume/util/subpath/subpath_linux_test.go +++ b/pkg/volume/util/subpath/subpath_linux_test.go @@ -611,7 +611,7 @@ func TestCleanSubPaths(t *testing.T) { t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) } - fm := mount.NewFakeMounter(mounts) + fm := NewFakeMounter(mounts) fm.UnmountFunc = test.unmount err = doCleanSubPaths(fm, base, testVol) @@ -636,12 +636,12 @@ var ( testSubpath = 1 ) -func setupFakeMounter(testMounts []string) *mount.FakeMounter { +func setupFakeMounter(testMounts []string) FakeMounter { mounts := []mount.MountPoint{} for _, mountPoint := range testMounts { mounts = append(mounts, mount.MountPoint{Device: "/foo", Path: mountPoint}) } - return mount.NewFakeMounter(mounts) + return NewFakeMounter(mounts) } func getTestPaths(base string) (string, string) { From 2a03546bf063adbef0e0eea09cd5a05de8298f26 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 21:32:04 +0000 Subject: [PATCH 6/7] Pass additional flags to subpath mount to avoid flakes in certain conditions --- pkg/volume/util/subpath/subpath_linux.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index b2f50c20409..c9e054be41f 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -209,8 +209,9 @@ func doBindSubPath(mounter MountInterface, subpath Subpath) (hostPath string, er // Do the bind mount options := []string{"bind"} + mountFlags := []string{"--no-canonicalize"} klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) - if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { + if err = mounter.MountSensitiveWithFlags(mountSource, bindPathTarget, "" /*fstype*/, options, nil /* sensitiveOptions */, mountFlags); err != nil { return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) } success = true From 2bfa2d7f2d1af7a4aa8ddb1fab41944da0c328ea Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 13 Aug 2021 22:59:09 +0000 Subject: [PATCH 7/7] Add subpath mount files to BUILD --- pkg/volume/util/subpath/BUILD | 24 +++++++------------ pkg/volume/util/subpath/subpath_linux_test.go | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/pkg/volume/util/subpath/BUILD b/pkg/volume/util/subpath/BUILD index ff6b29374e0..a44d3de3ab3 100644 --- a/pkg/volume/util/subpath/BUILD +++ b/pkg/volume/util/subpath/BUILD @@ -4,76 +4,68 @@ go_library( name = "go_default_library", srcs = [ "subpath.go", + "subpath_fake_mounter.go", "subpath_linux.go", + "subpath_mount.go", + "subpath_mount_linux.go", + "subpath_mount_unsupported.go", + "subpath_mount_windows.go", "subpath_unsupported.go", "subpath_windows.go", ], importpath = "k8s.io/kubernetes/pkg/volume/util/subpath", visibility = ["//visibility:public"], - deps = select({ + deps = [ + "//vendor/k8s.io/utils/mount:go_default_library", + ] + select({ "@io_bazel_rules_go//go/platform:aix": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:android": [ "//pkg/volume/util/hostutil:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/mount:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:illumos": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:ios": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:js": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ "//pkg/volume/util/hostutil:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/mount:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/mount:go_default_library", "//vendor/k8s.io/utils/nsenter:go_default_library", ], "//conditions:default": [], diff --git a/pkg/volume/util/subpath/subpath_linux_test.go b/pkg/volume/util/subpath/subpath_linux_test.go index e07bb54d720..65443506d4b 100644 --- a/pkg/volume/util/subpath/subpath_linux_test.go +++ b/pkg/volume/util/subpath/subpath_linux_test.go @@ -636,7 +636,7 @@ var ( testSubpath = 1 ) -func setupFakeMounter(testMounts []string) FakeMounter { +func setupFakeMounter(testMounts []string) *FakeMounter { mounts := []mount.MountPoint{} for _, mountPoint := range testMounts { mounts = append(mounts, mount.MountPoint{Device: "/foo", Path: mountPoint})