From 2632bdd5e0507ee156fefe443643125b63416531 Mon Sep 17 00:00:00 2001 From: Wei Wei Date: Mon, 15 May 2017 12:18:56 +0800 Subject: [PATCH 1/3] storageclass ceph add imageformat parameter --- pkg/volume/rbd/rbd.go | 15 +++++++++++++-- pkg/volume/rbd/rbd_util.go | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 3163cb8b597..d11b93ba551 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -51,8 +51,10 @@ var _ volume.DeletableVolumePlugin = &rbdPlugin{} var _ volume.ProvisionableVolumePlugin = &rbdPlugin{} const ( - rbdPluginName = "kubernetes.io/rbd" - secretKeyName = "key" // key name used in secret + rbdPluginName = "kubernetes.io/rbd" + secretKeyName = "key" // key name used in secret + rbdImageFormat1 = "1" + rbdImageFormat2 = "2" ) func (plugin *rbdPlugin) Init(host volume.VolumeHost) error { @@ -267,6 +269,7 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { adminSecretNamespace := "default" secretName := "" secret := "" + imageFormat := "" for k, v := range r.options.Parameters { switch dstrings.ToLower(k) { @@ -287,11 +290,18 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { r.Pool = v case "usersecretname": secretName = v + case "imageformat": + imageFormat = v default: return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName()) } } // sanity check + if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 { + return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s", + imageFormat, rbdImageFormat1, rbdImageFormat2) + } + r.imageFormat = imageFormat if adminSecretName == "" { return nil, fmt.Errorf("missing Ceph admin secret name") } @@ -384,6 +394,7 @@ type rbdMounter struct { adminSecret string adminId string mountOptions []string + imageFormat string } var _ volume.Mounter = &rbdMounter{} diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index a78ccd3ac97..b868103670a 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -357,7 +357,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource mon := p.Mon[i%l] glog.V(4).Infof("rbd: create %s size %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) output, err = p.rbdMounter.plugin.execCommand("rbd", - []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", "1"}) + []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}) if err == nil { break } else { From 0c7cdf773b7def770b28188d39333a51589fef40 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Fri, 16 Jun 2017 12:11:35 +0000 Subject: [PATCH 2/3] Add `imageFeatures` parameter for RBD volume plugin, which is used to customize RBD image format 2 features. Update RBD docs in examples/persistent-volume-provisioning/README.md. --- .../persistent-volume-provisioning/README.md | 5 ++++ pkg/volume/rbd/rbd.go | 26 ++++++++++++------- pkg/volume/rbd/rbd_util.go | 15 ++++++++--- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/examples/persistent-volume-provisioning/README.md b/examples/persistent-volume-provisioning/README.md index e664ae067d5..ada79f51cba 100644 --- a/examples/persistent-volume-provisioning/README.md +++ b/examples/persistent-volume-provisioning/README.md @@ -209,6 +209,7 @@ parameters: pool: kube userId: kube userSecretName: ceph-secret-user + imageFormat: "1" ``` * `monitors`: Ceph monitors, comma delimited. It is required. @@ -218,6 +219,10 @@ parameters: * `pool`: Ceph RBD pool. Default is "rbd". * `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`. * `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required. +* `imageFormat`: Ceph RBD image format, "1" or "2". Default is "1". +* `imageFeatures`: Ceph RBD image format 2 features, comma delimited. This is optional, and only be used if you set `imageFormat` to "2". For a complete list of available image features, please refer to [RBD docs](http://docs.ceph.com/docs/master/man/8/rbd/). By default, all features (except for striping) will be enabled. + +NOTE: If you want to use RBD image format 2, you may need to disable some image features, because some of them are not supported by [Ceph kernel module](https://github.com/ceph/ceph-client). For example, only "layering" feature is supported on Linux 4.9 LTS version, so you need to specify `imageFeatures` to "layering", if set `imageFormat` to "2". #### Quobyte diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index d11b93ba551..25f5bd76c74 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -269,7 +269,7 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { adminSecretNamespace := "default" secretName := "" secret := "" - imageFormat := "" + imageFormat := rbdImageFormat1 for k, v := range r.options.Parameters { switch dstrings.ToLower(k) { @@ -292,6 +292,11 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { secretName = v case "imageformat": imageFormat = v + case "imagefeatures": + arr := dstrings.Split(v, ",") + for _, f := range arr { + r.imageFeatures = append(r.imageFeatures, f) + } default: return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName()) } @@ -386,15 +391,16 @@ func (rbd *rbd) GetPath() string { type rbdMounter struct { *rbd // capitalized so they can be exported in persistRBD() - Mon []string - Id string - Keyring string - Secret string - fsType string - adminSecret string - adminId string - mountOptions []string - imageFormat string + Mon []string + Id string + Keyring string + Secret string + fsType string + adminSecret string + adminId string + mountOptions []string + imageFormat string + imageFeatures []string } var _ volume.Mounter = &rbdMounter{} diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index b868103670a..9a63c8e64bd 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -355,9 +355,18 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource // iterate all monitors until create succeeds. for i := start; i < start+l; i++ { mon := p.Mon[i%l] - glog.V(4).Infof("rbd: create %s size %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) - output, err = p.rbdMounter.plugin.execCommand("rbd", - []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}) + if p.rbdMounter.imageFormat == rbdImageFormat2 { + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) + } else { + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) + } + args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat} + if p.rbdMounter.imageFormat == rbdImageFormat2 && len(p.rbdMounter.imageFeatures) > 0 { + for _, f := range p.rbdMounter.imageFeatures { + args = append(args, "--image-feature", f) + } + } + output, err = p.rbdMounter.plugin.execCommand("rbd", args) if err == nil { break } else { From 4e5e9ca0cde963a4b607ca04f0d961659163c156 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Fri, 16 Jun 2017 16:29:58 +0000 Subject: [PATCH 3/3] Only `layering` RBD image format 2 feature should be supported for now. --- examples/persistent-volume-provisioning/README.md | 4 ++-- pkg/volume/rbd/BUILD | 1 + pkg/volume/rbd/rbd.go | 11 ++++++++++- pkg/volume/rbd/rbd_util.go | 9 +++++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/examples/persistent-volume-provisioning/README.md b/examples/persistent-volume-provisioning/README.md index ada79f51cba..4d3f00af648 100644 --- a/examples/persistent-volume-provisioning/README.md +++ b/examples/persistent-volume-provisioning/README.md @@ -220,9 +220,9 @@ parameters: * `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`. * `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required. * `imageFormat`: Ceph RBD image format, "1" or "2". Default is "1". -* `imageFeatures`: Ceph RBD image format 2 features, comma delimited. This is optional, and only be used if you set `imageFormat` to "2". For a complete list of available image features, please refer to [RBD docs](http://docs.ceph.com/docs/master/man/8/rbd/). By default, all features (except for striping) will be enabled. +* `imageFeatures`: Ceph RBD image format 2 features, comma delimited. This is optional, and only be used if you set `imageFormat` to "2". Currently supported features are `layering` only. Default is "", no features is turned on. -NOTE: If you want to use RBD image format 2, you may need to disable some image features, because some of them are not supported by [Ceph kernel module](https://github.com/ceph/ceph-client). For example, only "layering" feature is supported on Linux 4.9 LTS version, so you need to specify `imageFeatures` to "layering", if set `imageFormat` to "2". +NOTE: We cannot turn on `exclusive-lock` feature for now (and `object-map`, `fast-diff`, `journaling` which require `exclusive-lock`), because exclusive lock and advisory lock cannot work together. (See [#45805](https://issue.k8s.io/45805)) #### Quobyte diff --git a/pkg/volume/rbd/BUILD b/pkg/volume/rbd/BUILD index e79c795b5b3..ac71d2268d1 100644 --- a/pkg/volume/rbd/BUILD +++ b/pkg/volume/rbd/BUILD @@ -31,6 +31,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", ], ) diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 25f5bd76c74..9cc7d522da7 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" @@ -35,6 +36,10 @@ import ( "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) +var ( + supportedFeatures = sets.NewString("layering") +) + // This is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { return []volume.VolumePlugin{&rbdPlugin{nil, exec.New()}} @@ -295,7 +300,11 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { case "imagefeatures": arr := dstrings.Split(v, ",") for _, f := range arr { - r.imageFeatures = append(r.imageFeatures, f) + if !supportedFeatures.Has(f) { + return nil, fmt.Errorf("invalid feature %q for volume plugin %s, supported features are: %v", f, r.plugin.GetPluginName(), supportedFeatures) + } else { + r.imageFeatures = append(r.imageFeatures, f) + } } default: return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName()) diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index 9a63c8e64bd..b567d764ea8 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -361,10 +361,11 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) } args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat} - if p.rbdMounter.imageFormat == rbdImageFormat2 && len(p.rbdMounter.imageFeatures) > 0 { - for _, f := range p.rbdMounter.imageFeatures { - args = append(args, "--image-feature", f) - } + if p.rbdMounter.imageFormat == rbdImageFormat2 { + // if no image features is provided, it results in empty string + // which disable all RBD image format 2 features as we expected + features := strings.Join(p.rbdMounter.imageFeatures, ",") + args = append(args, "--image-feature", features) } output, err = p.rbdMounter.plugin.execCommand("rbd", args) if err == nil {