diff --git a/changelogs/unreleased/335-mhkarimi1383 b/changelogs/unreleased/335-mhkarimi1383 new file mode 100644 index 00000000..e29f464b --- /dev/null +++ b/changelogs/unreleased/335-mhkarimi1383 @@ -0,0 +1,3 @@ +extra format options (mkfs) added + +used to add extra options to mkfs while creating new volumes diff --git a/deploy/lvm-operator.yaml b/deploy/lvm-operator.yaml index a873aaa5..b51f2c74 100644 --- a/deploy/lvm-operator.yaml +++ b/deploy/lvm-operator.yaml @@ -95,6 +95,12 @@ spec: description: Capacity of the volume minLength: 1 type: string + formatOptions: + description: FormatOptions specifies additional options while formatting + filesystem + items: + type: string + type: array ownerNodeID: description: OwnerNodeID is the Node ID where the volume group is present which is where the volume has been provisioned. OwnerNodeID diff --git a/deploy/yamls/lvmvolume-crd.yaml b/deploy/yamls/lvmvolume-crd.yaml index 61b78125..faced51a 100644 --- a/deploy/yamls/lvmvolume-crd.yaml +++ b/deploy/yamls/lvmvolume-crd.yaml @@ -74,6 +74,12 @@ spec: description: Capacity of the volume minLength: 1 type: string + formatOptions: + description: FormatOptions specifies additional options while formatting + filesystem + items: + type: string + type: array ownerNodeID: description: OwnerNodeID is the Node ID where the volume group is present which is where the volume has been provisioned. OwnerNodeID diff --git a/go.mod b/go.mod index 7c6bc94a..394c90a6 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -77,6 +78,7 @@ require ( k8s.io/apiextensions-apiserver v0.27.2 // indirect k8s.io/gengo v0.0.0-20230306165830-ab3349d207d4 // indirect k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 // indirect + k8s.io/mount-utils v0.26.15 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/testing_frameworks v0.1.2 // indirect diff --git a/go.sum b/go.sum index 37055dc1..3ec24541 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -225,6 +226,7 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -233,6 +235,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -260,6 +263,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -272,6 +277,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -829,6 +835,8 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 h1:OmK1d0WrkD3IPfkskvroRykOulHVHf0s0ZIFRjyt+UI= k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515/go.mod h1:kzo02I3kQ4BTtEfVLaPbjvCkX97YqGve33wzlb3fofQ= +k8s.io/mount-utils v0.26.15 h1:TvTNwRNiXRlxjb7ZUlKObpfINRmcFqMF2i+rb3iJ/co= +k8s.io/mount-utils v0.26.15/go.mod h1:huSg2NI5P8ZNfE8PkQmm5a9fFZ9iHCXFxP/rasMCgYA= k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go b/pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go index d8d09823..751226d3 100644 --- a/pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go +++ b/pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go @@ -87,6 +87,10 @@ type VolumeInfo struct { // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=yes;no ThinProvision string `json:"thinProvision,omitempty"` + + // FormatOptions specifies additional options while formatting filesystem + // +kubebuilder:validation:Optional + FormatOptions []string `json:"formatOptions,omitempty"` } // VolStatus string that specifies the current state of the volume provisioning request. diff --git a/pkg/builder/volbuilder/volume.go b/pkg/builder/volbuilder/volume.go index c871c104..9adbc4f1 100644 --- a/pkg/builder/volbuilder/volume.go +++ b/pkg/builder/volbuilder/volume.go @@ -169,6 +169,12 @@ func (b *Builder) WithVgPattern(pattern string) *Builder { return b } +// WithFormatOptions sets volume format options +func (b *Builder) WithFormatOptions(formatOptions []string) *Builder { + b.volume.Object.Spec.FormatOptions = formatOptions + return b +} + // WithNodeName sets NodeID for creating the volume func (b *Builder) WithNodeName(name string) *Builder { if name == "" { diff --git a/pkg/driver/agent.go b/pkg/driver/agent.go index 01cf3e9a..424cb91a 100644 --- a/pkg/driver/agent.go +++ b/pkg/driver/agent.go @@ -215,6 +215,8 @@ func GetVolAndMountInfo( return nil, nil, err } + mountinfo.FormatOptions = vol.Spec.FormatOptions + return vol, &mountinfo, nil } diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index e0708321..9f563398 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -301,6 +301,7 @@ func CreateLVMVolume(ctx context.Context, req *csi.CreateVolumeRequest, WithOwnerNode(owner). WithVolumeStatus(lvm.LVMStatusPending). WithShared(params.Shared). + WithFormatOptions(params.FormatOptions). WithThinProvision(params.ThinProvision).Build() if err != nil { diff --git a/pkg/driver/params.go b/pkg/driver/params.go index 6e24fd78..9d420abe 100644 --- a/pkg/driver/params.go +++ b/pkg/driver/params.go @@ -39,9 +39,10 @@ type VolumeParams struct { // extra optional metadata passed by external provisioner // if enabled. See --extra-create-metadata flag for more details. // https://github.com/kubernetes-csi/external-provisioner#recommended-optional-arguments - PVCName string - PVCNamespace string - PVName string + PVCName string + PVCNamespace string + PVName string + FormatOptions []string } // SnapshotParams holds collection of supported settings that can @@ -73,6 +74,8 @@ func NewVolumeParams(m map[string]string) (*VolumeParams, error) { vgPattern = fmt.Sprintf("^%v$", volGroup) } + params.FormatOptions = strings.Split(m["formatoptions"], " ") + var err error if params.VgPattern, err = regexp.Compile(vgPattern); err != nil { return nil, fmt.Errorf("invalid volgroup/vgpattern param %v: %v", vgPattern, err) diff --git a/pkg/lvm/mount.go b/pkg/lvm/mount.go index 5f8874cc..36721b1e 100644 --- a/pkg/lvm/mount.go +++ b/pkg/lvm/mount.go @@ -28,8 +28,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" + "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" - "k8s.io/utils/mount" apis "github.com/openebs/lvm-localpv/pkg/apis/openebs.io/lvm/v1alpha1" ) @@ -53,6 +53,10 @@ type MountInfo struct { // MountOptions specifies the options with // which mount needs to be attempted MountOptions []string `json:"mountOptions"` + + // FormatOptions specifies the options that + // will be added to newly created volume on first use + FormatOptions []string `json:"formatOptions"` } // PodLVInfo contains the pod, LVGroup related info @@ -68,7 +72,14 @@ type PodLVInfo struct { func FormatAndMountVol(devicePath string, mountInfo *MountInfo) error { mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()} - err := mounter.FormatAndMount(devicePath, mountInfo.MountPath, mountInfo.FSType, mountInfo.MountOptions) + err := mounter.FormatAndMountSensitiveWithFormatOptions( + devicePath, + mountInfo.MountPath, + mountInfo.FSType, + mountInfo.MountOptions, + nil, // sensitiveOptions + mountInfo.FormatOptions, + ) if err != nil { klog.Errorf( "lvm: failed to mount volume %s [%s] to %s, error %v", diff --git a/tests/provision_test.go b/tests/provision_test.go index 8723027c..d04de8a9 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -86,6 +86,24 @@ func fsVolCreationTest() { } } +func formatOptionsTest() { + formatOptions := "-b 4096 -N 5000000" + By("####### Creating the storage class with formatOptions : " + formatOptions + " #######") + createFormatOptionsStorageClass(formatOptions) + By("Creating and verifying PVC bound status") + createAndVerifyPVC(true) + By("Creating and deploying app pod", createDeployVerifyFormatOptions) + By("Verifying LVMVolume object to be Ready") + VerifyLVMVolume(true, "") + By("Verifing LVM Volume format options") + VerifyLVMVolumeFormatOptions(formatOptions) + By("Deleting verifier pvc/pod") + deleteAppAndPvc([]string{"format-options-verifier"}, pvcName) + By("Verifying that PV is deleted after deletion") + verifyPVForPVC(false, pvcName) + By("Deleting storage class", deleteStorageClass) +} + func blockVolCreationTest() { By("Creating default storage class", createStorageClass) By("Creating and verifying PVC bound status") @@ -321,6 +339,7 @@ func volumeCreationTest() { device := setupVg(40, "lvmvg") defer cleanupVg(device, "lvmvg") By("###Running filesystem volume creation test###", fsVolCreationTest) + By("###Running filesystem with formatOptions creation test###", formatOptionsTest) By("###Running block volume creation test###", blockVolCreationTest) By("###Running thin volume creation test###", thinVolCreationTest) By("###Running leak protection test###", leakProtectionTest) diff --git a/tests/utils.go b/tests/utils.go index a624caf8..4ba12846 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -233,6 +234,29 @@ func createThinStorageClass() { gomega.Expect(err).To(gomega.BeNil(), "while creating a thinProvision storageclass {%s}", scName) } +func createFormatOptionsStorageClass(formatOptions string) { + var ( + err error + ) + + parameters := map[string]string{ + "volgroup": VOLGROUP, + "formatOptions": formatOptions, + } + + ginkgo.By("building a default storage class") + scObj, err = sc.NewBuilder(). + WithGenerateName(scName). + WithVolumeExpansion(true). + WithParametersNew(parameters). + WithProvisioner(LocalProvisioner).Build() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), + "while building formatOptions storageclass obj with prefix {%s}", scName) + + scObj, err = SCClient.Create(scObj) + gomega.Expect(err).To(gomega.BeNil(), "while creating a storageclass with formatOptions {%s}", scName) +} + // VerifyLVMVolume verify the properties of a lvm-volume // expected_vg is supposed to be passed only when vgpatten was used for scheduling. // If its volgroup in sc then we can just match volgroup with lvmvolume's vg field. @@ -271,6 +295,18 @@ func VerifyLVMVolume(expect_ready bool, expected_vg string) { } } +func VerifyLVMVolumeFormatOptions(formatOptions string) { + ginkgo.By("fetching lvm volume") + vol_name := pvcObj.Spec.VolumeName + vol, err := LVMClient.WithNamespace(OpenEBSNamespace). + Get(vol_name, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.BeNil(), "while fetching the lvm volume {%s}", pvcObj.Spec.VolumeName) + gomega.Expect(vol).To(gomega.Not(gomega.BeNil()), "LVMVolume Obj is Nil") + gomega.Expect(vol.ObjectMeta.Name).To(gomega.Not(gomega.Equal(vol_name)), "LVMVolume mismatch") + gomega.Expect(vol.Spec.FormatOptions).To(gomega.Equal(strings.Split(formatOptions, " ")), + "While checking if lvmvolume: %s has correct formatOptions", pvcObj.Spec.VolumeName) +} + func deleteStorageClass() { err := SCClient.Delete(scObj.Name, &metav1.DeleteOptions{}) gomega.Expect(err).To(gomega.BeNil(), @@ -560,6 +596,77 @@ func createAndDeployBlockAppPod() { } } +func createDeployVerifyFormatOptions() { + ginkgo.By("creating and deploying verifier pod") + createAndDeployVerifyFormatOptions() + ginkgo.By("verifying verifier pod is running", verifyAppPodRunning) +} + +func createAndDeployVerifyFormatOptions() { + var err error + appname := "format-options-verifier" + labels := map[string]string{ + "role": "test", + "app": appname, + } + ginkgo.By("building app " + appname + " using above lvm volume") + deployObj, err = deploy.NewBuilder(). + WithName(appname). + WithNamespace(OpenEBSNamespace). + WithLabelsNew(labels). + WithSelectorMatchLabelsNew(labels). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew(labels). + WithContainerBuilders( + container.NewBuilder(). + WithImage("debian:stable-slim"). + WithName("verifier"). + WithImagePullPolicy(corev1.PullIfNotPresent). + WithEnvsNew( + []corev1.EnvVar{ + { + Name: "EXPECTED_INODES", + Value: "5001264", + }, + }, + ). + WithCommandNew( + []string{ + "bash", + "-c", + `export DEVICE=$(df --output=source '/mnt/datadir' | tail -n 1); export ACTUAL_INODES=$(tune2fs -l "$DEVICE" | grep "Inode count" | awk '{print $3}'); [ "$ACTUAL_INODES" -eq "$EXPECTED_INODES" ] && echo "Inode count is correct: $ACTUAL_INODES" && exit 0 || echo "Inode count mismatch: expected $EXPECTED_INODES, but got $ACTUAL_INODES" && exit 1`, + }, + ). + WithVolumeMountsNew( + []corev1.VolumeMount{ + { + Name: "datavol1", + // If this path changes, modify the above command line accordingly (df command). + MountPath: "/mnt/datadir", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("datavol1"). + WithPVCSource(pvcObj.Name), + ), + ). + Build() + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while building app deployement {%s}", appname) + + deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) + gomega.Expect(err).ShouldNot( + gomega.HaveOccurred(), + "while creating pod {%s} in namespace {%s}", + appname, + OpenEBSNamespace, + ) +} + func createDeployVerifyBlockApp() { ginkgo.By("creating and deploying app pod", createAndDeployBlockAppPod) ginkgo.By("verifying app pod is running", verifyAppPodRunning)