[WIP ] Support for vSphere with Kubernetes (Tanzu)

charts/images.yaml

@@ -47,14 +47,19 @@ images:
   tag: v1.22.0-patch1
   #repository: gcr.io/cloud-provider-vsphere/cpi/release/manager
   #tag: v1.0.0
+- name: vsphere-with-k8s-cloud-controller-manager
+  sourceRepository: github.com/MartinWeindel/cloud-controller-manager-vsphere-with-k8s
+  repository: eu.gcr.io/gardener-project/test/cloud-controller-manager-vsphere-with-k8s
+  tag: 0.1.0-dev-mw20210907a
 - name: machine-controller-manager
   sourceRepository: github.com/gardener/machine-controller-manager
   repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager
   tag: "v0.40.0"
 - name: machine-controller-manager-provider-vsphere
   sourceRepository: github.com/gardener/machine-controller-manager-provider-vsphere
   repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager-provider-vsphere
-  tag: "v0.10.0"
+  # experimental support for vm-operator-api PR #14
+  tag: "v0.11.0-dev-vmop1"
 - name: vsphere-csi-driver-controller
   #sourceRepository: github.com/kubernetes-sigs/vsphere-csi-driver
   #repository: gcr.io/cloud-provider-vsphere/csi/release/driver

charts/internal/cloud-provider-config/templates/_cloud-provider-config.tpl

@@ -10,6 +10,7 @@ global:
   thumbprint: "{{ .Values.thumbprint }}"
   {{- end }}
 
+{{- if not .Values.vsphereWithKubernetes }}
 vcenter:
   {{ .Values.serverName }}:
     server: {{ .Values.serverName }}
@@ -21,6 +22,7 @@ vcenter:
     user: "{{ .Values.username }}"
     password: "{{ .Values.password }}"
     insecureFlag: {{ .Values.insecureFlag }}
+{{- end }}
 
 {{- if (or .Values.labelRegion .Values.labelZone) }}
 labels:
@@ -33,6 +35,9 @@ labels:
 {{- end }}
 
 loadBalancer:
+  {{- if .Values.vsphereWithKubernetes }}
+  enabled: true
+  {{- end }}
   {{- if .Values.loadbalancer.lbServiceId }}
   lbServiceId: "{{ .Values.loadbalancer.lbServiceId }}"
   {{- end }}
@@ -67,4 +72,21 @@ nsxt:
   host: "{{ .Values.nsxt.host }}"
   insecureFlag: {{ .Values.nsxt.insecureFlag }}
   remoteAuth: {{ .Values.nsxt.remoteAuth }}
+
+{{- if .Values.vsphereWithKubernetes }}
+supervisor:
+  token: {{ .Values.supervisor.token }}
+  namespace: {{ .Values.supervisor.namespace }}
+  apiserver: {{ .Values.supervisor.apiserver }}
+{{- if .Values.supervisor.caData }}
+  caData: {{ .Values.supervisor.caData }}
+{{- end }}
+{{- if .Values.supervisor.apiserverFQDN }}
+  apiserverFQDN: {{ .Values.supervisor.apiserverFQDN }}
+{{- end }}
+{{- if .Values.supervisor.insecure }}
+  insecure: {{ .Values.supervisor.insecure }}
+{{- end }}
+{{- end }}
+
 {{- end -}}

charts/internal/cloud-provider-config/values.yaml

@@ -37,3 +37,10 @@ nsxt:
   password: super-secret
   insecureFlag: true
   remoteAuth: false
+
+vsphereWithKubernetes: false
+
+#supervisor:
+#  token: foo
+#  namespace: foo
+#  apiserver: foo

charts/internal/machineclass/templates/machineclass.yaml

@@ -10,55 +10,77 @@ metadata:
 type: Opaque
 data:
   userData: {{ $machineClass.secret.cloudConfig | b64enc }}
+{{- if $.Values.vsphereWithKubernetes }}
+  vsphereKubeconfig: {{ $machineClass.secret.vsphereKubeconfig | b64enc }}
+{{- else }}
   vsphereHost: {{ $machineClass.secret.vsphereHost | b64enc }}
   vsphereUsername: {{ $machineClass.secret.vsphereUsername | b64enc }}
   vspherePassword: {{ $machineClass.secret.vspherePassword | b64enc }}
   vsphereInsecureSSL: {{ $machineClass.secret.vsphereInsecureSSL | b64enc }}
+{{- end }}
 ---
 apiVersion: machine.sapcloud.io/v1alpha1
 kind: MachineClass
 metadata:
   name: {{ $machineClass.name }}
   namespace: {{ $.Release.Namespace }}
 providerSpec:
-  region: {{ $machineClass.region }}
-  datacenter: {{ $machineClass.datacenter }}
+{{- if not $.Values.vsphereWithKubernetes }}
+  V1:
+    region: {{ $machineClass.region }}
+    datacenter: {{ $machineClass.datacenter }}
 {{- if $machineClass.hostSystem }}
-  hostSystem: {{ $machineClass.hostSystem }}
+    hostSystem: {{ $machineClass.hostSystem }}
 {{- else }}
 {{- if $machineClass.resourcePool }}
-  resourcePool: {{ $machineClass.resourcePool }}
+    resourcePool: {{ $machineClass.resourcePool }}
 {{- else }}
-  computeCluster: {{ $machineClass.computeCluster }}
+    computeCluster: {{ $machineClass.computeCluster }}
 {{- end }}
 {{- end }}
-  network: {{ $machineClass.network }}
+    network: {{ $machineClass.network }}
 {{- if $machineClass.switchUuid }}
-  switchUuid: {{ $machineClass.switchUuid }}
+    switchUuid: {{ $machineClass.switchUuid }}
 {{- end }}
 {{- if $machineClass.folder }}
-  folder: {{ $machineClass.folder }}
+    folder: {{ $machineClass.folder }}
 {{- end }}
 {{- if $machineClass.datastoreCluster }}
-  datastoreCluster: {{ $machineClass.datastoreCluster }}
+    datastoreCluster: {{ $machineClass.datastoreCluster }}
 {{- else }}
-  datastore: {{ $machineClass.datastore }}
+    datastore: {{ $machineClass.datastore }}
 {{- end }}
-  templateVM: {{ $machineClass.templateVM }}
+    templateVM: {{ $machineClass.templateVM }}
 {{- if $machineClass.guestId }}
-  guestId: {{ $machineClass.guestId }}
+    guestId: {{ $machineClass.guestId }}
 {{- end }}
-  numCpus: {{ $machineClass.numCpus }}
-  memory: {{ $machineClass.memory }}
+    numCpus: {{ $machineClass.numCpus }}
+    memory: {{ $machineClass.memory }}
 {{- if $machineClass.memoryReservationLockedToMax }}
-  memoryReservationLockedToMax: {{ $machineClass.memoryReservationLockedToMax }}
+    memoryReservationLockedToMax: {{ $machineClass.memoryReservationLockedToMax }}
 {{- end }}
 {{- if $machineClass.extraConfig }}
-  extraConfig:
-{{ toYaml $machineClass.extraConfig | indent 4 }}
+    extraConfig:
+{{ toYaml $machineClass.extraConfig | indent 6 }}
+{{- end }}
+    systemDisk:
+  {{ toYaml $machineClass.systemDisk | indent 4 }}
+{{- else }}
+  V2:
+    namespace: {{ $machineClass.namespace }}
+    imageName: {{ $machineClass.imageName }}
+    className: {{ $machineClass.className }}
+    networkType: {{ $machineClass.networkType }}
+    networkName: {{ $machineClass.networkName }}
+{{- if $machineClass.storageClass }}
+    storageClass: {{ $machineClass.storageClass }}
+{{- end}}
+{{- if $machineClass.resourcePolicyName }}
+    resourcePolicyName: {{ $machineClass.resourcePolicyName }}
+{{- end}}
+    systemDisk:
+  {{ toYaml $machineClass.systemDisk | indent 4 }}
 {{- end }}
-  systemDisk:
-  {{ toYaml $machineClass.systemDisk | indent 2 }}
   sshKeys:
 {{ toYaml $machineClass.sshKeys | indent 4 }}
 {{- if $machineClass.tags }}

charts/internal/machineclass/values.yaml

@@ -29,6 +29,17 @@ machineClasses:
     vsphereUsername: "a_user"
     vspherePassword: "a_password"
     vsphereInsecureSSL: "true"
+    # value for vsphereWithKubernetes == true
+    vsphereKubeconfig: ZHVtbXk=
   tags:
   - kubernetes.io/cluster/foo
   - kubernetes.io/role/node
+
+  # values for vsphereWithKubernetes == true
+  namespace: "shoot--foo--bar"
+  imageName: "gardenlinux-123.0"
+  className: "best-effort-medium"
+  networkType: "nsx-t"
+  networkName: "shoot--foo--bar"
+
+vsphereWithKubernetes: false

charts/internal/seed-controlplane/charts/csi-vsphere/templates/configmap-pvcsi-config.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.vsphereWithKubernetes }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vsphere-pvcsi-config
+  namespace: {{ .Release.Namespace }}
+data:
+  cns-csi.conf: |-
+    [GC]
+    endpoint = "supervisor.default.svc"
+    port = "{{ .Values.supervisor.port }}"
+    tanzukubernetescluster-uid = "{{ .Values.clusterID }}"
+    tanzukubernetescluster-name = "{{ .Values.clusterName }}"
+    {{- if .Values.clusterDistribution }}
+    cluster-distribution = "{{- .Values.clusterDistribution }}"
+    {{- end -}}
+{{- end }}

charts/internal/seed-controlplane/charts/csi-vsphere/templates/deployment-vsphere-csi-controller.yaml

@@ -37,6 +37,12 @@ spec:
         networking.gardener.cloud/to-shoot-apiserver: allowed
     spec:
       automountServiceAccountToken: false
+      {{- if .Values.vsphereWithKubernetes }}
+      hostAliases:
+        - ip: {{ .Values.supervisor.endpointIP }}
+          hostnames:
+            - "supervisor.default.svc"
+      {{- end }}
       containers:
         - name: csi-attacher
           image: {{ index .Values.images "csi-attacher" }}
@@ -80,6 +86,10 @@ spec:
         - name: vsphere-csi-controller
           image: {{ index .Values.images "vsphere-csi-driver-controller" }}
           args:
+            {{- if .Values.vsphereWithKubernetes }}
+            - "--supervisor-fss-name=internal-feature-states.csi.vsphere.vmware.com"
+            - "--supervisor-fss-namespace=$(CSI_NAMESPACE)"
+            {{- end }}
             - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
             - "--fss-namespace=$(CSI_NAMESPACE)"
             - "--kubeconfig=/var/lib/vsphere-csi-controller/kubeconfig"
@@ -88,8 +98,6 @@ spec:
               value: unix:///csi/csi.sock
             - name: X_CSI_MODE
               value: "controller"
-            - name: VSPHERE_CSI_CONFIG
-              value: "/etc/cloud/csi-vsphere.conf"
             - name: LOGGER_LEVEL
               value: "{{ .Values.loggerLevel }}" # Options: DEVELOPMENT, PRODUCTION
             - name: INCLUSTER_CLIENT_QPS
@@ -98,14 +106,36 @@ spec:
               value: "100"
             - name: CSI_NAMESPACE
               value: kube-system
+            {{- if .Values.vsphereWithKubernetes }}
+            - name: CLUSTER_FLAVOR
+              value: "GUEST_CLUSTER"
+            - name: SUPERVISOR_CLIENT_QPS
+              value: "50"
+            - name: SUPERVISOR_CLIENT_BURST
+              value: "50"
+            - name: GC_CONFIG
+              value: /etc/cloud/pvcsi-config/cns-csi.conf
+            {{- else }}
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            {{- end }}
 {{- if .Values.resources.controller }}
           resources:
 {{ toYaml .Values.resources.controller | indent 12 }}
 {{- end }}
           volumeMounts:
+            {{- if .Values.vsphereWithKubernetes }}
+            - mountPath: /etc/cloud/pvcsi-provider
+              name: pvcsi-provider-volume
+              readOnly: true
+            - mountPath: /etc/cloud/pvcsi-config
+              name: pvcsi-config-volume
+              readOnly: true
+            {{- else }}
             - mountPath: /etc/cloud
               name: vsphere-config-volume
               readOnly: true
+            {{- end }}
             - mountPath: /csi
               name: socket-dir
             - name: vsphere-csi-controller
@@ -132,11 +162,13 @@ spec:
             - "--kubeconfig=/var/lib/csi-syncer/kubeconfig"
             - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
             - "--fss-namespace=$(CSI_NAMESPACE)"
+            {{- if .Values.vsphereWithKubernetes }}
+            - "--supervisor-fss-name=csi-feature-states"
+            - "--supervisor-fss-namespace={{ .Values.supervisor.namespace }}"
+            {{- end }}
           env:
             - name: FULL_SYNC_INTERVAL_MINUTES
               value: "30"
-            - name: VSPHERE_CSI_CONFIG
-              value: "/etc/cloud/csi-vsphere.conf"
             - name: LOGGER_LEVEL
               value: "{{ .Values.loggerLevel }}" # Options: DEVELOPMENT, PRODUCTION
             - name: INCLUSTER_CLIENT_QPS
@@ -145,16 +177,38 @@ spec:
               value: "100"
             - name: CSI_NAMESPACE
               value: kube-system
+            {{- if .Values.vsphereWithKubernetes }}
+            - name: CLUSTER_FLAVOR
+              value: "GUEST_CLUSTER"
+            - name: SUPERVISOR_CLIENT_QPS
+              value: "50"
+            - name: SUPERVISOR_CLIENT_BURST
+              value: "50"
+            - name: GC_CONFIG
+              value: /etc/cloud/pvcsi-config/cns-csi.conf
+            {{- else }}
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            {{- end }}
 {{- if .Values.resources.syncer }}
           resources:
 {{ toYaml .Values.resources.syncer | indent 12 }}
 {{- end }}
           volumeMounts:
             - name: csi-syncer
               mountPath: /var/lib/csi-syncer
+            {{- if .Values.vsphereWithKubernetes }}
+            - mountPath: /etc/cloud/pvcsi-provider
+              name: pvcsi-provider-volume
+              readOnly: true
+            - mountPath: /etc/cloud/pvcsi-config
+              name: pvcsi-config-volume
+              readOnly: true
+            {{- else }}
             - mountPath: /etc/cloud
               name: vsphere-config-volume
               readOnly: true
+            {{- end }}
         - name: csi-liveness-probe
           image: {{ index .Values.images "liveness-probe" }}
           args:
@@ -175,8 +229,10 @@ spec:
             - "--csi-address=/csi/csi.sock"
             - "--kube-api-qps=100"
             - "--kube-api-burst=100"
+            {{- if not .Values.vsphereWithKubernetes }}
             - "--feature-gates=Topology=true"
             - "--strict-topology"
+            {{- end }}
             - "--leader-election"
             - "--kubeconfig=/var/lib/csi-provisioner/kubeconfig"
             - "--default-fstype=ext4"
@@ -231,8 +287,17 @@ spec:
           secret:
             secretName: csi-snapshotter
 {{- end }}
+{{- if .Values.vsphereWithKubernetes }}
+        - name: pvcsi-provider-volume
+          secret:
+            secretName: vsphere-pvcsi-provider-creds
+        - name: pvcsi-config-volume
+          configMap:
+            name: vsphere-pvcsi-config
+{{- else }}
         - name: vsphere-config-volume
           secret:
             secretName: csi-vsphere-config
+{{- end }}
         - name: socket-dir
           emptyDir: {}

charts/internal/seed-controlplane/charts/csi-vsphere/templates/secret-csi-vsphere-config.yaml

@@ -1,8 +1,10 @@
+{{- if not .Values.vsphereWithKubernetes }}
 apiVersion: v1
 kind: Secret
 metadata:
   name: csi-vsphere-config
   namespace: {{ .Release.Namespace }}
 type: Opaque
 data:
-  csi-vsphere.conf: {{ include "csi-vsphere-conf" . | b64enc }}
+  csi-vsphere.conf: {{ include "csi-vsphere-conf" . | b64enc }}
+{{- end }}

charts/internal/seed-controlplane/charts/csi-vsphere/templates/secret-pvcsi-provider-creds.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.vsphereWithKubernetes }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vsphere-pvcsi-provider-creds
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  namespace: {{ b64enc .Values.supervisor.namespace }}
+  token: {{ b64enc .Values.supervisor.token }}
+  ca.crt: {{ b64enc .Values.supervisor.caData }}
+{{- end }}