diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 8b5551d6a8e9..83457a2aef58 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -166,4 +166,5 @@ enum State { Long getEtcdNodeCount(); Long getCniConfigId(); String getCniConfigDetails(); + boolean isCsiEnabled(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 489d737b5bb9..e571903a33e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -211,6 +211,7 @@ public class ApiConstants { public static final String DURATION = "duration"; public static final String ELIGIBLE = "eligible"; public static final String EMAIL = "email"; + public static final String ENABLE_CSI = "enablecsi"; public static final String END_ASN = "endasn"; public static final String END_DATE = "enddate"; public static final String END_IP = "endip"; @@ -324,6 +325,7 @@ public class ApiConstants { public static final String IP_TOTAL = "iptotal"; public static final String IS_CONTROL_NODE = "iscontrolnode"; public static final String IS_CLEANUP_REQUIRED = "iscleanuprequired"; + public static final String IS_CSI_ENABLED = "iscsienabled"; public static final String IS_DYNAMIC = "isdynamic"; public static final String IS_EDGE = "isedge"; public static final String IS_ENCRYPTED = "isencrypted"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 167dd92730cc..eb9152bfff6a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -757,3 +757,6 @@ SET `cs`.`domain_id` = ( -- Re-apply VPC: update default network offering for vpc tier to conserve_mode=1 (#8309) UPDATE `cloud`.`network_offerings` SET conserve_mode = 1 WHERE name = 'DefaultIsolatedNetworkOfferingForVpcNetworks'; + +-- Add csi_enabled column to kubernetes_cluster table - Move to 4.22 +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'csi_enabled', 'TINYINT(1) unsigned NOT NULL DEFAULT 0 COMMENT "true if kubernetes cluster is using csi, false otherwise" '); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index dc092608ff3a..1cb20ac192ce 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -96,7 +96,16 @@ import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; +import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; +import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.KubernetesUserVmResponse; @@ -252,7 +261,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private static final List> PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList( QueryAsyncJobResultCmd.class, ListVMsCmd.class, + ListVolumesCmd.class, + CreateVolumeCmd.class, + DeleteVolumeCmd.class, + AttachVolumeCmd.class, + DetachVolumeCmd.class, + ResizeVolumeCmd.class, ListNetworksCmd.class, + CreateSnapshotCmd.class, + ListSnapshotsCmd.class, + DeleteSnapshotCmd.class, ListPublicIpAddressesCmd.class, AssociateIPAddrCmd.class, DisassociateIPAddrCmd.class, @@ -873,6 +891,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setMinSize(kubernetesCluster.getMinSize()); response.setMaxSize(kubernetesCluster.getMaxSize()); response.setClusterType(kubernetesCluster.getClusterType()); + response.setCsiEnabled(kubernetesCluster.isCsiEnabled()); response.setCreated(kubernetesCluster.getCreated()); return response; @@ -1598,6 +1617,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (zone.isSecurityGroupEnabled()) { newCluster.setSecurityGroupId(finalSecurityGroup.getId()); } + newCluster.setCsiEnabled(cmd.getEnableCsi()); kubernetesClusterDao.persist(newCluster); addKubernetesClusterDetails(newCluster, defaultNetwork, cmd); return newCluster; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 79fc15f68981..7dfd0043e320 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -145,6 +145,9 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "cni_config_details", updatable = true, length = 4096) private String cniConfigDetails; + @Column(name = "csi_enabled") + private boolean csiEnabled; + @Override public long getId() { return id; @@ -389,6 +392,14 @@ public void setClusterType(ClusterType clusterType) { this.clusterType = clusterType; } + public boolean isCsiEnabled() { + return csiEnabled; + } + + public void setCsiEnabled(boolean csiEnabled) { + this.csiEnabled = csiEnabled; + } + public KubernetesClusterVO() { this.uuid = UUID.randomUUID().toString(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 9be5f4c07768..45cba3b6fc75 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -232,12 +232,14 @@ public class KubernetesClusterActionWorker { protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; protected final String deployProviderScriptFilename = "deploy-provider"; + protected final String deployCsiDriverScriptFilename = "deploy-csi-driver"; protected final String autoscaleScriptFilename = "autoscale-kube-cluster"; protected final String validateNodeScript = "validate-cks-node"; protected final String removeNodeFromClusterScript = "remove-node-from-cluster"; protected final String scriptPath = "/opt/bin/"; protected File deploySecretsScriptFile; protected File deployProviderScriptFile; + protected File deployCsiDriverScriptFile; protected File autoscaleScriptFile; protected KubernetesClusterManagerImpl manager; protected String[] keys; @@ -715,12 +717,14 @@ protected File retrieveScriptFile(String filename) { protected void retrieveScriptFiles() { deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename); deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename); + deployCsiDriverScriptFile = retrieveScriptFile(deployCsiDriverScriptFilename); autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename); } protected void copyScripts(String nodeAddress, final int sshPort) { copyScriptFile(nodeAddress, sshPort, deploySecretsScriptFile, deploySecretsScriptFilename); copyScriptFile(nodeAddress, sshPort, deployProviderScriptFile, deployProviderScriptFilename); + copyScriptFile(nodeAddress, sshPort, deployCsiDriverScriptFile, deployCsiDriverScriptFilename); copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename); } @@ -821,6 +825,43 @@ protected boolean deployProvider() { } } + protected boolean deployCsiDriver() { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + + try { + String command = String.format("sudo %s/%s", scriptPath, deployCsiDriverScriptFilename); + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 60000); + + // Maybe the file isn't present. Try and copy it + if (!result.first()) { + logMessage(Level.INFO, "CSI files missing. Adding them now", null); + retrieveScriptFiles(); + copyScripts(publicIpAddress, sshPort); + + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + // If at first you don't succeed ... + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 60000); + if (!result.first()) { + throw new CloudRuntimeException(result.second()); + } + } + return true; + } catch (Exception e) { + String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage()); + logAndThrow(Level.ERROR, msg); + return false; + } + } + public void setKeys(String[] keys) { this.keys = keys; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 9ffee220a10f..2db252af27ff 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -143,7 +143,7 @@ private boolean isKubernetesVersionSupportsHA() { private Pair getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, final List etcdIps, final String hostName, final boolean haSupported, - final boolean ejectIso, final boolean externalCni) throws IOException { + final boolean ejectIso, final boolean externalCni, final boolean setupCsi) throws IOException { String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; @@ -161,6 +161,7 @@ private Pair getKubernetesControlNodeConfig(final String control final String certSans = "{{ k8s_control.server_ips }}"; final String k8sCertificate = "{{ k8s_control.certificate_key }}"; final String externalCniPlugin = "{{ k8s.external.cni.plugin }}"; + final String setupCsiDriver = "{{ k8s.setup.csi.driver }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); @@ -212,6 +213,7 @@ private Pair getKubernetesControlNodeConfig(final String control k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(setupCsiDriver, String.valueOf(setupCsi)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); @@ -246,7 +248,7 @@ private Pair createKubernetesControlNode(final Network network, S Long userDataId = kubernetesCluster.getCniConfigId(); Pair k8sControlNodeConfigAndControlIp = new Pair<>(null, null); try { - k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId)); + k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId), kubernetesCluster.isCsiEnabled()); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); } @@ -858,6 +860,9 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon } taintControlNodes(); deployProvider(); + if (kubernetesCluster.isCsiEnabled()) { + deployCsiDriver(); + } updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 4e92f9546f83..64d725797aab 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -207,6 +207,9 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { since = "4.21.0") private Map cniConfigDetails; + @Parameter(name = ApiConstants.ENABLE_CSI, type = CommandType.BOOLEAN, description = "if true, setups up CloudStack CSI driver", since = "4.21.0") + private Boolean enableCsi; + @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, description="the AS Number of the network") private Long asNumber; @@ -371,6 +374,10 @@ public Long getCniConfigId() { return cniConfigId; } + public boolean getEnableCsi() { + return Objects.nonNull(enableCsi) ? enableCsi : Boolean.FALSE; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index b811f4f9dcbd..b03d3b38411a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -204,6 +204,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "Maximum size of the cluster") private Long maxSize; + @SerializedName(ApiConstants.IS_CSI_ENABLED) + @Param(description = "Indicates if the CloudStack CSI driver has been setup in the cluster") + private Boolean isCsiEnabled; + @SerializedName(ApiConstants.CLUSTER_TYPE) @Param(description = "the type of the cluster") private KubernetesCluster.ClusterType clusterType; @@ -515,4 +519,8 @@ public void setCniConfigId(String cniConfigId) { public void setCniConfigName(String cniConfigName) { this.cniConfigName = cniConfigName; } + + public void setCsiEnabled(Boolean csiEnabled) { + isCsiEnabled = csiEnabled; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index dc066e10d06b..70291dd1c35a 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -179,6 +179,11 @@ write_files: mkdir -p /opt/provider cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml fi + if [ -e "${BINARIES_DIR}/snapshot-crds.yaml" ]; then + mkdir -p /opt/csi + cp "${BINARIES_DIR}/snapshot-crds.yaml" /opt/csi/snapshot-crds.yaml + cp "${BINARIES_DIR}/manifest.yaml" /opt/csi/manifest.yaml + fi PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` echo $PAUSE_IMAGE diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver new file mode 100644 index 000000000000..afb2e9f15f35 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver @@ -0,0 +1,50 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +(/opt/bin/kubectl get pods -A | grep cloudstack-csi-controller) && exit 0 + +if [ -e /opt/csi/snapshot-crds.yaml ]; then + /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml + sleep 5 + /opt/bin/kubectl apply -f /opt/csi/manifest.yaml + exit 0 +else + TARGET_DIR="/opt/csi" + mkdir -p "$TARGET_DIR" +# CSI_URLS=( +# "https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml" +# "https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml" +# ) + CSI_URLS=( + "http://10.0.3.130/cks/csi/snapshot-crds.yaml" + "http://10.0.3.130/cks/csi/manifest.yaml" + ) + for url in "${URLS[@]}"; do + filename=$(basename "$url") + + curl -sSL ${url} -o ${working_dir}/${filename} + if [ $? -ne 0 ]; then + echo "Unable to connect to the internet to download the relevant files to install and setup CloudStack CSI driver" + exit 1 + else + /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml + /opt/bin/kubectl apply -f /opt/csi/manifest.yaml + exit 0 + fi + done +fi diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index 58aec922b7f1..a12a995fa4e0 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -172,6 +172,13 @@ if [ -n "${9}" ]; then wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz fi +echo "Including CloudStack CSI Driver manifest" +# TODO: remove the below 2 lines and uncomment the ones below once csi-driver is officially released, this is for testing purposes only +wget http://10.0.3.130/cks/csi/snapshot-crds.yaml -O ${working_dir}/snapshot-crds.yaml +wget http://10.0.3.130/cks/csi/manifest.yaml -O ${working_dir}/manifest.yaml +#wget https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml -O ${working_dir}/snapshot-crds.yaml +#wget https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml -O ${working_dir}/manifest.yaml + mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}" rm -rf "${iso_dir}" diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 394de6ca6d26..8e8ef7d7edf4 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -961,6 +961,7 @@ "label.elastic": "Elastic", "label.email": "Email", "label.enable.autoscale.vmgroup": "Enable AutoScaling Group", +"label.enable.csi": "Enable CloudStack CSI Driver", "label.enable.custom.action": "Enable Custom Action", "label.enable.extension": "Enable Extension", "label.enable.host": "Enable Host", diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index f945da732138..68c7a35e3e5e 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -207,6 +207,12 @@ + + + +