Skip to content

Commit f4b6a74

Browse files
Pearl1594Pearl1594
authored
Add support for CSI driver in CKS (#11419)
* Support creation of PV(persistent volumes) in CloudStack projects * add support for snapshot APIs for project role * Add support to setup csi driver on k8s cluster creation * fix deploy script * update response * fix table name * fix linter * show if csi driver is setup in cluster * delete pvs whose reclaim policy is delete when cluster is destroyed * update ref * move changes to 4.22 * fix variables * fix eof
1 parent 046014b commit f4b6a74

File tree

18 files changed

+379
-3
lines changed

18 files changed

+379
-3
lines changed

‎api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,4 +172,5 @@ enum State {
172172
Long getEtcdNodeCount();
173173
Long getCniConfigId();
174174
String getCniConfigDetails();
175+
boolean isCsiEnabled();
175176
}

‎api/src/main/java/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ public class ApiConstants {
135135
public static final String CNI_CONFIG_ID = "cniconfigurationid";
136136
public static final String CNI_CONFIG_DETAILS = "cniconfigdetails";
137137
public static final String CNI_CONFIG_NAME = "cniconfigname";
138+
public static final String CSI_ENABLED = "csienabled";
138139
public static final String COMPONENT = "component";
139140
public static final String CPU = "CPU";
140141
public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket";
@@ -215,6 +216,7 @@ public class ApiConstants {
215216
public static final String DURATION = "duration";
216217
public static final String ELIGIBLE = "eligible";
217218
public static final String EMAIL = "email";
219+
public static final String ENABLE_CSI = "enablecsi";
218220
public static final String END_ASN = "endasn";
219221
public static final String END_DATE = "enddate";
220222
public static final String END_IP = "endip";

‎engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_inst
4848
UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%password%';
4949
UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%token%';
5050

51+
-- Add csi_enabled column to kubernetes_cluster table to indicate if the cluster is using csi or not
52+
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'csi_enabled', 'TINYINT(1) unsigned NOT NULL DEFAULT 0 COMMENT "true if kubernetes cluster is using csi, false otherwise" ');
53+
5154
-- VMware to KVM migration improvements
5255
CREATE TABLE IF NOT EXISTS `cloud`.`import_vm_task`(
5356
`id` bigint unsigned NOT NULL auto_increment COMMENT 'id',

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,16 @@
9696
import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
9797
import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
9898
import org.apache.cloudstack.api.command.user.network.ListNetworksCmd;
99+
import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd;
100+
import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd;
101+
import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd;
99102
import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
103+
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
104+
import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
105+
import org.apache.cloudstack.api.command.user.volume.DeleteVolumeCmd;
106+
import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd;
107+
import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd;
108+
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
100109
import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
101110
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
102111
import org.apache.cloudstack.api.response.KubernetesUserVmResponse;
@@ -252,7 +261,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
252261
private static final List<Class<?>> PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList(
253262
QueryAsyncJobResultCmd.class,
254263
ListVMsCmd.class,
264+
ListVolumesCmd.class,
265+
CreateVolumeCmd.class,
266+
DeleteVolumeCmd.class,
267+
AttachVolumeCmd.class,
268+
DetachVolumeCmd.class,
269+
ResizeVolumeCmd.class,
255270
ListNetworksCmd.class,
271+
CreateSnapshotCmd.class,
272+
ListSnapshotsCmd.class,
273+
DeleteSnapshotCmd.class,
256274
ListPublicIpAddressesCmd.class,
257275
AssociateIPAddrCmd.class,
258276
DisassociateIPAddrCmd.class,
@@ -880,6 +898,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes
880898
response.setMinSize(kubernetesCluster.getMinSize());
881899
response.setMaxSize(kubernetesCluster.getMaxSize());
882900
response.setClusterType(kubernetesCluster.getClusterType());
901+
response.setCsiEnabled(kubernetesCluster.isCsiEnabled());
883902
response.setCreated(kubernetesCluster.getCreated());
884903

885904
return response;
@@ -1605,6 +1624,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) {
16051624
if (zone.isSecurityGroupEnabled()) {
16061625
newCluster.setSecurityGroupId(finalSecurityGroup.getId());
16071626
}
1627+
newCluster.setCsiEnabled(cmd.getEnableCsi());
16081628
kubernetesClusterDao.persist(newCluster);
16091629
addKubernetesClusterDetails(newCluster, defaultNetwork, cmd);
16101630
return newCluster;

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ public class KubernetesClusterVO implements KubernetesCluster {
145145
@Column(name = "cni_config_details", updatable = true, length = 4096)
146146
private String cniConfigDetails;
147147

148+
@Column(name = "csi_enabled")
149+
private boolean csiEnabled;
150+
148151
@Override
149152
public long getId() {
150153
return id;
@@ -389,6 +392,14 @@ public void setClusterType(ClusterType clusterType) {
389392
this.clusterType = clusterType;
390393
}
391394

395+
public boolean isCsiEnabled() {
396+
return csiEnabled;
397+
}
398+
399+
public void setCsiEnabled(boolean csiEnabled) {
400+
this.csiEnabled = csiEnabled;
401+
}
402+
392403
public KubernetesClusterVO() {
393404
this.uuid = UUID.randomUUID().toString();
394405
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,17 @@ public class KubernetesClusterActionWorker {
232232

233233
protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
234234
protected final String deployProviderScriptFilename = "deploy-provider";
235+
protected final String deployCsiDriverScriptFilename = "deploy-csi-driver";
236+
protected final String deletePvScriptFilename = "delete-pv-reclaimpolicy-delete";
235237
protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
236238
protected final String validateNodeScript = "validate-cks-node";
237239
protected final String removeNodeFromClusterScript = "remove-node-from-cluster";
238240
protected final String scriptPath = "/opt/bin/";
239241
protected File deploySecretsScriptFile;
240242
protected File deployProviderScriptFile;
243+
protected File deployCsiDriverScriptFile;
241244
protected File autoscaleScriptFile;
245+
protected File deletePvScriptFile;
242246
protected KubernetesClusterManagerImpl manager;
243247
protected String[] keys;
244248

@@ -715,12 +719,16 @@ protected File retrieveScriptFile(String filename) {
715719
protected void retrieveScriptFiles() {
716720
deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
717721
deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename);
722+
deployCsiDriverScriptFile = retrieveScriptFile(deployCsiDriverScriptFilename);
723+
deletePvScriptFile = retrieveScriptFile(deletePvScriptFilename);
718724
autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
719725
}
720726

721727
protected void copyScripts(String nodeAddress, final int sshPort) {
722728
copyScriptFile(nodeAddress, sshPort, deploySecretsScriptFile, deploySecretsScriptFilename);
723729
copyScriptFile(nodeAddress, sshPort, deployProviderScriptFile, deployProviderScriptFilename);
730+
copyScriptFile(nodeAddress, sshPort, deployCsiDriverScriptFile, deployCsiDriverScriptFilename);
731+
copyScriptFile(nodeAddress, sshPort, deletePvScriptFile, deletePvScriptFilename);
724732
copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename);
725733
}
726734

@@ -821,6 +829,43 @@ protected boolean deployProvider() {
821829
}
822830
}
823831

832+
protected boolean deployCsiDriver() {
833+
File pkFile = getManagementServerSshPublicKeyFile();
834+
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
835+
publicIpAddress = publicIpSshPort.first();
836+
sshPort = publicIpSshPort.second();
837+
838+
try {
839+
String command = String.format("sudo %s/%s", scriptPath, deployCsiDriverScriptFilename);
840+
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
841+
pkFile, null, command, 10000, 10000, 60000);
842+
843+
// Maybe the file isn't present. Try and copy it
844+
if (!result.first()) {
845+
logMessage(Level.INFO, "CSI files missing. Adding them now", null);
846+
retrieveScriptFiles();
847+
copyScripts(publicIpAddress, sshPort);
848+
849+
if (!createCloudStackSecret(keys)) {
850+
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
851+
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
852+
}
853+
854+
// If at first you don't succeed ...
855+
result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
856+
pkFile, null, command, 10000, 10000, 60000);
857+
if (!result.first()) {
858+
throw new CloudRuntimeException(result.second());
859+
}
860+
}
861+
return true;
862+
} catch (Exception e) {
863+
String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage());
864+
logAndThrow(Level.ERROR, msg);
865+
return false;
866+
}
867+
}
868+
824869
public void setKeys(String[] keys) {
825870
this.keys = keys;
826871
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ private boolean destroyClusterVMs() {
106106
ApiCommandResourceType.VirtualMachine);
107107
vmContext.setEventResourceId(vmID);
108108
try {
109+
if (clusterVM.isControlNode() && kubernetesCluster.isCsiEnabled()) {
110+
deletePVsWithReclaimPolicyDelete();
111+
}
109112
UserVm vm = userVmService.destroyVm(vmID, true);
110113
if (!userVmManager.expunge(userVM)) {
111114
logger.warn("Unable to expunge VM {}, destroying Kubernetes cluster will probably fail", vm);

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,4 +944,47 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) {
944944
protected List<DedicatedResourceVO> listDedicatedHostsInDomain(Long domainId) {
945945
return dedicatedResourceDao.listByDomainId(domainId);
946946
}
947+
948+
public boolean deletePVsWithReclaimPolicyDelete() {
949+
File pkFile = getManagementServerSshPublicKeyFile();
950+
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
951+
publicIpAddress = publicIpSshPort.first();
952+
sshPort = publicIpSshPort.second();
953+
try {
954+
String command = String.format("sudo %s/%s", scriptPath, deletePvScriptFilename);
955+
logMessage(Level.INFO, "Starting PV deletion script for cluster: " + kubernetesCluster.getName(), null);
956+
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
957+
pkFile, null, command, 10000, 10000, 600000); // 10 minute timeout
958+
if (Boolean.FALSE.equals(result.first())) {
959+
logMessage(Level.INFO, "PV delete script missing. Adding it now", null);
960+
retrieveScriptFiles();
961+
if (deletePvScriptFile != null) {
962+
copyScriptFile(publicIpAddress, sshPort, deletePvScriptFile, deletePvScriptFilename);
963+
logMessage(Level.INFO, "Executing PV deletion script (this may take several minutes)...", null);
964+
result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
965+
pkFile, null, command, 10000, 10000, 600000); // 10 minute timeout
966+
if (Boolean.FALSE.equals(result.first())) {
967+
logMessage(Level.ERROR, "PV deletion script failed: " + result.second(), null);
968+
throw new CloudRuntimeException(result.second());
969+
}
970+
logMessage(Level.INFO, "PV deletion script completed successfully", null);
971+
} else {
972+
logMessage(Level.WARN, "PV delete script file not found in resources, skipping PV deletion", null);
973+
return false;
974+
}
975+
} else {
976+
logMessage(Level.INFO, "PV deletion script completed successfully", null);
977+
}
978+
979+
if (result.second() != null && !result.second().trim().isEmpty()) {
980+
logMessage(Level.INFO, "PV deletion script output: " + result.second(), null);
981+
}
982+
983+
return true;
984+
} catch (Exception e) {
985+
String msg = String.format("Failed to delete PVs with reclaimPolicy=Delete: %s : %s", kubernetesCluster.getName(), e.getMessage());
986+
logMessage(Level.WARN, msg, e);
987+
return false;
988+
}
989+
}
947990
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ private boolean isKubernetesVersionSupportsHA() {
143143

144144
private Pair<String, String> getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
145145
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
146-
final boolean ejectIso, final boolean externalCni) throws IOException {
146+
final boolean ejectIso, final boolean externalCni, final boolean setupCsi) throws IOException {
147147
String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml");
148148
final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
149149
final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
@@ -161,6 +161,7 @@ private Pair<String, String> getKubernetesControlNodeConfig(final String control
161161
final String certSans = "{{ k8s_control.server_ips }}";
162162
final String k8sCertificate = "{{ k8s_control.certificate_key }}";
163163
final String externalCniPlugin = "{{ k8s.external.cni.plugin }}";
164+
final String setupCsiDriver = "{{ k8s.setup.csi.driver }}";
164165

165166
final List<String> addresses = new ArrayList<>();
166167
addresses.add(controlNodeIp);
@@ -212,6 +213,7 @@ private Pair<String, String> getKubernetesControlNodeConfig(final String control
212213
k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp));
213214
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
214215
k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni));
216+
k8sControlNodeConfig = k8sControlNodeConfig.replace(setupCsiDriver, String.valueOf(setupCsi));
215217

216218
k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
217219

@@ -246,7 +248,7 @@ private Pair<UserVm,String> createKubernetesControlNode(final Network network, S
246248
Long userDataId = kubernetesCluster.getCniConfigId();
247249
Pair<String, String> k8sControlNodeConfigAndControlIp = new Pair<>(null, null);
248250
try {
249-
k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId));
251+
k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId), kubernetesCluster.isCsiEnabled());
250252
} catch (IOException e) {
251253
logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
252254
}
@@ -858,6 +860,9 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon
858860
}
859861
taintControlNodes();
860862
deployProvider();
863+
if (kubernetesCluster.isCsiEnabled()) {
864+
deployCsiDriver();
865+
}
861866
updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList()));
862867
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
863868
return true;

‎plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,9 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
207207
since = "4.21.0")
208208
private Map cniConfigDetails;
209209

210+
@Parameter(name = ApiConstants.ENABLE_CSI, type = CommandType.BOOLEAN, description = "if true, setups up CloudStack CSI driver", since = "4.22.0")
211+
private Boolean enableCsi;
212+
210213
@Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, description="the AS Number of the network")
211214
private Long asNumber;
212215

@@ -371,6 +374,10 @@ public Long getCniConfigId() {
371374
return cniConfigId;
372375
}
373376

377+
public boolean getEnableCsi() {
378+
return Objects.nonNull(enableCsi) ? enableCsi : Boolean.FALSE;
379+
}
380+
374381
/////////////////////////////////////////////////////
375382
/////////////// API Implementation///////////////////
376383
/////////////////////////////////////////////////////

0 commit comments

Comments
 (0)