From ebb6c0271ef3744b7febf5b1bcdd7a425b8f765e Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@apple.com>
Date: Tue, 8 Apr 2025 16:58:37 -0700
Subject: [PATCH 01/14] Cumulative enhancements fix for ScaleIO: MDM
 add/remove, Host prepare/unprepare, validate Storage Pool can be created in
 Agent.

- Implemented validation to fail Host disconnect from Storage Pool if there are Volumes attached and SDC client MDM removal requires scini service to be restarted
- Implemented Storage Pool validation by checking whether MDM addresses from configuration file and from memory (using CLI) matches, otherwise file ModifyStoragePool command.
- Introduced configuration key to apply timeout after making MDM changes for ScaleIO: powerflex.mdm.change.apply.timeout.ms (default 1000ms)
- Implemented logic to apply timeout after making MDM changes for ScaleIO in prepare and unprepare logic
- Added detection of MDM removal support via CLI
- If MDM removal support via CLI supported then use CLI, fall back to edit drv_cfg.txt and restart scini instead
---
 ...ibvirtModifyStoragePoolCommandWrapper.java |  16 +-
 .../kvm/storage/ScaleIOStorageAdaptor.java    |  98 +++++-
 .../storage/ScaleIOStorageAdaptorTest.java    |   7 +-
 .../kvm/storage/ScaleIOStoragePoolTest.java   |   6 +-
 .../client/ScaleIOGatewayClient.java          |   3 +
 .../ScaleIOPrimaryDataStoreLifeCycle.java     |   4 +
 .../datastore/manager/ScaleIOSDCManager.java  |  18 +
 .../manager/ScaleIOSDCManagerImpl.java        |   7 +-
 .../provider/ScaleIOHostListener.java         |   4 +
 .../storage/datastore/util/ScaleIOUtil.java   | 313 +++++++++++++++---
 .../java/com/cloud/utils/script/Script.java   |  19 ++
 11 files changed, 436 insertions(+), 59 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
index 06883c708d96..990cefda8f33 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
@@ -31,6 +31,7 @@
 import com.cloud.resource.CommandWrapper;
 import com.cloud.resource.ResourceWrapper;
 import com.cloud.storage.template.TemplateProp;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 @ResourceWrapper(handles =  ModifyStoragePoolCommand.class)
 public final class LibvirtModifyStoragePoolCommandWrapper extends CommandWrapper<ModifyStoragePoolCommand, Answer, LibvirtComputingResource> {
@@ -49,11 +50,16 @@ public Answer execute(final ModifyStoragePoolCommand command, final LibvirtCompu
             return answer;
         }
 
-        final KVMStoragePool storagepool =
-                storagePoolMgr.createStoragePool(command.getPool().getUuid(), command.getPool().getHost(), command.getPool().getPort(), command.getPool().getPath(), command.getPool()
-                        .getUserInfo(), command.getPool().getType(), command.getDetails());
-        if (storagepool == null) {
-            return new Answer(command, false, " Failed to create storage pool");
+        final KVMStoragePool storagepool;
+        try {
+            storagepool =
+                    storagePoolMgr.createStoragePool(command.getPool().getUuid(), command.getPool().getHost(), command.getPool().getPort(), command.getPool().getPath(), command.getPool()
+                            .getUserInfo(), command.getPool().getType(), command.getDetails());
+            if (storagepool == null) {
+                return new Answer(command, false, " Failed to create storage pool");
+            }
+        } catch (CloudRuntimeException e) {
+            return new Answer(command, false, String.format("Failed to create storage pool: %s", e.getLocalizedMessage()));
         }
 
         final Map<String, TemplateProp> tInfo = new HashMap<String, TemplateProp>();
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index 29c152f934f6..f4d769a183a0 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -22,11 +22,13 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
+import com.cloud.agent.api.PrepareStorageClientCommand;
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
 import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager;
 import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
@@ -158,10 +160,45 @@ public KVMStoragePool createStoragePool(String uuid, String host, int port, Stri
                 }
             }
         }
+
+        validateMdmState(details);
+
         MapStorageUuidToStoragePool.put(uuid, storagePool);
         return storagePool;
     }
 
+    /**
+     * Validate Storage Pool state to ensure it healthy and can operate requests.
+     * There is observed situation where ScaleIO configuration file has different values than ScaleIO CLI.
+     * Validation compares values from both drv_cfg.txt and drv_cfg CLI and throws exception if there is mismatch.
+     *
+     * @param details see {@link PrepareStorageClientCommand#getDetails()}
+     *                and {@link @UnprepareStorageClientCommand#getDetails()}, expected to contain
+     *                {@link ScaleIOSDCManager#ValidateMdmsOnConnect#key()}
+     * @throws CloudRuntimeException in case if Storage Pool is not operate-able
+     */
+    private void validateMdmState(Map<String, String> details) {
+
+        String configKey = ScaleIOSDCManager.ValidateMdmsOnConnect.key();
+        String configValue = details.get(configKey);
+
+        // be as much verbose as possible, otherwise it will be difficult to troubleshoot operational issue without logs
+        if (StringUtils.isEmpty(configValue)) {
+            LOGGER.debug(String.format("Skipped ScaleIO validation as property %s not sent by Management Server", configKey));
+        } else if (Boolean.valueOf(configValue).equals(Boolean.FALSE)) {
+            LOGGER.debug(String.format("Skipped ScaleIO validation as property %s received as %s", configKey, configValue));
+        } else {
+            Collection<String> mdmsConfig = ScaleIOUtil.getMdmsFromConfig();
+            Collection<String> mdmsCli = ScaleIOUtil.getMdmsFromCli();
+            if (!mdmsCli.equals(mdmsConfig)) {
+                String msg = String.format("MDM addresses from memory and configuration file don't match. " +
+                        "Memory values: %s, configuration file values: %s", mdmsCli, mdmsConfig);
+                LOGGER.warn(msg);
+                throw new CloudRuntimeException(msg);
+            }
+        }
+    }
+
     @Override
     public boolean deleteStoragePool(String uuid) {
         ScaleIOStoragePool storagePool = (ScaleIOStoragePool) MapStorageUuidToStoragePool.get(uuid);
@@ -617,13 +654,16 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
             String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
             String[] mdmAddresses = mdms.split(",");
             if (mdmAddresses.length > 0) {
-                if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client");
                 }
 
-                ScaleIOUtil.addMdms(Arrays.asList(mdmAddresses));
-                if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                ScaleIOUtil.addMdms(mdmAddresses);
+                if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Ternary<>(false, null, "Failed to add MDMs");
+                } else {
+                    LOGGER.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid));
+                    applyTimeout(details);
                 }
             }
         }
@@ -646,20 +686,66 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
             String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
             String[] mdmAddresses = mdms.split(",");
             if (mdmAddresses.length > 0) {
-                if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Pair<>(true, "MDM not added, no need to unprepare the SDC client");
+                } else if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty()) {
+                    return new Pair<>(false, "Failed to remove MDMs, SDC client requires service to be restarted, but there are Volumes attached to the Host");
                 }
 
-                ScaleIOUtil.removeMdms(Arrays.asList(mdmAddresses));
-                if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                ScaleIOUtil.removeMdms(mdmAddresses);
+                if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Pair<>(false, "Failed to remove MDMs, unable to unprepare the SDC client");
+                } else {
+                    LOGGER.debug(String.format("MDMs %s removed from storage pool %s", mdms, uuid));
+                    applyTimeout(details);
                 }
             }
         }
 
+        /*
+         * TODO:
+         * 1. Verify on-demand is true
+         * 2. If on-demand is true check whether other MDM addresses are still present
+         * 3. If there are no MDM addresses, then stop SDC service.
+         */
+
         return new Pair<>(true, "Unprepared SDC client successfully");
     }
 
+    /**
+     * Check whether details map has timeout configured and do "apply timeout" pause before returning response
+     * (to have ScaleIO changes applied).
+     *
+     * @param details see {@link PrepareStorageClientCommand#getDetails()}
+     *                and {@link @UnprepareStorageClientCommand#getDetails()}, expected to contain
+     *                {@link ScaleIOSDCManager#MdmsChangeApplyTimeout#key()}
+     */
+    private void applyTimeout(Map<String, String> details) {
+        String configKey = ScaleIOSDCManager.MdmsChangeApplyTimeout.key();
+        String configValue = details.get(configKey);
+
+        if (StringUtils.isEmpty(configValue)) {
+            LOGGER.debug(String.format("Apply timeout value not defined in property %s, skip", configKey));
+            return;
+        }
+        long timeoutMs;
+        try {
+            timeoutMs = Long.parseLong(configValue);
+        } catch (NumberFormatException e) {
+            LOGGER.warn(String.format("Invalid apply timeout value defined in property %s, skip", configKey), e);
+            return;
+        }
+        if (timeoutMs < 1) {
+            LOGGER.warn(String.format("Apply timeout value is too small (%s ms), skipping", timeoutMs));
+            return;
+        }
+        try {
+            Thread.sleep(timeoutMs);
+        } catch (InterruptedException e) {
+            LOGGER.warn(String.format("Apply timeout %s ms interrupted", timeoutMs), e);
+        }
+    }
+
     private Map<String, String> getSDCDetails(Map<String, String> details) {
         Map<String, String> sdcDetails = new HashMap<String, String>();
         if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID))  {
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
index 421fee096341..6bcd43797be5 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
@@ -180,7 +180,7 @@ public void testUnprepareStorageClient_MDMNotAdded() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
 
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
 
@@ -195,7 +195,10 @@ public void testUnprepareStorageClient_RemoveMDMFailed() {
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
+        when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null));
+        when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols --file /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>("", null));
+
 
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
 
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
index ba6f9ce4b0a8..7bf3e95314ed 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
@@ -96,7 +96,7 @@ public void testSdcIdAttribute() {
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
+                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
                     sdcId);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
@@ -117,9 +117,9 @@ public void testSdcGuidAttribute() {
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
+                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
                     null);
-            when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid")).thenReturn(sdcGuid);
+            when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid --file /etc/emc/scaleio/drv_cfg.txt")).thenReturn(sdcGuid);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
                     details, adapter);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
index 2dc5acffcbc1..77a8677e6b14 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
@@ -38,6 +38,9 @@ public interface ScaleIOGatewayClient {
     String GATEWAY_API_PASSWORD = "powerflex.gw.password";
     String STORAGE_POOL_NAME = "powerflex.storagepool.name";
     String STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id";
+    /**
+     * Storage Pool Metadata Management (MDM) IP address(es).
+     */
     String STORAGE_POOL_MDMS = "powerflex.storagepool.mdms";
     String SDC_ID = "powerflex.sdc.id";
     String SDC_GUID = "powerflex.sdc.guid";
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 461992be1022..31a655cc20e2 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -307,6 +307,8 @@ public boolean attachZone(DataStore dataStore, ZoneScope scope, Hypervisor.Hyper
     @Override
     public boolean maintain(DataStore store) {
         Map<String,String> details = new HashMap<>();
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
         StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
         if (systemIdDetail != null) {
             details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -328,6 +330,8 @@ public boolean cancelMaintain(DataStore store) {
         StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
         if (systemIdDetail != null) {
             details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
+            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
+            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
             sdcManager = ComponentContext.inject(sdcManager);
             if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
                 StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index b02732051b5e..ad5f444d12ed 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -32,6 +32,24 @@ public interface ScaleIOSDCManager {
             Boolean.TRUE,
             ConfigKey.Scope.Zone);
 
+    /**
+     * Timeout for Host to wait after MDM changes made on Host until changes will be applied.
+     * Needed to avoid cases when Storage Pool is not connected yet, but Agent already starts to use Storage Pool.
+     */
+    ConfigKey<Integer> MdmsChangeApplyTimeout = new ConfigKey<>("Storage",
+            Integer.class,
+            "powerflex.mdm.change.apply.timeout.ms",
+            "1000",
+            "Timeout for Host to wait after MDM changes made on Host until changes will be applied, default value: 1000 ms",
+            Boolean.TRUE);
+
+    ConfigKey<Boolean> ValidateMdmsOnConnect = new ConfigKey<>("Storage",
+            Boolean.class,
+            "powerflex.mdm.validate.on.connect",
+            Boolean.FALSE.toString(),
+            "Flag to validate MDMs on Host, present in configuration file and in CLI, default value: false",
+            Boolean.TRUE);
+
     /**
      * Checks SDC connections limit.
      * @param storagePoolId the storage pool id
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
index 79f82a1e0df0..1031b2522b4c 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
@@ -204,9 +204,11 @@ public String prepareSDC(Host host, DataStore dataStore) {
 
     private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId, String mdms) {
         logger.debug("Preparing SDC on the host {}", host);
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
+        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.value()));
+
         PrepareStorageClientCommand cmd = new PrepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details);
         int timeoutSeconds = 60;
         cmd.setWait(timeoutSeconds);
@@ -324,6 +326,7 @@ private boolean unprepareSDCOnHost(Host host, DataStore dataStore, String mdms)
         logger.debug(String.format("Unpreparing SDC on the host %s (%s)", host.getId(), host.getName()));
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
+        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.value()));
         UnprepareStorageClientCommand cmd = new UnprepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details);
         int timeoutSeconds = 60;
         cmd.setWait(timeoutSeconds);
@@ -477,6 +480,6 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey[]{ConnectOnDemand};
+        return new ConfigKey[]{ConnectOnDemand, MdmsChangeApplyTimeout, ValidateMdmsOnConnect};
     }
 }
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index 0db329204e8c..f80a36911e31 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -111,6 +111,8 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
         }
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -203,6 +205,8 @@ public boolean hostDisconnected(long hostId, long poolId) {
         }
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.canUnprepareSDC(host, dataStore)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index a8c02a90bf9e..31fb172e6914 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -17,16 +17,26 @@
 
 package org.apache.cloudstack.storage.datastore.util;
 
-import java.util.List;
-
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
+import com.cloud.utils.Pair;
 import com.cloud.utils.UuidUtils;
 import com.cloud.utils.script.Script;
 import org.apache.commons.lang3.StringUtils;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 public class ScaleIOUtil {
     protected static Logger LOGGER = LogManager.getLogger(ScaleIOUtil.class);
 
@@ -39,7 +49,7 @@ public class ScaleIOUtil {
     public static final String VMSNAPSHOT_PREFIX = "vmsnap";
 
     public static final int IDENTIFIER_LENGTH = 16;
-    public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = Long.valueOf(10);
+    public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = 10L;
 
     public static final String DISK_PATH = "/dev/disk/by-id";
     public static final String DISK_NAME_PREFIX = "emc-vol-";
@@ -63,9 +73,10 @@ public class ScaleIOUtil {
     private static final String SDC_SERVICE_ENABLE_CMD = "systemctl enable scini";
 
     public static final String CONNECTED_SDC_COUNT_STAT = "ConnectedSDCCount";
+
     /**
      * Cmd for querying volumes in SDC
-     * Sample output for cmd: drv_cfg --query_vols:
+     * Sample output for cmd {@code drv_cfg --query_vols}:
      * Retrieved 2 volume(s)
      * VOL-ID 6c33633100000009 MDM-ID 218ce1797566a00f
      * VOL-ID 6c3362a30000000a MDM-ID 218ce1797566a00f
@@ -74,14 +85,14 @@ public class ScaleIOUtil {
 
     /**
      * Cmd for querying guid in SDC
-     * Sample output for cmd: drv_cfg --query_guid:
+     * Sample output for cmd {@code drv_cfg --query_guid}:
      * B0E3BFB8-C20B-43BF-93C8-13339E85AA50
      */
     private static final String QUERY_GUID_CMD = "drv_cfg --query_guid";
 
     /**
      * Cmd for querying MDMs in SDC
-     * Sample output for cmd: drv_cfg --query_mdms:
+     * Sample output for cmd {@code drv_cfg --query_mdms}:
      * Retrieved 2 mdm(s)
      * MDM-ID 3ef46cbf2aaf5d0f SDC ID 6b18479c00000003 INSTALLATION ID 68ab55462cbb3ae4 IPs [0]-x.x.x.x [1]-x.x.x.x
      * MDM-ID 2e706b2740ec200f SDC ID 301b852c00000003 INSTALLATION ID 33f8662e7a5c1e6c IPs [0]-x.x.x.x [1]-x.x.x.x
@@ -89,61 +100,246 @@ public class ScaleIOUtil {
     private static final String QUERY_MDMS_CMD = "drv_cfg --query_mdms";
 
     private static final String ADD_MDMS_CMD = "drv_cfg --add_mdm";
+
+    private static final String REMOVE_MDM_PARAMETER = "--remove_mdm";
+
+    /**
+     * Calls the kernel module to remove MDM.
+     */
+    private static final String REMOVE_MDMS_CMD = "drv_cfg " + REMOVE_MDM_PARAMETER;
+
+    /**
+     * Command to get back "Usage" response. As of ow it is just drv_cfg without parameters.
+     */
+    private static final String USAGE_CMD = "drv_cfg";
+
     private static final String DRV_CFG_FILE = "/etc/emc/scaleio/drv_cfg.txt";
 
-    public static void addMdms(List<String> mdmAddresses) {
-        if (CollectionUtils.isEmpty(mdmAddresses)) {
-            return;
+    /**
+     * Sample Command - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt
+     */
+    private static final String REMOVE_MDM_CMD_TEMPLATE = "sed -i '/%s\\,/d' %s";
+
+    /**
+     * Patterns to parse {@link ScaleIOUtil#DRV_CFG_FILE} and {@code --query_mdms} command output.
+     * The format is:
+     * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs [{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ...
+     */
+    private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\\r?\\n");
+    /**
+     * Pattern to find "IPs" substring in {@link ScaleIOUtil#QUERY_MDMS_CMD} command output.
+     * The output format is:
+     * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs [{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ...
+     */
+    private static final Pattern USAGE_IPS_LINE_PATTERN = Pattern.compile("IPs\\s*(.*)$");
+    /**
+     * Pattern to find individual IP address in {@link ScaleIOUtil#QUERY_MDMS_CMD} command output.
+     */
+    private static final Pattern USAGE_IP_TOKEN_PATTERN = Pattern.compile("(\\s*\\[\\d\\]\\-)([^\\s$]+)");
+
+    /**
+     * Pattern to find Volume ID in {@link  ScaleIOUtil#QUERY_VOLUMES_CMD}
+     */
+    private static final Pattern VOLUME_ID_TOKEN_PATTERN = Pattern.compile("VOL-ID\\s*([^\\s$]+)");
+    /**
+     * Pattern to find MDM entries line in {@link ScaleIOUtil#DRV_CFG_FILE}.
+     */
+    private static final Pattern DRV_CFG_MDM_LINE_PATTERN = Pattern.compile("^mdm\\s*([0-9A-F:\\.,]+)$", Pattern.CASE_INSENSITIVE);
+    /**
+     * Pattern to split comma separated string of IP addresses (space aware).
+     */
+    private static final Pattern DRV_CFG_MDM_IPS_PATTERN = Pattern.compile("\\s*,\\s*");
+
+    public static boolean addMdms(String... mdmAddresses) {
+        if (mdmAddresses.length < 1) {
+            return false;
         }
         // Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --add_mdm --ip x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt
-        String addMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.ADD_MDMS_CMD;
-        addMdmsCmd += " --ip " + String.join(",", mdmAddresses);
-        addMdmsCmd += " --file " + DRV_CFG_FILE;
-        String result = Script.runSimpleBashScript(addMdmsCmd);
-        if (result == null) {
-            LOGGER.warn("Failed to add mdms");
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.ADD_MDMS_CMD;
+        command += " --ip " + String.join(",", mdmAddresses);
+        command += " --file " + DRV_CFG_FILE;
+        return runCmd(command);
+    }
+
+    /**
+     * Remove MDM via ScaleIO via CLI.
+     *
+     * @param mdmAddress MDM address to remove
+     * @return true if IP address successfully removed
+     */
+    private static boolean removeMdm(String mdmAddress) {
+        // Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --remove_mdm --ip x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.REMOVE_MDMS_CMD;
+        command += " --ip " + String.join(",", mdmAddress);
+        command += " --file " + DRV_CFG_FILE;
+        return runCmd(command);
+    }
+
+    /**
+     * Run command, log command result and return {@link Boolean#TRUE} if command succeeded.
+     * FIXME: may need to do refactoring and replace static method callss with dynamic.
+     */
+    private static boolean runCmd(String command) {
+        Pair<String, String> result = Script.executeCommand(command);
+        String stdOut = result.first();
+        String stdErr = result.second();
+        boolean succeeded = StringUtils.isEmpty(stdErr);
+        if (succeeded) {
+            LOGGER.debug(String.format("Successfully executed command '%s': %s", command, stdOut));
+        } else {
+            LOGGER.warn(String.format("Failed to execute command '%s': %s", command, stdErr));
         }
+        return succeeded;
     }
 
-    public static void removeMdms(List<String> mdmAddresses) {
-        if (CollectionUtils.isEmpty(mdmAddresses)) {
-            return;
+    /**
+     * Remove MDMs either via ScaleIO CLI (if supported) or by updating configuration file.
+     *
+     * @param mdmAddresses MDM addresses
+     * @return returns {@link Boolean#TRUE} if changes were applied to the configuration
+     */
+    public static boolean removeMdms(String... mdmAddresses) {
+        if (mdmAddresses.length < 1) {
+            return false;
         }
-        // (i) Remove MDMs from config file (ii) Restart scini
-        //  Sample Cmd - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt
+
+        boolean changesApplied = false;
+        boolean removeMdmCliSupported = isRemoveMdmCliSupported();
         boolean restartSDC = false;
-        String removeMdmsCmdFormat = "sed -i '/%s\\,/d' %s";
         for (String mdmAddress : mdmAddresses) {
-            if (mdmAdded(mdmAddress)) {
+            // continue to next address if current MDM is not present in configuratino
+            if (!isMdmPresent(mdmAddress)) {
+                continue;
+            }
+            // remove MDM via CLI if it is supported
+            if (removeMdmCliSupported) {
+                removeMdm(mdmAddress);
+                changesApplied = true;
+            } else {
+                String command = String.format(REMOVE_MDM_CMD_TEMPLATE, mdmAddress, DRV_CFG_FILE);
+                Script.runSimpleBashScript(command);
+                // restart SDC needed only if configuration file modified manually (not by CLI)
                 restartSDC = true;
+                changesApplied = true;
             }
-            String removeMdmsCmd = String.format(removeMdmsCmdFormat, mdmAddress, DRV_CFG_FILE);
-            Script.runSimpleBashScript(removeMdmsCmd);
         }
         if (restartSDC) {
             restartSDCService();
         }
+        return changesApplied;
+    }
+
+    /**
+     * Returns MDM entries from {@link ScaleIOUtil#DRV_CFG_FILE}.
+     */
+    public static Collection<String> getMdmsFromConfig() {
+        List<String> configFileLines;
+        try {
+            configFileLines = Files.readAllLines(Path.of(DRV_CFG_FILE));
+        } catch (IOException e) {
+            LOGGER.error(String.format("Failed to read MDMs from %s", DRV_CFG_FILE), e);
+            return List.of();
+        }
+        Set<String> mdms = new LinkedHashSet<>();
+        for (String line : configFileLines) {
+            Matcher mdmLineMatcher = DRV_CFG_MDM_LINE_PATTERN.matcher(line);
+            if(mdmLineMatcher.find() && mdmLineMatcher.groupCount() > 0) {
+                String mdmLine = mdmLineMatcher.group(1);
+                String[] mdmValues = DRV_CFG_MDM_IPS_PATTERN.split(mdmLine);
+                mdms.addAll(Arrays.asList(mdmValues));
+            }
+        }
+        return mdms;
+    }
+    /**
+     * Returns Volume Ids from {@link ScaleIOUtil#DRV_CFG_FILE}.
+     */
+    public static Collection<String> getVolumeIds() {
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD + " --file " + DRV_CFG_FILE;
+        Pair<String, String> result = Script.executeCommand(command);
+        String stdOut = result.first();
+
+        Set<String> volumeIds = new LinkedHashSet<>();
+        String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut);
+        for (String line : stdOutLines) {
+            Matcher volumeIdMatcher = VOLUME_ID_TOKEN_PATTERN.matcher(line);
+            if (volumeIdMatcher.find() && volumeIdMatcher.groupCount() > 0) {
+                volumeIds.add(volumeIdMatcher.group(1));
+            }
+        }
+        return volumeIds;
+    }
+
+    /**
+     * Returns MDM entries from CLI using {@code --query_mdms}.
+     */
+    public static Collection<String> getMdmsFromCli() {
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
+        Pair<String, String> result = Script.executeCommand(command);
+        String stdOut = result.first();
+
+        Set<String> mdms = new LinkedHashSet<>();
+        String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut);
+        for (String line : stdOutLines) {
+            Matcher ipsLineMatcher = USAGE_IPS_LINE_PATTERN.matcher(line);
+            if (ipsLineMatcher.find() && ipsLineMatcher.groupCount() > 0) {
+                String ipToken = ipsLineMatcher.group(1);
+                Matcher ipMatcher = USAGE_IP_TOKEN_PATTERN.matcher(ipToken);
+                while (ipMatcher.find()) {
+                    if (ipMatcher.groupCount() > 1) {
+                        mdms.add(ipMatcher.group(2));
+                    }
+                }
+            }
+        }
+        return mdms;
+    }
+
+    /**
+     * Returns {@link Boolean#TRUE} if ScaleIO CLI tool (drv_cfg) supports MDMs removal.
+     */
+    public static boolean isRemoveMdmCliSupported() {
+        /*
+         * New version of drv_cfg supports remove mdm API.
+         * Instead of defining supported version and checking it, the logic is to check drv_cfg "Usage" output
+         * and see whether remove_mdm command supported.
+         * The "Usage" returned if tool executed without parameters or with invalid parameters.
+         */
+        String command = SDC_HOME_PATH + "/bin/" + USAGE_CMD;
+
+        Pair<String, String> result = Script.executeCommand(command);
+        String stdErr = result.first();
+        String stdOut = result.second();
+
+        /*
+         * Check whether stderr or stdout contains mdm removal parameter.
+         *
+         * Current version returns "Usage" in stderr, check stdout as well in case this will be changed in the future,
+         * as returned "Usage" is not an error.
+         */
+        return (stdOut + stdErr).toLowerCase().contains(REMOVE_MDM_PARAMETER);
     }
 
-    public static boolean mdmAdded(String mdmAddress) {
+    /**
+     * Returns true if provided MDM address is present in configuration.
+     */
+    public static boolean isMdmPresent(String mdmAddress) {
         //query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
-        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
+        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
         queryMdmsCmd += "|grep " + mdmAddress;
         String result = Script.runSimpleBashScript(queryMdmsCmd);
-        if (StringUtils.isNotBlank(result) && result.contains(mdmAddress)) {
-            return true;
-        }
-        return false;
+
+        return StringUtils.isNotBlank(result) && result.contains(mdmAddress);
     }
 
     public static String getSdcHomePath() {
-        String sdcHomePath = DEFAULT_SDC_HOME_PATH;
         String sdcHomePropertyCmdFormat = "sed -n '/%s/p' '%s' 2>/dev/null  | sed 's/%s=//g' 2>/dev/null";
         String sdcHomeCmd = String.format(sdcHomePropertyCmdFormat, SDC_HOME_PARAMETER, AGENT_PROPERTIES_FILE, SDC_HOME_PARAMETER);
-
         String result = Script.runSimpleBashScript(sdcHomeCmd);
+        String sdcHomePath;
         if (result == null) {
-            LOGGER.warn("Failed to get sdc home path from agent.properties, fallback to default path");
+            sdcHomePath = DEFAULT_SDC_HOME_PATH;
+            LOGGER.warn(String.format("Failed to get sdc home path from agent.properties, fallback to default path %s", sdcHomePath));
         } else {
             sdcHomePath = result;
         }
@@ -151,9 +347,9 @@ public static String getSdcHomePath() {
         return sdcHomePath;
     }
 
-    public static final void rescanForNewVolumes() {
+    public static void rescanForNewVolumes() {
         // Detecting new volumes
-        String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD;
+        String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD + " --file " + DRV_CFG_FILE;
 
         String result = Script.runSimpleBashScript(rescanCmd);
         if (result == null) {
@@ -161,9 +357,9 @@ public static final void rescanForNewVolumes() {
         }
     }
 
-    public static final String getSystemIdForVolume(String volumeId) {
+    public static String getSystemIdForVolume(String volumeId) {
         //query_vols outputs "VOL-ID <VolumeID> MDM-ID <SystemID>" for a volume with ID: <VolumeID>
-        String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD;
+        String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD + " --file " + DRV_CFG_FILE;
         queryDiskCmd += "|grep " + volumeId + "|awk '{print $4}'";
 
         String result = Script.runSimpleBashScript(queryDiskCmd);
@@ -181,7 +377,7 @@ public static final String getSystemIdForVolume(String volumeId) {
     }
 
     public static String getSdcGuid() {
-        String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD;
+        String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD + " --file " + DRV_CFG_FILE;
         String result = Script.runSimpleBashScript(queryGuidCmd);
         if (result == null) {
             LOGGER.warn("Failed to get SDC guid");
@@ -203,7 +399,7 @@ public static String getSdcGuid() {
 
     public static String getSdcId(String mdmId) {
         //query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
-        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
+        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
         queryMdmsCmd += "|grep " + mdmId + "|awk '{print $5}'";
         String result = Script.runSimpleBashScript(queryMdmsCmd);
         if (result == null) {
@@ -225,7 +421,7 @@ public static String getSdcId(String mdmId) {
         return result;
     }
 
-    public static final String getVolumePath(String volumePathWithName) {
+    public static String getVolumePath(String volumePathWithName) {
         if (StringUtils.isEmpty(volumePathWithName)) {
             return volumePathWithName;
         }
@@ -237,7 +433,7 @@ public static final String getVolumePath(String volumePathWithName) {
         return volumePathWithName;
     }
 
-    public static final String updatedPathWithVolumeName(String volumePath, String volumeName) {
+    public static String updatedPathWithVolumeName(String volumePath, String volumeName) {
         if (StringUtils.isAnyEmpty(volumePath, volumeName)) {
             return volumePath;
         }
@@ -279,4 +475,39 @@ public static boolean restartSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD);
         return exitValue == 0;
     }
+
+    /**
+     * Represents {@link ScaleIOUtil#DRV_CFG_FILE} MDM entry (SDC and Installation Ids are skipped).
+     */
+    public static class MdmEntry {
+        private String mdmId;
+        private Collection<String> ips;
+
+        /**
+         * MDM entry constructor.
+         *
+         * @param mdmId MDM Id
+         * @param ips   IP Addresses
+         */
+        public MdmEntry(String mdmId, Collection<String> ips) {
+            this.mdmId = mdmId;
+            this.ips = ips;
+        }
+
+        public String getMdmId() {
+            return mdmId;
+        }
+
+        public void setMdmId(String mdmId) {
+            this.mdmId = mdmId;
+        }
+
+        public Collection<String> getIps() {
+            return ips;
+        }
+
+        public void setIps(Collection<String> ips) {
+            this.ips = ips;
+        }
+    }
 }
diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java
index a1104b37c27b..8feeaad61015 100644
--- a/utils/src/main/java/com/cloud/utils/script/Script.java
+++ b/utils/src/main/java/com/cloud/utils/script/Script.java
@@ -671,6 +671,25 @@ public static String executeCommand(String... command) {
         return runScript(getScriptForCommandRun(command));
     }
 
+    /**
+     * Execute command and return standard output and standard error.
+     *
+     * @param command OS command to be executed
+     * @return {@link Pair} with standard output as first and standard error as second field
+     */
+    public static Pair<String, String> executeCommand(String command) {
+        // wrap command into bash
+        Script script = new Script("/bin/bash");
+        script.add("-c");
+        script.add(command);
+
+        // parse all lines from the output
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+        String stdErr = script.execute(parser);
+        String stdOut = parser.getLines();
+        return new Pair(stdOut, stdErr);
+    }
+
     public static int executeCommandForExitValue(long timeout, String... command) {
         return runScriptForExitValue(getScriptForCommandRun(timeout, command));
     }

From 3cf8eb192364d7d865b500ac89a33ebb68fe765d Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@apple.com>
Date: Tue, 8 Apr 2025 17:13:21 -0700
Subject: [PATCH 02/14] Add feature flag to enable/disable blocking unprepare
 ScaleIO storage pool when SDC client restart required and there are volumes
 attached to the Host

---
 .../kvm/storage/ScaleIOStorageAdaptor.java       | 16 ++++++++++++++--
 .../ScaleIOPrimaryDataStoreLifeCycle.java        |  2 ++
 .../datastore/manager/ScaleIOSDCManager.java     |  7 +++++++
 .../datastore/manager/ScaleIOSDCManagerImpl.java |  2 +-
 .../datastore/provider/ScaleIOHostListener.java  |  2 ++
 5 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index f4d769a183a0..ebd86c1960f7 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -688,8 +688,20 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
             if (mdmAddresses.length > 0) {
                 if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Pair<>(true, "MDM not added, no need to unprepare the SDC client");
-                } else if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty()) {
-                    return new Pair<>(false, "Failed to remove MDMs, SDC client requires service to be restarted, but there are Volumes attached to the Host");
+                } else {
+                    String configKey =
+                            details.get(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key());
+                    String configValue = details.get(configKey);
+
+                    if (StringUtils.isEmpty(configValue)) {
+                        LOGGER.debug(String.format("Configuration key %s not provided", configKey));
+                    } else {
+                        LOGGER.debug(String.format("Configuration key %s provided as %s", configKey, configValue));
+                    }
+                    Boolean blockUnprepare = Boolean.valueOf(configKey);
+                    if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty() && Boolean.TRUE.equals(blockUnprepare)) {
+                        return new Pair<>(false, "Failed to remove MDMs, SDC client requires service to be restarted, but there are Volumes attached to the Host");
+                    }
                 }
 
                 ScaleIOUtil.removeMdms(mdmAddresses);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 31a655cc20e2..2ce7d49ab07c 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -309,6 +309,7 @@ public boolean maintain(DataStore store) {
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
         details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
         StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
         if (systemIdDetail != null) {
             details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -332,6 +333,7 @@ public boolean cancelMaintain(DataStore store) {
             details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
             details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
             details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
+            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
             sdcManager = ComponentContext.inject(sdcManager);
             if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
                 StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index ad5f444d12ed..5051efb30f96 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -50,6 +50,13 @@ public interface ScaleIOSDCManager {
             "Flag to validate MDMs on Host, present in configuration file and in CLI, default value: false",
             Boolean.TRUE);
 
+    ConfigKey<Boolean> BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached = new ConfigKey<>("Storage",
+            Boolean.class,
+            "powerflex.block.sdc.unprepare.if.service.restart.needed.and.volumes.attached",
+            Boolean.FALSE.toString(),
+            "Block Storage Client un-preparation if SDC service restart needed but there are Volumes attached to the Host",
+            Boolean.TRUE);
+
     /**
      * Checks SDC connections limit.
      * @param storagePoolId the storage pool id
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
index 1031b2522b4c..9fb64bfadb3f 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
@@ -480,6 +480,6 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey[]{ConnectOnDemand, MdmsChangeApplyTimeout, ValidateMdmsOnConnect};
+        return new ConfigKey[]{ConnectOnDemand, MdmsChangeApplyTimeout, ValidateMdmsOnConnect, BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached};
     }
 }
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index f80a36911e31..7ba7e17c86fc 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -113,6 +113,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
         details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
         details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -207,6 +208,7 @@ public boolean hostDisconnected(long hostId, long poolId) {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
         details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
         details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.canUnprepareSDC(host, dataStore)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));

From 31518c2296bccb423b3b55a60884b93680536d73 Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@gmail.com>
Date: Wed, 9 Apr 2025 10:51:14 -0700
Subject: [PATCH 03/14] Update
 plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java

Co-authored-by: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
---
 .../cloudstack/storage/datastore/manager/ScaleIOSDCManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index 5051efb30f96..e170defb025d 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -52,7 +52,7 @@ public interface ScaleIOSDCManager {
 
     ConfigKey<Boolean> BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached = new ConfigKey<>("Storage",
             Boolean.class,
-            "powerflex.block.sdc.unprepare.if.service.restart.needed.and.volumes.attached",
+            "powerflex.block.sdc.unprepare",
             Boolean.FALSE.toString(),
             "Block Storage Client un-preparation if SDC service restart needed but there are Volumes attached to the Host",
             Boolean.TRUE);

From aa982ebecc2a3bee9c7627deb8ff5a4a22e1bb27 Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@gmail.com>
Date: Fri, 11 Apr 2025 10:58:02 -0700
Subject: [PATCH 04/14] Update
 plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java

Co-authored-by: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
---
 .../cloudstack/storage/datastore/manager/ScaleIOSDCManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index e170defb025d..0ba40217056c 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -40,7 +40,7 @@ public interface ScaleIOSDCManager {
             Integer.class,
             "powerflex.mdm.change.apply.timeout.ms",
             "1000",
-            "Timeout for Host to wait after MDM changes made on Host until changes will be applied, default value: 1000 ms",
+            "Timeout (in ms) for Host to wait after MDM changes made on Host until changes will be applied, default value: 1000 ms",
             Boolean.TRUE);
 
     ConfigKey<Boolean> ValidateMdmsOnConnect = new ConfigKey<>("Storage",

From c4251deb7f8de8ce91c164764c62b7e76066b400 Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@gmail.com>
Date: Fri, 11 Apr 2025 11:01:57 -0700
Subject: [PATCH 05/14] Update
 plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java

Co-authored-by: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
---
 .../apache/cloudstack/storage/datastore/util/ScaleIOUtil.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index 31fb172e6914..f5a5fa3892e1 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -214,7 +214,9 @@ public static boolean removeMdms(String... mdmAddresses) {
             // remove MDM via CLI if it is supported
             if (removeMdmCliSupported) {
                 removeMdm(mdmAddress);
-                changesApplied = true;
+                if (removeMdm(mdmAddress)) {
+                    changesApplied = true;
+                }
             } else {
                 String command = String.format(REMOVE_MDM_CMD_TEMPLATE, mdmAddress, DRV_CFG_FILE);
                 Script.runSimpleBashScript(command);

From 2a78099c33bb57c74dc8a9139a21ad24951dd9fd Mon Sep 17 00:00:00 2001
From: mprokopchuk <mprokopchuk@apple.com>
Date: Fri, 11 Apr 2025 11:58:19 -0700
Subject: [PATCH 06/14] - Addressed code review comments: Added checking mdm
 removal response validation

---
 .../kvm/storage/ScaleIOStorageAdaptorTest.java      |  1 +
 .../storage/datastore/util/ScaleIOUtil.java         | 13 ++++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
index 6bcd43797be5..9cfab83b8b97 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
@@ -194,6 +194,7 @@ public void testUnprepareStorageClient_RemoveMDMFailed() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
+        when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d' /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null));
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
         when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
         when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null));
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index f5a5fa3892e1..74b37f620ab5 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -213,16 +213,19 @@ public static boolean removeMdms(String... mdmAddresses) {
             }
             // remove MDM via CLI if it is supported
             if (removeMdmCliSupported) {
-                removeMdm(mdmAddress);
                 if (removeMdm(mdmAddress)) {
                     changesApplied = true;
                 }
             } else {
                 String command = String.format(REMOVE_MDM_CMD_TEMPLATE, mdmAddress, DRV_CFG_FILE);
-                Script.runSimpleBashScript(command);
-                // restart SDC needed only if configuration file modified manually (not by CLI)
-                restartSDC = true;
-                changesApplied = true;
+                String stdErr = Script.executeCommand(command).second();
+                if(StringUtils.isEmpty(stdErr)) {
+                    // restart SDC needed only if configuration file modified manually (not by CLI)
+                    restartSDC = true;
+                    changesApplied = true;
+                } else {
+                    LOGGER.error(String.format("Failed to remove MDM %s from %s: %s", mdmAddress, DRV_CFG_FILE, stdErr));
+                }
             }
         }
         if (restartSDC) {

From fc9404eb9b6e527c14b28e9169927e90c6203bbb Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
Date: Mon, 21 Apr 2025 16:39:33 +0530
Subject: [PATCH 07/14] Updated scope to Zone for the new powerflex settings

---
 .../storage/datastore/manager/ScaleIOSDCManager.java     | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index 0ba40217056c..c4a59fe81abe 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -41,21 +41,24 @@ public interface ScaleIOSDCManager {
             "powerflex.mdm.change.apply.timeout.ms",
             "1000",
             "Timeout (in ms) for Host to wait after MDM changes made on Host until changes will be applied, default value: 1000 ms",
-            Boolean.TRUE);
+            Boolean.TRUE,
+            ConfigKey.Scope.Zone);
 
     ConfigKey<Boolean> ValidateMdmsOnConnect = new ConfigKey<>("Storage",
             Boolean.class,
             "powerflex.mdm.validate.on.connect",
             Boolean.FALSE.toString(),
             "Flag to validate MDMs on Host, present in configuration file and in CLI, default value: false",
-            Boolean.TRUE);
+            Boolean.TRUE,
+            ConfigKey.Scope.Zone);
 
     ConfigKey<Boolean> BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached = new ConfigKey<>("Storage",
             Boolean.class,
             "powerflex.block.sdc.unprepare",
             Boolean.FALSE.toString(),
             "Block Storage Client un-preparation if SDC service restart needed but there are Volumes attached to the Host",
-            Boolean.TRUE);
+            Boolean.TRUE,
+            ConfigKey.Scope.Zone);
 
     /**
      * Checks SDC connections limit.

From 1d504ef1c50949ffcdb68653aaa755a4a13bf797 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
Date: Mon, 16 Jun 2025 17:44:56 +0530
Subject: [PATCH 08/14] Get powerflex storage config settings from zone, and
 some fixes

---
 .../kvm/storage/ScaleIOStorageAdaptor.java    |  6 +--
 .../ScaleIOPrimaryDataStoreLifeCycle.java     | 45 ++++++++++---------
 .../manager/ScaleIOSDCManagerImpl.java        |  4 +-
 .../provider/ScaleIOHostListener.java         | 12 ++---
 4 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index ebd86c1960f7..8a7f216c7e2f 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -178,7 +178,6 @@ public KVMStoragePool createStoragePool(String uuid, String host, int port, Stri
      * @throws CloudRuntimeException in case if Storage Pool is not operate-able
      */
     private void validateMdmState(Map<String, String> details) {
-
         String configKey = ScaleIOSDCManager.ValidateMdmsOnConnect.key();
         String configValue = details.get(configKey);
 
@@ -689,8 +688,7 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
                 if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Pair<>(true, "MDM not added, no need to unprepare the SDC client");
                 } else {
-                    String configKey =
-                            details.get(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key());
+                    String configKey = ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key();
                     String configValue = details.get(configKey);
 
                     if (StringUtils.isEmpty(configValue)) {
@@ -698,7 +696,7 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
                     } else {
                         LOGGER.debug(String.format("Configuration key %s provided as %s", configKey, configValue));
                     }
-                    Boolean blockUnprepare = Boolean.valueOf(configKey);
+                    Boolean blockUnprepare = Boolean.valueOf(configValue);
                     if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty() && Boolean.TRUE.equals(blockUnprepare)) {
                         return new Pair<>(false, "Failed to remove MDMs, SDC client requires service to be restarted, but there are Volumes attached to the Host");
                     }
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 2ce7d49ab07c..0d9ef608609d 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -307,16 +307,20 @@ public boolean attachZone(DataStore dataStore, ZoneScope scope, Hypervisor.Hyper
     @Override
     public boolean maintain(DataStore store) {
         Map<String,String> details = new HashMap<>();
-        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
-        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
-        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
-        StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
-        if (systemIdDetail != null) {
-            details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
-            StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS);
-            if (mdmsDetail != null) {
-                details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdmsDetail.getValue());
-                details.put(ScaleIOSDCManager.ConnectOnDemand.key(), "false");
+        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
+        if (storagePoolVO != null) {
+            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
+            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
+            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
+
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
+                StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS);
+                if (mdmsDetail != null) {
+                    details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdmsDetail.getValue());
+                    details.put(ScaleIOSDCManager.ConnectOnDemand.key(), "false");
+                }
             }
         }
 
@@ -328,16 +332,17 @@ public boolean maintain(DataStore store) {
     @Override
     public boolean cancelMaintain(DataStore store) {
         Map<String,String> details = new HashMap<>();
-        StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
-        if (systemIdDetail != null) {
-            details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
-            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
-            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
-            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
-            sdcManager = ComponentContext.inject(sdcManager);
-            if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
-                StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
-                if (storagePoolVO != null) {
+        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
+        if (storagePoolVO != null) {
+            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
+            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
+            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
+
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
+                sdcManager = ComponentContext.inject(sdcManager);
+                if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
                     details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(storagePoolVO.getDataCenterId())));
                     StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS);
                     if (mdmsDetail != null) {
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
index 9fb64bfadb3f..dec009bd0e1a 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
@@ -207,7 +207,7 @@ private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId,
         Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
-        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.value()));
+        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
 
         PrepareStorageClientCommand cmd = new PrepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details);
         int timeoutSeconds = 60;
@@ -326,7 +326,7 @@ private boolean unprepareSDCOnHost(Host host, DataStore dataStore, String mdms)
         logger.debug(String.format("Unpreparing SDC on the host %s (%s)", host.getId(), host.getName()));
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
-        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.value()));
+        details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
         UnprepareStorageClientCommand cmd = new UnprepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details);
         int timeoutSeconds = 60;
         cmd.setWait(timeoutSeconds);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index 7ba7e17c86fc..c21ed8518fdc 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -111,9 +111,9 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
         }
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
-        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
-        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -206,9 +206,9 @@ public boolean hostDisconnected(long hostId, long poolId) {
         }
         Map<String,String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.value()));
-        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.value()));
-        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.value()));
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.canUnprepareSDC(host, dataStore)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));

From 1bc40ebacb4d23a05df6d1e4b06588123a751e41 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
Date: Mon, 16 Jun 2025 17:55:22 +0530
Subject: [PATCH 09/14] Removed --file parameter used with --query_mdms,
 --query_vols, --query_guid, --rescan in drv_cfg cmd as it is not supported.
 --file parameter is supported with --add_mdm, --mod_mdm_ip, --remove_mdm,
 --set_guid, --set_mdm_password, --reset_mdm_password

---
 .../kvm/storage/ScaleIOStorageAdaptorTest.java |  6 +++---
 .../kvm/storage/ScaleIOStoragePoolTest.java    |  6 +++---
 .../storage/datastore/util/ScaleIOUtil.java    | 18 +++++++++---------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
index 9cfab83b8b97..eddaa8f64996 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
@@ -180,7 +180,7 @@ public void testUnprepareStorageClient_MDMNotAdded() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
 
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
 
@@ -196,9 +196,9 @@ public void testUnprepareStorageClient_RemoveMDMFailed() {
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
         when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d' /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null));
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
         when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null));
-        when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols --file /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>("", null));
+        when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols"))).thenReturn(new Pair<>("", null));
 
 
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
index 7bf3e95314ed..ba6f9ce4b0a8 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
@@ -96,7 +96,7 @@ public void testSdcIdAttribute() {
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
+                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
                     sdcId);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
@@ -117,9 +117,9 @@ public void testSdcGuidAttribute() {
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
             when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
+                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
                     null);
-            when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid --file /etc/emc/scaleio/drv_cfg.txt")).thenReturn(sdcGuid);
+            when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid")).thenReturn(sdcGuid);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
                     details, adapter);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index 74b37f620ab5..54201afa13c2 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -109,7 +109,7 @@ public class ScaleIOUtil {
     private static final String REMOVE_MDMS_CMD = "drv_cfg " + REMOVE_MDM_PARAMETER;
 
     /**
-     * Command to get back "Usage" response. As of ow it is just drv_cfg without parameters.
+     * Command to get back "Usage" response. As of now it is just drv_cfg without parameters.
      */
     private static final String USAGE_CMD = "drv_cfg";
 
@@ -260,7 +260,7 @@ public static Collection<String> getMdmsFromConfig() {
      * Returns Volume Ids from {@link ScaleIOUtil#DRV_CFG_FILE}.
      */
     public static Collection<String> getVolumeIds() {
-        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD + " --file " + DRV_CFG_FILE;
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD;
         Pair<String, String> result = Script.executeCommand(command);
         String stdOut = result.first();
 
@@ -279,7 +279,7 @@ public static Collection<String> getVolumeIds() {
      * Returns MDM entries from CLI using {@code --query_mdms}.
      */
     public static Collection<String> getMdmsFromCli() {
-        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
+        String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
         Pair<String, String> result = Script.executeCommand(command);
         String stdOut = result.first();
 
@@ -317,7 +317,7 @@ public static boolean isRemoveMdmCliSupported() {
         String stdOut = result.second();
 
         /*
-         * Check whether stderr or stdout contains mdm removal parameter.
+         * Check whether stderr or stdout contains mdm removal "--remove_mdm" parameter.
          *
          * Current version returns "Usage" in stderr, check stdout as well in case this will be changed in the future,
          * as returned "Usage" is not an error.
@@ -330,7 +330,7 @@ public static boolean isRemoveMdmCliSupported() {
      */
     public static boolean isMdmPresent(String mdmAddress) {
         //query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
-        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
+        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
         queryMdmsCmd += "|grep " + mdmAddress;
         String result = Script.runSimpleBashScript(queryMdmsCmd);
 
@@ -354,7 +354,7 @@ public static String getSdcHomePath() {
 
     public static void rescanForNewVolumes() {
         // Detecting new volumes
-        String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD + " --file " + DRV_CFG_FILE;
+        String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD;
 
         String result = Script.runSimpleBashScript(rescanCmd);
         if (result == null) {
@@ -364,7 +364,7 @@ public static void rescanForNewVolumes() {
 
     public static String getSystemIdForVolume(String volumeId) {
         //query_vols outputs "VOL-ID <VolumeID> MDM-ID <SystemID>" for a volume with ID: <VolumeID>
-        String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD + " --file " + DRV_CFG_FILE;
+        String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD;
         queryDiskCmd += "|grep " + volumeId + "|awk '{print $4}'";
 
         String result = Script.runSimpleBashScript(queryDiskCmd);
@@ -382,7 +382,7 @@ public static String getSystemIdForVolume(String volumeId) {
     }
 
     public static String getSdcGuid() {
-        String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD + " --file " + DRV_CFG_FILE;
+        String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD;
         String result = Script.runSimpleBashScript(queryGuidCmd);
         if (result == null) {
             LOGGER.warn("Failed to get SDC guid");
@@ -404,7 +404,7 @@ public static String getSdcGuid() {
 
     public static String getSdcId(String mdmId) {
         //query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
-        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
+        String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
         queryMdmsCmd += "|grep " + mdmId + "|awk '{print $5}'";
         String result = Script.runSimpleBashScript(queryMdmsCmd);
         if (result == null) {

From 2be6c48c0422e5cdccf89137568cecc9747c9a25 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Tue, 17 Jun 2025 11:44:24 +0530
Subject: [PATCH 10/14] rebase fixes

---
 .../kvm/storage/ScaleIOStorageAdaptor.java    | 22 +++++++++----------
 .../storage/datastore/util/ScaleIOUtil.java   |  1 -
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index 8a7f216c7e2f..c4d0a2a32ca5 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -183,16 +183,16 @@ private void validateMdmState(Map<String, String> details) {
 
         // be as much verbose as possible, otherwise it will be difficult to troubleshoot operational issue without logs
         if (StringUtils.isEmpty(configValue)) {
-            LOGGER.debug(String.format("Skipped ScaleIO validation as property %s not sent by Management Server", configKey));
+            logger.debug(String.format("Skipped ScaleIO validation as property %s not sent by Management Server", configKey));
         } else if (Boolean.valueOf(configValue).equals(Boolean.FALSE)) {
-            LOGGER.debug(String.format("Skipped ScaleIO validation as property %s received as %s", configKey, configValue));
+            logger.debug(String.format("Skipped ScaleIO validation as property %s received as %s", configKey, configValue));
         } else {
             Collection<String> mdmsConfig = ScaleIOUtil.getMdmsFromConfig();
             Collection<String> mdmsCli = ScaleIOUtil.getMdmsFromCli();
             if (!mdmsCli.equals(mdmsConfig)) {
                 String msg = String.format("MDM addresses from memory and configuration file don't match. " +
                         "Memory values: %s, configuration file values: %s", mdmsCli, mdmsConfig);
-                LOGGER.warn(msg);
+                logger.warn(msg);
                 throw new CloudRuntimeException(msg);
             }
         }
@@ -661,7 +661,7 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
                 if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Ternary<>(false, null, "Failed to add MDMs");
                 } else {
-                    LOGGER.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid));
+                    logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid));
                     applyTimeout(details);
                 }
             }
@@ -692,9 +692,9 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
                     String configValue = details.get(configKey);
 
                     if (StringUtils.isEmpty(configValue)) {
-                        LOGGER.debug(String.format("Configuration key %s not provided", configKey));
+                        logger.debug(String.format("Configuration key %s not provided", configKey));
                     } else {
-                        LOGGER.debug(String.format("Configuration key %s provided as %s", configKey, configValue));
+                        logger.debug(String.format("Configuration key %s provided as %s", configKey, configValue));
                     }
                     Boolean blockUnprepare = Boolean.valueOf(configValue);
                     if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty() && Boolean.TRUE.equals(blockUnprepare)) {
@@ -706,7 +706,7 @@ public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, Str
                 if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
                     return new Pair<>(false, "Failed to remove MDMs, unable to unprepare the SDC client");
                 } else {
-                    LOGGER.debug(String.format("MDMs %s removed from storage pool %s", mdms, uuid));
+                    logger.debug(String.format("MDMs %s removed from storage pool %s", mdms, uuid));
                     applyTimeout(details);
                 }
             }
@@ -735,24 +735,24 @@ private void applyTimeout(Map<String, String> details) {
         String configValue = details.get(configKey);
 
         if (StringUtils.isEmpty(configValue)) {
-            LOGGER.debug(String.format("Apply timeout value not defined in property %s, skip", configKey));
+            logger.debug(String.format("Apply timeout value not defined in property %s, skip", configKey));
             return;
         }
         long timeoutMs;
         try {
             timeoutMs = Long.parseLong(configValue);
         } catch (NumberFormatException e) {
-            LOGGER.warn(String.format("Invalid apply timeout value defined in property %s, skip", configKey), e);
+            logger.warn(String.format("Invalid apply timeout value defined in property %s, skip", configKey), e);
             return;
         }
         if (timeoutMs < 1) {
-            LOGGER.warn(String.format("Apply timeout value is too small (%s ms), skipping", timeoutMs));
+            logger.warn(String.format("Apply timeout value is too small (%s ms), skipping", timeoutMs));
             return;
         }
         try {
             Thread.sleep(timeoutMs);
         } catch (InterruptedException e) {
-            LOGGER.warn(String.format("Apply timeout %s ms interrupted", timeoutMs), e);
+            logger.warn(String.format("Apply timeout %s ms interrupted", timeoutMs), e);
         }
     }
 
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index 54201afa13c2..113659cb62b2 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -17,7 +17,6 @@
 
 package org.apache.cloudstack.storage.datastore.util;
 
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 

From 661082d4cfff7a17340e48e2d8cc079230ee16f1 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Tue, 17 Jun 2025 12:34:08 +0530
Subject: [PATCH 11/14] typo fix

---
 .../apache/cloudstack/storage/datastore/util/ScaleIOUtil.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index 113659cb62b2..cc46e598cbe9 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -206,7 +206,7 @@ public static boolean removeMdms(String... mdmAddresses) {
         boolean removeMdmCliSupported = isRemoveMdmCliSupported();
         boolean restartSDC = false;
         for (String mdmAddress : mdmAddresses) {
-            // continue to next address if current MDM is not present in configuratino
+            // continue to next address if current MDM is not present in configuration
             if (!isMdmPresent(mdmAddress)) {
                 continue;
             }

From 0ae2156792e943d49e6ceb72a7c4320e95cafe12 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Tue, 17 Jun 2025 13:42:04 +0530
Subject: [PATCH 12/14] review fix

---
 utils/src/main/java/com/cloud/utils/script/Script.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java
index 8feeaad61015..993f8f627287 100644
--- a/utils/src/main/java/com/cloud/utils/script/Script.java
+++ b/utils/src/main/java/com/cloud/utils/script/Script.java
@@ -687,7 +687,7 @@ public static Pair<String, String> executeCommand(String command) {
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         String stdErr = script.execute(parser);
         String stdOut = parser.getLines();
-        return new Pair(stdOut, stdErr);
+        return new Pair<>(stdOut, stdErr);
     }
 
     public static int executeCommandForExitValue(long timeout, String... command) {

From da477032c9576419dffb62b60e6060ccd8e40e85 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Wed, 25 Jun 2025 11:49:55 +0530
Subject: [PATCH 13/14] Added wait time after SDC service start/restart/stop,
 retries to fetch SDC id/guid and some code improvements

---
 .../kvm/storage/ScaleIOStorageAdaptor.java    | 40 +++++++++++++++----
 .../ScaleIOPrimaryDataStoreLifeCycle.java     | 27 +++++++------
 .../provider/ScaleIOHostListener.java         | 26 +++++++-----
 .../storage/datastore/util/ScaleIOUtil.java   | 30 ++++++++++++--
 4 files changed, 87 insertions(+), 36 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index c4d0a2a32ca5..e800fc5e8404 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -40,6 +40,7 @@
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
 import org.apache.cloudstack.utils.qemu.QemuObject;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -643,12 +644,13 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
         }
 
         if (!ScaleIOUtil.isSDCServiceActive()) {
+            logger.debug("SDC service is not active on host, starting it");
             if (!ScaleIOUtil.startSDCService()) {
                 return new Ternary<>(false, null, "Couldn't start SDC service on host");
             }
         }
 
-        if (details != null && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
+        if (MapUtils.isNotEmpty(details) && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
             // Assuming SDC service is started, add mdms
             String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
             String[] mdmAddresses = mdms.split(",");
@@ -667,7 +669,12 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
             }
         }
 
-        return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully");
+        Map<String, String> sdcDetails = getSDCDetails(details);
+        if (MapUtils.isEmpty(sdcDetails)) {
+            return new Ternary<>(false, null, "Couldn't get the SDC details on the host");
+        }
+
+        return new Ternary<>(true, sdcDetails, "Prepared client successfully");
     }
 
     public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, String> details) {
@@ -758,20 +765,37 @@ private void applyTimeout(Map<String, String> details) {
 
     private Map<String, String> getSDCDetails(Map<String, String> details) {
         Map<String, String> sdcDetails = new HashMap<String, String>();
-        if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID))  {
+        if (MapUtils.isEmpty(details) || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID))  {
             return sdcDetails;
         }
 
         String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
-        String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
-        if (sdcId != null) {
-            sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
-        } else {
+        if (StringUtils.isEmpty(storageSystemId)) {
+            return sdcDetails;
+        }
+
+        int waitTimeInSecs = 5;
+        int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found
+        do {
+            String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
+            if (sdcId != null) {
+                sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
+                return sdcDetails;
+            }
+
             String sdcGuId = ScaleIOUtil.getSdcGuid();
             if (sdcGuId != null) {
                 sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId);
+                return sdcDetails;
             }
-        }
+
+            try {
+                Thread.sleep(timeBetweenTries);
+            } catch (Exception ignore) {
+            }
+            waitTimeInSecs--;
+        } while (waitTimeInSecs > 0);
+
         return sdcDetails;
     }
 
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 0d9ef608609d..76fced4bd219 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -31,7 +31,6 @@
 import javax.inject.Inject;
 
 import com.cloud.host.HostVO;
-import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
 import org.apache.cloudstack.api.ApiConstants;
 import com.cloud.utils.StringUtils;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
@@ -105,8 +104,6 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy
     @Inject
     private AgentManager agentMgr;
     private ScaleIOSDCManager sdcManager;
-    @Inject
-    private StoragePoolAndAccessGroupMapDao storagePoolAndAccessGroupMapDao;
 
     public ScaleIOPrimaryDataStoreLifeCycle() {
         sdcManager = new ScaleIOSDCManagerImpl();
@@ -306,13 +303,10 @@ public boolean attachZone(DataStore dataStore, ZoneScope scope, Hypervisor.Hyper
 
     @Override
     public boolean maintain(DataStore store) {
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
         if (storagePoolVO != null) {
-            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
-            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
-            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
-
+            populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId());
             StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
             if (systemIdDetail != null) {
                 details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -331,13 +325,10 @@ public boolean maintain(DataStore store) {
 
     @Override
     public boolean cancelMaintain(DataStore store) {
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
         if (storagePoolVO != null) {
-            details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
-            details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
-            details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
-
+            populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId());
             StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
             if (systemIdDetail != null) {
                 details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -420,4 +411,14 @@ private Hypervisor.HypervisorType getHypervisorTypeForCluster(long clusterId) {
     private static boolean isSupportedHypervisorType(Hypervisor.HypervisorType hypervisorType) {
         return Hypervisor.HypervisorType.KVM.equals(hypervisorType);
     }
+
+    private void populateScaleIOConfiguration(Map<String, String> details, long dataCenterId) {
+        if (details == null) {
+            details = new HashMap<>();
+        }
+
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId)));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId)));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId)));
+    }
 }
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index c21ed8518fdc..d569071b82ca 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -27,7 +27,6 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager;
@@ -59,7 +58,6 @@ public class ScaleIOHostListener implements HypervisorHostListener {
     @Inject private DataStoreManager _dataStoreMgr;
     @Inject private HostDao _hostDao;
     @Inject private StoragePoolHostDao _storagePoolHostDao;
-    @Inject private PrimaryDataStoreDao _primaryDataStoreDao;
     @Inject private StoragePoolDetailsDao _storagePoolDetailsDao;
     private ScaleIOSDCManager _sdcManager = new ScaleIOSDCManagerImpl();
 
@@ -109,11 +107,9 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
         if (systemId == null) {
             throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName());
         }
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
-        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
-        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
+        populateScaleIOConfiguration(details,host.getDataCenterId());
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -123,7 +119,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
 
         ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, storagePool, storagePool.getPath(), details);
         ModifyStoragePoolAnswer answer  = sendModifyStoragePoolCommand(cmd, storagePool, host);
-        Map<String,String> poolDetails = answer.getPoolInfo().getDetails();
+        Map<String, String> poolDetails = answer.getPoolInfo().getDetails();
         if (MapUtils.isEmpty(poolDetails)) {
             String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host);
             logger.warn(msg);
@@ -204,11 +200,9 @@ public boolean hostDisconnected(long hostId, long poolId) {
         if (systemId == null) {
             throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName());
         }
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
-        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
-        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
+        populateScaleIOConfiguration(details,host.getDataCenterId());
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.canUnprepareSDC(host, dataStore)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -253,4 +247,14 @@ private String getStoragePoolDetails(StoragePool storagePool) {
         }
         return poolDetails;
     }
+
+    private void populateScaleIOConfiguration(Map<String, String> details, long dataCenterId) {
+        if (details == null) {
+            details = new HashMap<>();
+        }
+
+        details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId)));
+        details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId)));
+        details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId)));
+    }
 }
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index cc46e598cbe9..3700c0a5d219 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -176,7 +176,7 @@ private static boolean removeMdm(String mdmAddress) {
 
     /**
      * Run command, log command result and return {@link Boolean#TRUE} if command succeeded.
-     * FIXME: may need to do refactoring and replace static method callss with dynamic.
+     * FIXME: may need to do refactoring and replace static method calls with dynamic.
      */
     private static boolean runCmd(String command) {
         Pair<String, String> result = Script.executeCommand(command);
@@ -467,17 +467,39 @@ public static boolean enableSDCService() {
 
     public static boolean startSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(3);
+        return true;
     }
 
     public static boolean stopSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(1);
+        return true;
     }
 
     public static boolean restartSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(3);
+        return true;
+    }
+
+    private static void waitForSecs(long waitTimeInSecs) {
+        if (waitTimeInSecs < 0) {
+            waitTimeInSecs = 1;
+        }
+        try {
+            Thread.sleep(waitTimeInSecs * 1000);
+        } catch (InterruptedException ignore) {
+        }
     }
 
     /**

From 38db39eff87ba14ade274bf3be8deeebb12c51aa Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Wed, 25 Jun 2025 12:10:14 +0530
Subject: [PATCH 14/14] Apply suggestions from code review

---
 .../storage/datastore/provider/ScaleIOHostListener.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index d569071b82ca..f4226c6a6d2b 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -109,7 +109,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
         }
         Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        populateScaleIOConfiguration(details,host.getDataCenterId());
+        populateScaleIOConfiguration(details, host.getDataCenterId());
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -202,7 +202,7 @@ public boolean hostDisconnected(long hostId, long poolId) {
         }
         Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
-        populateScaleIOConfiguration(details,host.getDataCenterId());
+        populateScaleIOConfiguration(details, host.getDataCenterId());
         _sdcManager = ComponentContext.inject(_sdcManager);
         if (_sdcManager.canUnprepareSDC(host, dataStore)) {
             details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));