diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java index 06883c708d96..990cefda8f33 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java @@ -31,6 +31,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.template.TemplateProp; +import com.cloud.utils.exception.CloudRuntimeException; @ResourceWrapper(handles = ModifyStoragePoolCommand.class) public final class LibvirtModifyStoragePoolCommandWrapper extends CommandWrapper { @@ -49,11 +50,16 @@ public Answer execute(final ModifyStoragePoolCommand command, final LibvirtCompu return answer; } - final KVMStoragePool storagepool = - storagePoolMgr.createStoragePool(command.getPool().getUuid(), command.getPool().getHost(), command.getPool().getPort(), command.getPool().getPath(), command.getPool() - .getUserInfo(), command.getPool().getType(), command.getDetails()); - if (storagepool == null) { - return new Answer(command, false, " Failed to create storage pool"); + final KVMStoragePool storagepool; + try { + storagepool = + storagePoolMgr.createStoragePool(command.getPool().getUuid(), command.getPool().getHost(), command.getPool().getPort(), command.getPool().getPath(), command.getPool() + .getUserInfo(), command.getPool().getType(), command.getDetails()); + if (storagepool == null) { + return new Answer(command, false, " Failed to create storage pool"); + } + } catch (CloudRuntimeException e) { + return new Answer(command, false, String.format("Failed to create storage pool: %s", e.getLocalizedMessage())); } final Map tInfo = new HashMap(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 29c152f934f6..e800fc5e8404 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -22,11 +22,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.agent.api.PrepareStorageClientCommand; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; @@ -38,6 +40,7 @@ import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -158,10 +161,44 @@ public KVMStoragePool createStoragePool(String uuid, String host, int port, Stri } } } + + validateMdmState(details); + MapStorageUuidToStoragePool.put(uuid, storagePool); return storagePool; } + /** + * Validate Storage Pool state to ensure it healthy and can operate requests. + * There is observed situation where ScaleIO configuration file has different values than ScaleIO CLI. + * Validation compares values from both drv_cfg.txt and drv_cfg CLI and throws exception if there is mismatch. + * + * @param details see {@link PrepareStorageClientCommand#getDetails()} + * and {@link @UnprepareStorageClientCommand#getDetails()}, expected to contain + * {@link ScaleIOSDCManager#ValidateMdmsOnConnect#key()} + * @throws CloudRuntimeException in case if Storage Pool is not operate-able + */ + private void validateMdmState(Map details) { + String configKey = ScaleIOSDCManager.ValidateMdmsOnConnect.key(); + String configValue = details.get(configKey); + + // be as much verbose as possible, otherwise it will be difficult to troubleshoot operational issue without logs + if (StringUtils.isEmpty(configValue)) { + logger.debug(String.format("Skipped ScaleIO validation as property %s not sent by Management Server", configKey)); + } else if (Boolean.valueOf(configValue).equals(Boolean.FALSE)) { + logger.debug(String.format("Skipped ScaleIO validation as property %s received as %s", configKey, configValue)); + } else { + Collection mdmsConfig = ScaleIOUtil.getMdmsFromConfig(); + Collection mdmsCli = ScaleIOUtil.getMdmsFromCli(); + if (!mdmsCli.equals(mdmsConfig)) { + String msg = String.format("MDM addresses from memory and configuration file don't match. " + + "Memory values: %s, configuration file values: %s", mdmsCli, mdmsConfig); + logger.warn(msg); + throw new CloudRuntimeException(msg); + } + } + } + @Override public boolean deleteStoragePool(String uuid) { ScaleIOStoragePool storagePool = (ScaleIOStoragePool) MapStorageUuidToStoragePool.get(uuid); @@ -607,28 +644,37 @@ public Ternary, String> prepareStorageClient(String } if (!ScaleIOUtil.isSDCServiceActive()) { + logger.debug("SDC service is not active on host, starting it"); if (!ScaleIOUtil.startSDCService()) { return new Ternary<>(false, null, "Couldn't start SDC service on host"); } } - if (details != null && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) { + if (MapUtils.isNotEmpty(details) && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) { // Assuming SDC service is started, add mdms String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); String[] mdmAddresses = mdms.split(","); if (mdmAddresses.length > 0) { - if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) { + if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); } - ScaleIOUtil.addMdms(Arrays.asList(mdmAddresses)); - if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) { + ScaleIOUtil.addMdms(mdmAddresses); + if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { return new Ternary<>(false, null, "Failed to add MDMs"); + } else { + logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); + applyTimeout(details); } } } - return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully"); + Map sdcDetails = getSDCDetails(details); + if (MapUtils.isEmpty(sdcDetails)) { + return new Ternary<>(false, null, "Couldn't get the SDC details on the host"); + } + + return new Ternary<>(true, sdcDetails, "Prepared client successfully"); } public Pair unprepareStorageClient(String uuid, Map details) { @@ -646,36 +692,110 @@ public Pair unprepareStorageClient(String uuid, Map 0) { - if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) { + if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { return new Pair<>(true, "MDM not added, no need to unprepare the SDC client"); + } else { + String configKey = ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(); + String configValue = details.get(configKey); + + if (StringUtils.isEmpty(configValue)) { + logger.debug(String.format("Configuration key %s not provided", configKey)); + } else { + logger.debug(String.format("Configuration key %s provided as %s", configKey, configValue)); + } + Boolean blockUnprepare = Boolean.valueOf(configValue); + if (!ScaleIOUtil.isRemoveMdmCliSupported() && !ScaleIOUtil.getVolumeIds().isEmpty() && Boolean.TRUE.equals(blockUnprepare)) { + return new Pair<>(false, "Failed to remove MDMs, SDC client requires service to be restarted, but there are Volumes attached to the Host"); + } } - ScaleIOUtil.removeMdms(Arrays.asList(mdmAddresses)); - if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) { + ScaleIOUtil.removeMdms(mdmAddresses); + if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { return new Pair<>(false, "Failed to remove MDMs, unable to unprepare the SDC client"); + } else { + logger.debug(String.format("MDMs %s removed from storage pool %s", mdms, uuid)); + applyTimeout(details); } } } + /* + * TODO: + * 1. Verify on-demand is true + * 2. If on-demand is true check whether other MDM addresses are still present + * 3. If there are no MDM addresses, then stop SDC service. + */ + return new Pair<>(true, "Unprepared SDC client successfully"); } + /** + * Check whether details map has timeout configured and do "apply timeout" pause before returning response + * (to have ScaleIO changes applied). + * + * @param details see {@link PrepareStorageClientCommand#getDetails()} + * and {@link @UnprepareStorageClientCommand#getDetails()}, expected to contain + * {@link ScaleIOSDCManager#MdmsChangeApplyTimeout#key()} + */ + private void applyTimeout(Map details) { + String configKey = ScaleIOSDCManager.MdmsChangeApplyTimeout.key(); + String configValue = details.get(configKey); + + if (StringUtils.isEmpty(configValue)) { + logger.debug(String.format("Apply timeout value not defined in property %s, skip", configKey)); + return; + } + long timeoutMs; + try { + timeoutMs = Long.parseLong(configValue); + } catch (NumberFormatException e) { + logger.warn(String.format("Invalid apply timeout value defined in property %s, skip", configKey), e); + return; + } + if (timeoutMs < 1) { + logger.warn(String.format("Apply timeout value is too small (%s ms), skipping", timeoutMs)); + return; + } + try { + Thread.sleep(timeoutMs); + } catch (InterruptedException e) { + logger.warn(String.format("Apply timeout %s ms interrupted", timeoutMs), e); + } + } + private Map getSDCDetails(Map details) { Map sdcDetails = new HashMap(); - if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { + if (MapUtils.isEmpty(details) || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { return sdcDetails; } String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - String sdcId = ScaleIOUtil.getSdcId(storageSystemId); - if (sdcId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); - } else { + if (StringUtils.isEmpty(storageSystemId)) { + return sdcDetails; + } + + int waitTimeInSecs = 5; + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found + do { + String sdcId = ScaleIOUtil.getSdcId(storageSystemId); + if (sdcId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); + return sdcDetails; + } + String sdcGuId = ScaleIOUtil.getSdcGuid(); if (sdcGuId != null) { sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); + return sdcDetails; } - } + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ignore) { + } + waitTimeInSecs--; + } while (waitTimeInSecs > 0); + return sdcDetails; } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java index 421fee096341..eddaa8f64996 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java @@ -194,8 +194,12 @@ public void testUnprepareStorageClient_RemoveMDMFailed() { details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2"); when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3); when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0); + when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d' /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null)); when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0); when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2"); + when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null)); + when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols"))).thenReturn(new Pair<>("", null)); + Pair result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java index 2dc5acffcbc1..77a8677e6b14 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java @@ -38,6 +38,9 @@ public interface ScaleIOGatewayClient { String GATEWAY_API_PASSWORD = "powerflex.gw.password"; String STORAGE_POOL_NAME = "powerflex.storagepool.name"; String STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id"; + /** + * Storage Pool Metadata Management (MDM) IP address(es). + */ String STORAGE_POOL_MDMS = "powerflex.storagepool.mdms"; String SDC_ID = "powerflex.sdc.id"; String SDC_GUID = "powerflex.sdc.guid"; diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java index 461992be1022..76fced4bd219 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java @@ -31,7 +31,6 @@ import javax.inject.Inject; import com.cloud.host.HostVO; -import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao; import org.apache.cloudstack.api.ApiConstants; import com.cloud.utils.StringUtils; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -105,8 +104,6 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy @Inject private AgentManager agentMgr; private ScaleIOSDCManager sdcManager; - @Inject - private StoragePoolAndAccessGroupMapDao storagePoolAndAccessGroupMapDao; public ScaleIOPrimaryDataStoreLifeCycle() { sdcManager = new ScaleIOSDCManagerImpl(); @@ -306,14 +303,18 @@ public boolean attachZone(DataStore dataStore, ZoneScope scope, Hypervisor.Hyper @Override public boolean maintain(DataStore store) { - Map details = new HashMap<>(); - StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - if (systemIdDetail != null) { - details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue()); - StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS); - if (mdmsDetail != null) { - details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdmsDetail.getValue()); - details.put(ScaleIOSDCManager.ConnectOnDemand.key(), "false"); + Map details = new HashMap<>(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId()); + if (storagePoolVO != null) { + populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId()); + StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + if (systemIdDetail != null) { + details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue()); + StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS); + if (mdmsDetail != null) { + details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdmsDetail.getValue()); + details.put(ScaleIOSDCManager.ConnectOnDemand.key(), "false"); + } } } @@ -324,14 +325,15 @@ public boolean maintain(DataStore store) { @Override public boolean cancelMaintain(DataStore store) { - Map details = new HashMap<>(); - StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - if (systemIdDetail != null) { - details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue()); - sdcManager = ComponentContext.inject(sdcManager); - if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) { - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId()); - if (storagePoolVO != null) { + Map details = new HashMap<>(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId()); + if (storagePoolVO != null) { + populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId()); + StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + if (systemIdDetail != null) { + details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue()); + sdcManager = ComponentContext.inject(sdcManager); + if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) { details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(storagePoolVO.getDataCenterId()))); StoragePoolDetailVO mdmsDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_MDMS); if (mdmsDetail != null) { @@ -409,4 +411,14 @@ private Hypervisor.HypervisorType getHypervisorTypeForCluster(long clusterId) { private static boolean isSupportedHypervisorType(Hypervisor.HypervisorType hypervisorType) { return Hypervisor.HypervisorType.KVM.equals(hypervisorType); } + + private void populateScaleIOConfiguration(Map details, long dataCenterId) { + if (details == null) { + details = new HashMap<>(); + } + + details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId))); + details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId))); + details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId))); + } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java index b02732051b5e..c4a59fe81abe 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java @@ -32,6 +32,34 @@ public interface ScaleIOSDCManager { Boolean.TRUE, ConfigKey.Scope.Zone); + /** + * Timeout for Host to wait after MDM changes made on Host until changes will be applied. + * Needed to avoid cases when Storage Pool is not connected yet, but Agent already starts to use Storage Pool. + */ + ConfigKey MdmsChangeApplyTimeout = new ConfigKey<>("Storage", + Integer.class, + "powerflex.mdm.change.apply.timeout.ms", + "1000", + "Timeout (in ms) for Host to wait after MDM changes made on Host until changes will be applied, default value: 1000 ms", + Boolean.TRUE, + ConfigKey.Scope.Zone); + + ConfigKey ValidateMdmsOnConnect = new ConfigKey<>("Storage", + Boolean.class, + "powerflex.mdm.validate.on.connect", + Boolean.FALSE.toString(), + "Flag to validate MDMs on Host, present in configuration file and in CLI, default value: false", + Boolean.TRUE, + ConfigKey.Scope.Zone); + + ConfigKey BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached = new ConfigKey<>("Storage", + Boolean.class, + "powerflex.block.sdc.unprepare", + Boolean.FALSE.toString(), + "Block Storage Client un-preparation if SDC service restart needed but there are Volumes attached to the Host", + Boolean.TRUE, + ConfigKey.Scope.Zone); + /** * Checks SDC connections limit. * @param storagePoolId the storage pool id diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java index 79f82a1e0df0..dec009bd0e1a 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java @@ -204,9 +204,11 @@ public String prepareSDC(Host host, DataStore dataStore) { private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId, String mdms) { logger.debug("Preparing SDC on the host {}", host); - Map details = new HashMap<>(); + Map details = new HashMap<>(); details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms); + details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.valueIn(host.getDataCenterId()))); + PrepareStorageClientCommand cmd = new PrepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details); int timeoutSeconds = 60; cmd.setWait(timeoutSeconds); @@ -324,6 +326,7 @@ private boolean unprepareSDCOnHost(Host host, DataStore dataStore, String mdms) logger.debug(String.format("Unpreparing SDC on the host %s (%s)", host.getId(), host.getName())); Map details = new HashMap<>(); details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms); + details.put(MdmsChangeApplyTimeout.key(), String.valueOf(MdmsChangeApplyTimeout.valueIn(host.getDataCenterId()))); UnprepareStorageClientCommand cmd = new UnprepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(), dataStore.getUuid(), details); int timeoutSeconds = 60; cmd.setWait(timeoutSeconds); @@ -477,6 +480,6 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[]{ConnectOnDemand}; + return new ConfigKey[]{ConnectOnDemand, MdmsChangeApplyTimeout, ValidateMdmsOnConnect, BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached}; } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java index 0db329204e8c..f4226c6a6d2b 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java @@ -27,7 +27,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager; @@ -59,7 +58,6 @@ public class ScaleIOHostListener implements HypervisorHostListener { @Inject private DataStoreManager _dataStoreMgr; @Inject private HostDao _hostDao; @Inject private StoragePoolHostDao _storagePoolHostDao; - @Inject private PrimaryDataStoreDao _primaryDataStoreDao; @Inject private StoragePoolDetailsDao _storagePoolDetailsDao; private ScaleIOSDCManager _sdcManager = new ScaleIOSDCManagerImpl(); @@ -109,8 +107,9 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) { if (systemId == null) { throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName()); } - Map details = new HashMap<>(); + Map details = new HashMap<>(); details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); + populateScaleIOConfiguration(details, host.getDataCenterId()); _sdcManager = ComponentContext.inject(_sdcManager); if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) { details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId()))); @@ -120,7 +119,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) { ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, storagePool, storagePool.getPath(), details); ModifyStoragePoolAnswer answer = sendModifyStoragePoolCommand(cmd, storagePool, host); - Map poolDetails = answer.getPoolInfo().getDetails(); + Map poolDetails = answer.getPoolInfo().getDetails(); if (MapUtils.isEmpty(poolDetails)) { String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host); logger.warn(msg); @@ -201,8 +200,9 @@ public boolean hostDisconnected(long hostId, long poolId) { if (systemId == null) { throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName()); } - Map details = new HashMap<>(); + Map details = new HashMap<>(); details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); + populateScaleIOConfiguration(details, host.getDataCenterId()); _sdcManager = ComponentContext.inject(_sdcManager); if (_sdcManager.canUnprepareSDC(host, dataStore)) { details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId()))); @@ -247,4 +247,14 @@ private String getStoragePoolDetails(StoragePool storagePool) { } return poolDetails; } + + private void populateScaleIOConfiguration(Map details, long dataCenterId) { + if (details == null) { + details = new HashMap<>(); + } + + details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId))); + details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId))); + details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId))); + } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java index a8c02a90bf9e..3700c0a5d219 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java @@ -17,16 +17,25 @@ package org.apache.cloudstack.storage.datastore.util; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import com.cloud.utils.Pair; import com.cloud.utils.UuidUtils; import com.cloud.utils.script.Script; import org.apache.commons.lang3.StringUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class ScaleIOUtil { protected static Logger LOGGER = LogManager.getLogger(ScaleIOUtil.class); @@ -39,7 +48,7 @@ public class ScaleIOUtil { public static final String VMSNAPSHOT_PREFIX = "vmsnap"; public static final int IDENTIFIER_LENGTH = 16; - public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = Long.valueOf(10); + public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = 10L; public static final String DISK_PATH = "/dev/disk/by-id"; public static final String DISK_NAME_PREFIX = "emc-vol-"; @@ -63,9 +72,10 @@ public class ScaleIOUtil { private static final String SDC_SERVICE_ENABLE_CMD = "systemctl enable scini"; public static final String CONNECTED_SDC_COUNT_STAT = "ConnectedSDCCount"; + /** * Cmd for querying volumes in SDC - * Sample output for cmd: drv_cfg --query_vols: + * Sample output for cmd {@code drv_cfg --query_vols}: * Retrieved 2 volume(s) * VOL-ID 6c33633100000009 MDM-ID 218ce1797566a00f * VOL-ID 6c3362a30000000a MDM-ID 218ce1797566a00f @@ -74,14 +84,14 @@ public class ScaleIOUtil { /** * Cmd for querying guid in SDC - * Sample output for cmd: drv_cfg --query_guid: + * Sample output for cmd {@code drv_cfg --query_guid}: * B0E3BFB8-C20B-43BF-93C8-13339E85AA50 */ private static final String QUERY_GUID_CMD = "drv_cfg --query_guid"; /** * Cmd for querying MDMs in SDC - * Sample output for cmd: drv_cfg --query_mdms: + * Sample output for cmd {@code drv_cfg --query_mdms}: * Retrieved 2 mdm(s) * MDM-ID 3ef46cbf2aaf5d0f SDC ID 6b18479c00000003 INSTALLATION ID 68ab55462cbb3ae4 IPs [0]-x.x.x.x [1]-x.x.x.x * MDM-ID 2e706b2740ec200f SDC ID 301b852c00000003 INSTALLATION ID 33f8662e7a5c1e6c IPs [0]-x.x.x.x [1]-x.x.x.x @@ -89,61 +99,251 @@ public class ScaleIOUtil { private static final String QUERY_MDMS_CMD = "drv_cfg --query_mdms"; private static final String ADD_MDMS_CMD = "drv_cfg --add_mdm"; + + private static final String REMOVE_MDM_PARAMETER = "--remove_mdm"; + + /** + * Calls the kernel module to remove MDM. + */ + private static final String REMOVE_MDMS_CMD = "drv_cfg " + REMOVE_MDM_PARAMETER; + + /** + * Command to get back "Usage" response. As of now it is just drv_cfg without parameters. + */ + private static final String USAGE_CMD = "drv_cfg"; + private static final String DRV_CFG_FILE = "/etc/emc/scaleio/drv_cfg.txt"; - public static void addMdms(List mdmAddresses) { - if (CollectionUtils.isEmpty(mdmAddresses)) { - return; + /** + * Sample Command - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt + */ + private static final String REMOVE_MDM_CMD_TEMPLATE = "sed -i '/%s\\,/d' %s"; + + /** + * Patterns to parse {@link ScaleIOUtil#DRV_CFG_FILE} and {@code --query_mdms} command output. + * The format is: + * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs [{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ... + */ + private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\\r?\\n"); + /** + * Pattern to find "IPs" substring in {@link ScaleIOUtil#QUERY_MDMS_CMD} command output. + * The output format is: + * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs [{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ... + */ + private static final Pattern USAGE_IPS_LINE_PATTERN = Pattern.compile("IPs\\s*(.*)$"); + /** + * Pattern to find individual IP address in {@link ScaleIOUtil#QUERY_MDMS_CMD} command output. + */ + private static final Pattern USAGE_IP_TOKEN_PATTERN = Pattern.compile("(\\s*\\[\\d\\]\\-)([^\\s$]+)"); + + /** + * Pattern to find Volume ID in {@link ScaleIOUtil#QUERY_VOLUMES_CMD} + */ + private static final Pattern VOLUME_ID_TOKEN_PATTERN = Pattern.compile("VOL-ID\\s*([^\\s$]+)"); + /** + * Pattern to find MDM entries line in {@link ScaleIOUtil#DRV_CFG_FILE}. + */ + private static final Pattern DRV_CFG_MDM_LINE_PATTERN = Pattern.compile("^mdm\\s*([0-9A-F:\\.,]+)$", Pattern.CASE_INSENSITIVE); + /** + * Pattern to split comma separated string of IP addresses (space aware). + */ + private static final Pattern DRV_CFG_MDM_IPS_PATTERN = Pattern.compile("\\s*,\\s*"); + + public static boolean addMdms(String... mdmAddresses) { + if (mdmAddresses.length < 1) { + return false; } // Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --add_mdm --ip x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt - String addMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.ADD_MDMS_CMD; - addMdmsCmd += " --ip " + String.join(",", mdmAddresses); - addMdmsCmd += " --file " + DRV_CFG_FILE; - String result = Script.runSimpleBashScript(addMdmsCmd); - if (result == null) { - LOGGER.warn("Failed to add mdms"); + String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.ADD_MDMS_CMD; + command += " --ip " + String.join(",", mdmAddresses); + command += " --file " + DRV_CFG_FILE; + return runCmd(command); + } + + /** + * Remove MDM via ScaleIO via CLI. + * + * @param mdmAddress MDM address to remove + * @return true if IP address successfully removed + */ + private static boolean removeMdm(String mdmAddress) { + // Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --remove_mdm --ip x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt + String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.REMOVE_MDMS_CMD; + command += " --ip " + String.join(",", mdmAddress); + command += " --file " + DRV_CFG_FILE; + return runCmd(command); + } + + /** + * Run command, log command result and return {@link Boolean#TRUE} if command succeeded. + * FIXME: may need to do refactoring and replace static method calls with dynamic. + */ + private static boolean runCmd(String command) { + Pair result = Script.executeCommand(command); + String stdOut = result.first(); + String stdErr = result.second(); + boolean succeeded = StringUtils.isEmpty(stdErr); + if (succeeded) { + LOGGER.debug(String.format("Successfully executed command '%s': %s", command, stdOut)); + } else { + LOGGER.warn(String.format("Failed to execute command '%s': %s", command, stdErr)); } + return succeeded; } - public static void removeMdms(List mdmAddresses) { - if (CollectionUtils.isEmpty(mdmAddresses)) { - return; + /** + * Remove MDMs either via ScaleIO CLI (if supported) or by updating configuration file. + * + * @param mdmAddresses MDM addresses + * @return returns {@link Boolean#TRUE} if changes were applied to the configuration + */ + public static boolean removeMdms(String... mdmAddresses) { + if (mdmAddresses.length < 1) { + return false; } - // (i) Remove MDMs from config file (ii) Restart scini - // Sample Cmd - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt + + boolean changesApplied = false; + boolean removeMdmCliSupported = isRemoveMdmCliSupported(); boolean restartSDC = false; - String removeMdmsCmdFormat = "sed -i '/%s\\,/d' %s"; for (String mdmAddress : mdmAddresses) { - if (mdmAdded(mdmAddress)) { - restartSDC = true; + // continue to next address if current MDM is not present in configuration + if (!isMdmPresent(mdmAddress)) { + continue; + } + // remove MDM via CLI if it is supported + if (removeMdmCliSupported) { + if (removeMdm(mdmAddress)) { + changesApplied = true; + } + } else { + String command = String.format(REMOVE_MDM_CMD_TEMPLATE, mdmAddress, DRV_CFG_FILE); + String stdErr = Script.executeCommand(command).second(); + if(StringUtils.isEmpty(stdErr)) { + // restart SDC needed only if configuration file modified manually (not by CLI) + restartSDC = true; + changesApplied = true; + } else { + LOGGER.error(String.format("Failed to remove MDM %s from %s: %s", mdmAddress, DRV_CFG_FILE, stdErr)); + } } - String removeMdmsCmd = String.format(removeMdmsCmdFormat, mdmAddress, DRV_CFG_FILE); - Script.runSimpleBashScript(removeMdmsCmd); } if (restartSDC) { restartSDCService(); } + return changesApplied; + } + + /** + * Returns MDM entries from {@link ScaleIOUtil#DRV_CFG_FILE}. + */ + public static Collection getMdmsFromConfig() { + List configFileLines; + try { + configFileLines = Files.readAllLines(Path.of(DRV_CFG_FILE)); + } catch (IOException e) { + LOGGER.error(String.format("Failed to read MDMs from %s", DRV_CFG_FILE), e); + return List.of(); + } + Set mdms = new LinkedHashSet<>(); + for (String line : configFileLines) { + Matcher mdmLineMatcher = DRV_CFG_MDM_LINE_PATTERN.matcher(line); + if(mdmLineMatcher.find() && mdmLineMatcher.groupCount() > 0) { + String mdmLine = mdmLineMatcher.group(1); + String[] mdmValues = DRV_CFG_MDM_IPS_PATTERN.split(mdmLine); + mdms.addAll(Arrays.asList(mdmValues)); + } + } + return mdms; + } + /** + * Returns Volume Ids from {@link ScaleIOUtil#DRV_CFG_FILE}. + */ + public static Collection getVolumeIds() { + String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD; + Pair result = Script.executeCommand(command); + String stdOut = result.first(); + + Set volumeIds = new LinkedHashSet<>(); + String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut); + for (String line : stdOutLines) { + Matcher volumeIdMatcher = VOLUME_ID_TOKEN_PATTERN.matcher(line); + if (volumeIdMatcher.find() && volumeIdMatcher.groupCount() > 0) { + volumeIds.add(volumeIdMatcher.group(1)); + } + } + return volumeIds; + } + + /** + * Returns MDM entries from CLI using {@code --query_mdms}. + */ + public static Collection getMdmsFromCli() { + String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD; + Pair result = Script.executeCommand(command); + String stdOut = result.first(); + + Set mdms = new LinkedHashSet<>(); + String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut); + for (String line : stdOutLines) { + Matcher ipsLineMatcher = USAGE_IPS_LINE_PATTERN.matcher(line); + if (ipsLineMatcher.find() && ipsLineMatcher.groupCount() > 0) { + String ipToken = ipsLineMatcher.group(1); + Matcher ipMatcher = USAGE_IP_TOKEN_PATTERN.matcher(ipToken); + while (ipMatcher.find()) { + if (ipMatcher.groupCount() > 1) { + mdms.add(ipMatcher.group(2)); + } + } + } + } + return mdms; + } + + /** + * Returns {@link Boolean#TRUE} if ScaleIO CLI tool (drv_cfg) supports MDMs removal. + */ + public static boolean isRemoveMdmCliSupported() { + /* + * New version of drv_cfg supports remove mdm API. + * Instead of defining supported version and checking it, the logic is to check drv_cfg "Usage" output + * and see whether remove_mdm command supported. + * The "Usage" returned if tool executed without parameters or with invalid parameters. + */ + String command = SDC_HOME_PATH + "/bin/" + USAGE_CMD; + + Pair result = Script.executeCommand(command); + String stdErr = result.first(); + String stdOut = result.second(); + + /* + * Check whether stderr or stdout contains mdm removal "--remove_mdm" parameter. + * + * Current version returns "Usage" in stderr, check stdout as well in case this will be changed in the future, + * as returned "Usage" is not an error. + */ + return (stdOut + stdErr).toLowerCase().contains(REMOVE_MDM_PARAMETER); } - public static boolean mdmAdded(String mdmAddress) { + /** + * Returns true if provided MDM address is present in configuration. + */ + public static boolean isMdmPresent(String mdmAddress) { //query_mdms outputs "MDM-ID SDC ID INSTALLATION ID IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD; queryMdmsCmd += "|grep " + mdmAddress; String result = Script.runSimpleBashScript(queryMdmsCmd); - if (StringUtils.isNotBlank(result) && result.contains(mdmAddress)) { - return true; - } - return false; + + return StringUtils.isNotBlank(result) && result.contains(mdmAddress); } public static String getSdcHomePath() { - String sdcHomePath = DEFAULT_SDC_HOME_PATH; String sdcHomePropertyCmdFormat = "sed -n '/%s/p' '%s' 2>/dev/null | sed 's/%s=//g' 2>/dev/null"; String sdcHomeCmd = String.format(sdcHomePropertyCmdFormat, SDC_HOME_PARAMETER, AGENT_PROPERTIES_FILE, SDC_HOME_PARAMETER); - String result = Script.runSimpleBashScript(sdcHomeCmd); + String sdcHomePath; if (result == null) { - LOGGER.warn("Failed to get sdc home path from agent.properties, fallback to default path"); + sdcHomePath = DEFAULT_SDC_HOME_PATH; + LOGGER.warn(String.format("Failed to get sdc home path from agent.properties, fallback to default path %s", sdcHomePath)); } else { sdcHomePath = result; } @@ -151,7 +351,7 @@ public static String getSdcHomePath() { return sdcHomePath; } - public static final void rescanForNewVolumes() { + public static void rescanForNewVolumes() { // Detecting new volumes String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD; @@ -161,7 +361,7 @@ public static final void rescanForNewVolumes() { } } - public static final String getSystemIdForVolume(String volumeId) { + public static String getSystemIdForVolume(String volumeId) { //query_vols outputs "VOL-ID MDM-ID " for a volume with ID: String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD; queryDiskCmd += "|grep " + volumeId + "|awk '{print $4}'"; @@ -225,7 +425,7 @@ public static String getSdcId(String mdmId) { return result; } - public static final String getVolumePath(String volumePathWithName) { + public static String getVolumePath(String volumePathWithName) { if (StringUtils.isEmpty(volumePathWithName)) { return volumePathWithName; } @@ -237,7 +437,7 @@ public static final String getVolumePath(String volumePathWithName) { return volumePathWithName; } - public static final String updatedPathWithVolumeName(String volumePath, String volumeName) { + public static String updatedPathWithVolumeName(String volumePath, String volumeName) { if (StringUtils.isAnyEmpty(volumePath, volumeName)) { return volumePath; } @@ -267,16 +467,73 @@ public static boolean enableSDCService() { public static boolean startSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSecs(3); + return true; } public static boolean stopSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSecs(1); + return true; } public static boolean restartSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSecs(3); + return true; + } + + private static void waitForSecs(long waitTimeInSecs) { + if (waitTimeInSecs < 0) { + waitTimeInSecs = 1; + } + try { + Thread.sleep(waitTimeInSecs * 1000); + } catch (InterruptedException ignore) { + } + } + + /** + * Represents {@link ScaleIOUtil#DRV_CFG_FILE} MDM entry (SDC and Installation Ids are skipped). + */ + public static class MdmEntry { + private String mdmId; + private Collection ips; + + /** + * MDM entry constructor. + * + * @param mdmId MDM Id + * @param ips IP Addresses + */ + public MdmEntry(String mdmId, Collection ips) { + this.mdmId = mdmId; + this.ips = ips; + } + + public String getMdmId() { + return mdmId; + } + + public void setMdmId(String mdmId) { + this.mdmId = mdmId; + } + + public Collection getIps() { + return ips; + } + + public void setIps(Collection ips) { + this.ips = ips; + } } } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index a1104b37c27b..993f8f627287 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -671,6 +671,25 @@ public static String executeCommand(String... command) { return runScript(getScriptForCommandRun(command)); } + /** + * Execute command and return standard output and standard error. + * + * @param command OS command to be executed + * @return {@link Pair} with standard output as first and standard error as second field + */ + public static Pair executeCommand(String command) { + // wrap command into bash + Script script = new Script("/bin/bash"); + script.add("-c"); + script.add(command); + + // parse all lines from the output + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String stdErr = script.execute(parser); + String stdOut = parser.getLines(); + return new Pair<>(stdOut, stdErr); + } + public static int executeCommandForExitValue(long timeout, String... command) { return runScriptForExitValue(getScriptForCommandRun(timeout, command)); }