diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e5593f104608..c781c07c227f 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -823,7 +823,7 @@ public static class Property { private T defaultValue; private Class typeClass; - Property(String name, T value) { + public Property(String name, T value) { init(name, value); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 335ea0d03d26..195ce6c9984b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -581,14 +582,23 @@ public Ternary, String> prepareStorageClient(Storag } if (!ScaleIOUtil.isSDCServiceActive()) { + logger.debug("SDC service is not active on host, starting it"); if (!ScaleIOUtil.startSDCService()) { return new Ternary<>(false, null, "Couldn't start SDC service on host"); } - } else if (!ScaleIOUtil.restartSDCService()) { - return new Ternary<>(false, null, "Couldn't restart SDC service on host"); + } else { + logger.debug("SDC service is active on host, re-starting it"); + if (!ScaleIOUtil.restartSDCService()) { + return new Ternary<>(false, null, "Couldn't restart SDC service on host"); + } + } + + Map sdcDetails = getSDCDetails(details); + if (MapUtils.isEmpty(sdcDetails)) { + return new Ternary<>(false, null, "Couldn't get the SDC details on the host"); } - return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully"); + return new Ternary<>( true, sdcDetails, "Prepared client successfully"); } public Pair unprepareStorageClient(Storage.StoragePoolType type, String uuid) { @@ -611,20 +621,40 @@ public Pair unprepareStorageClient(Storage.StoragePoolType type private Map getSDCDetails(Map details) { Map sdcDetails = new HashMap(); - if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { + if (MapUtils.isEmpty(details) || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { return sdcDetails; } String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - String sdcId = ScaleIOUtil.getSdcId(storageSystemId); - if (sdcId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); - } else { - String sdcGuId = ScaleIOUtil.getSdcGuid(); - if (sdcGuId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); - } + if (StringUtils.isEmpty(storageSystemId)) { + return sdcDetails; } + + int numberOfTries = 5; + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found + int attempt = 1; + do { + logger.debug("Get SDC details, attempt #{}", attempt); + String sdcId = ScaleIOUtil.getSdcId(storageSystemId); + if (sdcId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); + return sdcDetails; + } else { + String sdcGuId = ScaleIOUtil.getSdcGuid(); + if (sdcGuId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); + return sdcDetails; + } + } + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ignore) { + } + numberOfTries--; + attempt++; + } while (numberOfTries > 0); + return sdcDetails; } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java index 07aea0cfbee5..c2002f565601 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java @@ -116,9 +116,9 @@ public void testPrepareStorageClient_SDCServiceRestarted() { Ternary, String> result = scaleIOStorageAdaptor.prepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, new HashMap<>()); - Assert.assertTrue(result.first()); - Assert.assertNotNull(result.second()); - Assert.assertTrue(result.second().isEmpty()); + Assert.assertFalse(result.first()); + Assert.assertNull(result.second()); + Assert.assertEquals("Couldn't get the SDC details on the host", result.third()); } @Test diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java index f1177acc7b4a..5f098badaa1b 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java @@ -183,12 +183,13 @@ public String prepareSDC(Host host, DataStore dataStore) { storagePoolHost.setLocalPath(sdcId); storagePoolHostDao.update(storagePoolHost.getId(), storagePoolHost); } - } - int waitTimeInSecs = 15; // Wait for 15 secs (usual tests with SDC service start took 10-15 secs) - if (hostSdcConnected(sdcId, dataStore, waitTimeInSecs)) { - return sdcId; + int waitTimeInSecs = 15; // Wait for 15 secs (usual tests with SDC service start took 10-15 secs) + if (hostSdcConnected(sdcId, dataStore, waitTimeInSecs)) { + return sdcId; + } } + return null; } finally { if (storageSystemIdLock != null) { @@ -246,7 +247,7 @@ private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId) } if (StringUtils.isBlank(sdcId)) { - logger.warn("Couldn't retrieve PowerFlex storage SDC details from the host: {}, try (re)install SDC and restart agent", host); + logger.warn("Couldn't retrieve PowerFlex storage SDC details from the host: {}, add MDMs if not or try (re)install SDC & restart agent", host); return null; } @@ -381,6 +382,9 @@ private boolean isHostSdcConnected(String sdcId, long poolId) { private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception { StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); + if (storagePool == null) { + throw new CloudRuntimeException("Unable to find the storage pool with id " + storagePoolId); + } return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePool, storagePoolDetailsDao); } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java index 5fc4868902eb..4f4400ffacca 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java @@ -102,12 +102,12 @@ private String getSdcIdOfHost(HostVO host, StoragePool storagePool) { if (systemId == null) { throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName()); } - Map details = new HashMap<>(); + Map details = new HashMap<>(); details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, storagePool, storagePool.getPath(), details); ModifyStoragePoolAnswer answer = sendModifyStoragePoolCommand(cmd, storagePool, host); - Map poolDetails = answer.getPoolInfo().getDetails(); + Map poolDetails = answer.getPoolInfo().getDetails(); if (MapUtils.isEmpty(poolDetails)) { String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host); logger.warn(msg); @@ -124,7 +124,7 @@ private String getSdcIdOfHost(HostVO host, StoragePool storagePool) { } if (StringUtils.isBlank(sdcId)) { - String msg = String.format("Couldn't retrieve PowerFlex storage SDC details from the host: %s, (re)install SDC and restart agent", host); + String msg = String.format("Couldn't retrieve PowerFlex storage SDC details from the host: %s, add MDMs if not or try (re)install SDC & restart agent", host); logger.warn(msg); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " + host.getUuid(), msg); return null; diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java index 4bb8df9b60db..d91321a907f1 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.storage.datastore.util; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -60,6 +62,14 @@ public class ScaleIOUtil { private static final String SDC_SERVICE_ENABLE_CMD = "systemctl enable scini"; public static final String CONNECTED_SDC_COUNT_STAT = "ConnectedSDCCount"; + + /** + * Time (in seconds) to wait after SDC service 'scini' start/restart/stop.
+ * Data type: Integer.
+ * Default value: 3 + */ + public static final AgentProperties.Property SDC_SERVICE_ACTION_WAIT = new AgentProperties.Property<>("powerflex.sdc.service.wait", 3); + /** * Cmd for querying volumes in SDC * Sample output for cmd: drv_cfg --query_vols: @@ -216,16 +226,41 @@ public static boolean enableSDCService() { public static boolean startSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSdcServiceActionToComplete(); + return true; } public static boolean stopSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSdcServiceActionToComplete(); + return true; } public static boolean restartSDCService() { int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD); - return exitValue == 0; + if (exitValue != 0) { + return false; + } + waitForSdcServiceActionToComplete(); + return true; + } + + private static void waitForSdcServiceActionToComplete() { + // Wait for the SDC service to settle after start/restart/stop and reaches a stable state + int waitTimeInSecs = AgentPropertiesFileHandler.getPropertyValue(SDC_SERVICE_ACTION_WAIT); + if (waitTimeInSecs < 0) { + waitTimeInSecs = SDC_SERVICE_ACTION_WAIT.getDefaultValue(); + } + try { + LOGGER.debug(String.format("Waiting for %d secs after SDC service action, to reach a stable state", waitTimeInSecs)); + Thread.sleep(waitTimeInSecs * 1000L); + } catch (InterruptedException ignore) { + } } }