Changes to consider Recovered/Available Host HA state along with the agent connection status to determine the Host HA inspection in progress or not, and some code improvements

Author: sureshanaparti

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java

@@ -68,17 +68,18 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh
 
     @Override
     public boolean recover(Host r) throws HARecoveryException {
+        logger.debug("Recover the host {}", r);
         try {
-            if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){
+            if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) {
                 final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null);
                 return resp.getSuccess();
             } else {
                 logger.warn("OOBM recover operation failed for the host {}", r);
                 return false;
             }
-        } catch (Exception e){
+        } catch (Exception e) {
             logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage());
-            throw new HARecoveryException(String.format(" OOBM service is not configured or enabled for this host %s", r), e);
+            throw new HARecoveryException(String.format("OOBM service is not configured or enabled for this host %s", r), e);
         }
     }
 

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -265,7 +265,23 @@ private boolean isHostHAInspectionInProgress(long hostId) {
 
         HAConfig.HAState state = haConfig.getState();
         logger.debug("Checking Host HA inspection is in progress or not for the host {} from HAConfig, HA state is {}", hostId, state);
-        return state == HAConfig.HAState.Suspect || state == HAConfig.HAState.Checking;
+        if (state == HAConfig.HAState.Suspect || state == HAConfig.HAState.Checking) {
+            return true;
+        }
+
+        if (state == HAConfig.HAState.Recovered || state == HAConfig.HAState.Available) {
+            // If the host HA state is Recovered, it indicates that the host has restarted successfully.
+            // If the host HA state is Available, it means the host has restarted successfully and the recovery waiting period has completed.
+            // In both states, the agent can connect as soon as the host is ready (and can move to Suspect -> Checking HA state if the agent connection fails again before Fencing).
+            final HostVO host = _hostDao.findById(hostId);
+            if (host != null && host.getStatus() != Status.Up) {
+                logger.debug("{} is in {} status and HA state is {}, considering Host HA inspection is still in progress" +
+                        " until we are sure the host is ready after a recovery wait period and agent is connected/Up", host, host.getStatus(), state);
+                return true;
+            }
+        }
+
+        return false;
     }
 
     @Override

utils/src/main/java/org/apache/cloudstack/utils/redfish/RedfishClient.java

@@ -231,21 +231,19 @@ private HttpResponse executeHttpRequest(String url, HttpRequestBase httpReq) {
     }
 
     protected HttpResponse retryHttpRequest(String url, HttpRequestBase httpReq, HttpClient client) {
-        logger.warn(String.format("Failed to execute HTTP %s request [URL: %s]. Executing the request again.", httpReq.getMethod(), url));
+        logger.warn("Failed to execute HTTP {} request [URL: {}]. Executing the request again.", httpReq.getMethod(), url);
         HttpResponse response = null;
         for (int attempt = 1; attempt < redfishRequestMaxRetries + 1; attempt++) {
             try {
                 TimeUnit.SECONDS.sleep(WAIT_FOR_REQUEST_RETRY);
-                logger.debug(String.format("HTTP %s request retry attempt %d/%d [URL: %s].", httpReq.getMethod(), attempt, redfishRequestMaxRetries, url));
+                logger.debug("HTTP {} request retry attempt {}/{} [URL: {}].", httpReq.getMethod(), attempt, redfishRequestMaxRetries, url);
                 response = client.execute(httpReq);
                 break;
             } catch (IOException | InterruptedException e) {
                 if (attempt == redfishRequestMaxRetries) {
                     throw new RedfishException(String.format("Failed to execute HTTP %s request retry attempt %d/%d [URL: %s] due to exception %s", httpReq.getMethod(), attempt, redfishRequestMaxRetries,url, e));
                 } else {
-                    logger.warn(
-                            String.format("Failed to execute HTTP %s request retry attempt %d/%d [URL: %s] due to exception %s", httpReq.getMethod(), attempt, redfishRequestMaxRetries,
-                                    url, e));
+                    logger.warn("Failed to execute HTTP {} request retry attempt {}/{} [URL: {}] due to exception {}", httpReq.getMethod(), attempt, redfishRequestMaxRetries, url, e);
                 }
             }
         }
@@ -312,7 +310,7 @@ public void executeComputerSystemReset(String hostAddress, RedfishResetCmd reset
             throw new RedfishException(String.format("Failed to execute System power command for host by performing '%s' request on URL '%s' and host address '%s'. The expected HTTP status code is '%s' but it got '%s'.",
                     HttpPost.METHOD_NAME, url, hostAddress, EXPECTED_HTTP_STATUS, statusCode));
         }
-        logger.debug(String.format("Sending ComputerSystem.Reset Command '%s' to host '%s' with request '%s %s'", resetCommand, hostAddress, HttpPost.METHOD_NAME, url));
+        logger.debug("Sending ComputerSystem.Reset Command '{}' to host '{}' with request '{} {}'", resetCommand, hostAddress, HttpPost.METHOD_NAME, url);
     }
 
     /**
@@ -330,7 +328,7 @@ public String getSystemId(String hostAddress) {
 
         String systemId = processGetSystemIdResponse(response);
 
-        logger.debug(String.format("Retrieved System ID '%s' with request '%s: %s'", systemId, HttpGet.METHOD_NAME, url));
+        logger.debug("Retrieved System ID '{}' with request '{}: {}'", systemId, HttpGet.METHOD_NAME, url);
 
         return systemId;
     }
@@ -384,7 +382,7 @@ public RedfishPowerState getSystemPowerState(String hostAddress) {
         }
 
         RedfishPowerState powerState = processGetSystemRequestResponse(response);
-        logger.debug(String.format("Retrieved System power state '%s' with request '%s: %s'", powerState, HttpGet.METHOD_NAME, url));
+        logger.debug("Retrieved System power state '{}' with request '{}: {}'", powerState, HttpGet.METHOD_NAME, url);
         return powerState;
     }