v2.0.7

Author: arnett, stu

src/main/java/com/emc/rest/smart/Host.java

@@ -34,16 +34,19 @@
 /**
  * Some basic statements about response index calculation:
  * <p>
- * - lower response index means the host is more likely to be used
- * - should be based primarily on number of open connections to the host
- * - an error will mark the host as unhealthy for <code>errorWaitTime</code> milliseconds
- * - multiple consecutive errors compound the unhealthy (cool down) period up to 8x the errorWaitTime
+ *     <ul>
+ *         <li>lower response index means the host is more likely to be used</li>
+ *         <li>should be based primarily on number of open connections to the host</li>
+ *         <li>an error will mark the host as unhealthy for <code>errorWaitTime</code> milliseconds</li>
+ *         <li>multiple consecutive errors compound the unhealthy (cool down) period up to 16x the errorWaitTime</li>
+ *     </ul>
  */
 public class Host implements HostStats {
     private static final Logger l4j = Logger.getLogger(Host.class);
 
     public static final int DEFAULT_ERROR_WAIT_MS = 1500;
     public static final int LOG_DELAY = 60000; // 1 minute
+    public static final int MAX_COOL_DOWN_EXP = 4;
 
     private String name;
     private boolean healthy = true;
@@ -74,8 +77,11 @@ public synchronized void connectionClosed() {
 
         // Just in case our stats get out of whack somehow, make sure people know about it
         if (openConnections < 0) {
-            if (System.currentTimeMillis() - lastLogTime > LOG_DELAY)
+            long currentTime = System.currentTimeMillis();
+            if (currentTime - lastLogTime > LOG_DELAY) {
                 LogMF.warn(l4j, "openConnections for host %s is %d !", this, openConnections);
+                lastLogTime = currentTime;
+            }
         }
     }
 
@@ -98,9 +104,9 @@ public boolean isHealthy() {
         if (!healthy) return false;
         else if (consecutiveErrors == 0) return true;
         else {
-            long coolDownPower = consecutiveErrors > 3 ? 3 : consecutiveErrors - 1;
+            long coolDownExp = consecutiveErrors > MAX_COOL_DOWN_EXP ? MAX_COOL_DOWN_EXP : consecutiveErrors - 1;
             long msSinceLastUse = System.currentTimeMillis() - lastConnectionTime;
-            long errorCoolDown = (long) Math.pow(2, coolDownPower) * errorWaitTime;
+            long errorCoolDown = (long) Math.pow(2, coolDownExp) * errorWaitTime;
             return msSinceLastUse > errorCoolDown;
         }
     }

src/main/java/com/emc/rest/smart/SmartClientFactory.java

@@ -36,9 +36,6 @@
 import com.sun.jersey.core.impl.provider.entity.ByteArrayProvider;
 import com.sun.jersey.core.impl.provider.entity.FileProvider;
 import com.sun.jersey.core.impl.provider.entity.InputStreamProvider;
-import org.apache.http.impl.client.AbstractHttpClient;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
-import org.apache.http.impl.conn.PoolingClientConnectionManager;
 import org.apache.log4j.Logger;
 
 public final class SmartClientFactory {
@@ -137,7 +134,7 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi
         ClientConfig clientConfig = new DefaultClientConfig();
 
         // set up multi-threaded connection pool
-        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
+        org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager = new org.apache.http.impl.conn.PoolingClientConnectionManager();
         // 200 maximum active connections (should be more than enough for any JVM instance)
         connectionManager.setDefaultMaxPerRoute(200);
         connectionManager.setMaxTotal(200);
@@ -151,11 +148,18 @@ static ApacheHttpClient4Handler createApacheClientHandler(SmartConfig smartConfi
         if (smartConfig.getProxyPass() != null)
             clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_PASSWORD, smartConfig.getProxyPass());
 
+        // pass in jersey parameters from calling code (allows customization of client)
+        for (String propName : smartConfig.getProperties().keySet()) {
+            clientConfig.getProperties().put(propName, smartConfig.getProperty(propName));
+        }
+
         ApacheHttpClient4Handler handler = ApacheHttpClient4.create(clientConfig).getClientHandler();
 
         // disable the retry handler if necessary
-        if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null)
-            ((AbstractHttpClient) handler.getHttpClient()).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
+        if (smartConfig.getProperty(DISABLE_APACHE_RETRY) != null) {
+            org.apache.http.impl.client.AbstractHttpClient httpClient = (org.apache.http.impl.client.AbstractHttpClient) handler.getHttpClient();
+            httpClient.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false));
+        }
 
         return handler;
     }

src/main/java/com/emc/rest/smart/SmartFilter.java

@@ -30,8 +30,6 @@
 import com.sun.jersey.api.client.ClientRequest;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.filter.ClientFilter;
-import org.apache.http.HttpHost;
-import org.apache.http.client.utils.URIUtils;
 
 import java.io.FilterInputStream;
 import java.io.IOException;
@@ -62,7 +60,8 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio
         // replace the host in the request
         URI uri = request.getURI();
         try {
-            uri = URIUtils.rewriteURI(uri, new HttpHost(host.getName(), uri.getPort(), uri.getScheme()));
+            org.apache.http.HttpHost httpHost = new org.apache.http.HttpHost(host.getName(), uri.getPort(), uri.getScheme());
+            uri = org.apache.http.client.utils.URIUtils.rewriteURI(uri, httpHost);
         } catch (URISyntaxException e) {
             throw new RuntimeException("load-balanced host generated invalid URI", e);
         }

src/main/java/com/emc/rest/smart/ecs/EcsHostListProvider.java

@@ -99,7 +99,18 @@ public List<Host> getHostList() {
     @Override
     public void runHealthCheck(Host host) {
         // header is workaround for STORAGE-1833
-        client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x").get(String.class);
+        PingResponse response = client.resource(getRequestUri(host, "/?ping")).header("x-emc-namespace", "x")
+                .get(PingResponse.class);
+
+        if (host instanceof VdcHost) {
+            PingItem.Status status = PingItem.Status.OFF;
+            if (response != null && response.getPingItemMap() != null) {
+                PingItem pingItem = response.getPingItemMap().get(PingItem.MAINTENANCE_MODE);
+                if (pingItem != null) status = pingItem.getStatus();
+            }
+            if (status == PingItem.Status.ON) ((VdcHost) host).setMaintenanceMode(true);
+            else ((VdcHost) host).setMaintenanceMode(false);
+        }
     }
 
     @Override

src/main/java/com/emc/rest/smart/ecs/PingItem.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2015, EMC Corporation.
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *     + Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *     + Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     + The name of EMC Corporation may not be used to endorse or promote
+ *       products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.emc.rest.smart.ecs;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlEnum;
+
+public class PingItem {
+    public static final String MAINTENANCE_MODE = "MAINTENANCE_MODE";
+
+    String name;
+    Status status;
+    String text;
+
+    @XmlElement(name = "Name")
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    @XmlElement(name = "Status")
+    public Status getStatus() {
+        return status;
+    }
+
+    public void setStatus(Status status) {
+        this.status = status;
+    }
+
+    @XmlElement(name = "Text")
+    public String getText() {
+        return text;
+    }
+
+    public void setText(String text) {
+        this.text = text;
+    }
+
+    @XmlEnum
+    public static enum Status {
+        OFF, UNKNOWN, ON
+    }
+}

src/main/java/com/emc/rest/smart/ecs/PingResponse.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2015, EMC Corporation.
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *     + Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *     + Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     + The name of EMC Corporation may not be used to endorse or promote
+ *       products derived from this software without specific prior written
+ *       permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.emc.rest.smart.ecs;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@XmlRootElement(name = "PingList")
+public class PingResponse {
+    Map<String, PingItem> pingItemMap;
+
+    @XmlElement(name = "PingItem")
+    public List<PingItem> getPingItems() {
+        if (pingItemMap == null) return null;
+        return new ArrayList<PingItem>(pingItemMap.values());
+    }
+
+    public void setPingItems(List<PingItem> pingItems) {
+        if (pingItems != null) {
+            pingItemMap = new HashMap<String, PingItem>();
+            for (PingItem item : pingItems) {
+                pingItemMap.put(item.getName(), item);
+            }
+        }
+    }
+
+    @XmlTransient
+    public Map<String, PingItem> getPingItemMap() {
+        return pingItemMap;
+    }
+
+    public void setPingItemMap(Map<String, PingItem> pingItemMap) {
+        this.pingItemMap = pingItemMap;
+    }
+}

src/main/java/com/emc/rest/smart/ecs/VdcHost.java

@@ -30,12 +30,18 @@
 
 public class VdcHost extends Host {
     private Vdc vdc;
+    private boolean maintenanceMode;
 
     public VdcHost(Vdc vdc, String name) {
         super(name);
         this.vdc = vdc;
     }
 
+    @Override
+    public boolean isHealthy() {
+        return !isMaintenanceMode() && super.isHealthy();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -63,4 +69,12 @@ public String toString() {
     public Vdc getVdc() {
         return vdc;
     }
+
+    public boolean isMaintenanceMode() {
+        return maintenanceMode;
+    }
+
+    public void setMaintenanceMode(boolean maintenanceMode) {
+        this.maintenanceMode = maintenanceMode;
+    }
 }

src/test/java/com/emc/rest/smart/HostTest.java

@@ -121,4 +121,27 @@ public void testHost() throws Exception {
         Assert.assertEquals(0, host.getResponseIndex());
         Assert.assertTrue(host.isHealthy());
     }
+
+    @Test
+    public void testErrorWaitLimit() throws Exception {
+        Host host = new Host("bar");
+        host.setErrorWaitTime(100); // don't want this test to take forever
+
+        Assert.assertTrue(host.isHealthy());
+
+        // 8 consecutive errors
+        long errors = 8;
+        for (int i = 0; i < errors; i++) {
+            host.connectionOpened();
+            host.callComplete(true);
+            host.connectionClosed();
+        }
+
+        Assert.assertEquals(errors, host.getConsecutiveErrors());
+        long maxCoolDownMs = host.getErrorWaitTime() * (long) Math.pow(2, Host.MAX_COOL_DOWN_EXP) + 10; // add a few ms
+
+        Thread.sleep(maxCoolDownMs);
+
+        Assert.assertTrue(host.isHealthy());
+    }
 }

src/test/java/com/emc/rest/smart/SmartClientTest.java

@@ -28,9 +28,14 @@
 
 import com.emc.util.TestConfig;
 import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config;
 import org.apache.commons.codec.binary.Base64;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
 import org.apache.log4j.Logger;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -42,9 +47,7 @@
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class SmartClientTest {
@@ -105,6 +108,36 @@ public void run() {
         Assert.assertEquals("at least one task failed", 100, successCount.intValue());
     }
 
+    @Test
+    public void testConnTimeout() throws Exception {
+        int CONNECTION_TIMEOUT_MILLIS = 10000; // 10 seconds
+
+        HttpParams httpParams = new BasicHttpParams();
+        HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT_MILLIS);
+
+        SmartConfig smartConfig = new SmartConfig("10.4.4.180");
+        smartConfig.setProperty(ApacheHttpClient4Config.PROPERTY_HTTP_PARAMS, httpParams);
+
+        final Client client = SmartClientFactory.createStandardClient(smartConfig);
+
+        Future future = Executors.newSingleThreadExecutor().submit(new Runnable() {
+            @Override
+            public void run() {
+                client.resource("http://10.4.4.180:9020/?ping").get(String.class);
+                Assert.fail("response was not expected; choose an IP that is not in use");
+            }
+        });
+
+        try {
+            future.get(CONNECTION_TIMEOUT_MILLIS + 1000, TimeUnit.MILLISECONDS); // give an extra second leeway
+        } catch (TimeoutException e) {
+            Assert.fail("connection did not timeout");
+        } catch (ExecutionException e) {
+            Assert.assertTrue(e.getCause() instanceof ClientHandlerException);
+            Assert.assertTrue(e.getMessage().contains("timed out"));
+        }
+    }
+
     private void getServiceInfo(Client client, URI serverUri, String uid, String secretKey) {
         String path = "/rest/service";
         String date = getDateFormat().format(new Date());