VFS-824 HttpFileSystem free Unused Resources lead to HttpClient Conn…

commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/AbstractFileProvider.java

@@ -137,17 +137,23 @@ protected FileSystem findFileSystem(final Comparable<?> key, final FileSystemOpt
      * Frees unused resources.
      */
     public void freeUnusedResources() {
-        final AbstractFileSystem[] abstractFileSystems;
-        synchronized (fileSystemMap) {
-            // create snapshot under lock
-            abstractFileSystems = fileSystemMap.values().toArray(EMPTY_ABSTRACT_FILE_SYSTEMS);
-        }
-
         // process snapshot outside lock
-        Stream.of(abstractFileSystems).filter(AbstractFileSystem::isReleaseable)
+        Stream.of(getAllFileSystemSnapshot()).filter(AbstractFileSystem::isReleaseable)
                                       .forEach(AbstractFileSystem::closeCommunicationLink);
     }
 
+    /**
+     * Gets a snapshot of all AbstractFileSystems.
+     *
+     * @return a snapshot of all AbstractFileSystems.
+     * @since 2.10.0
+     */
+    public AbstractFileSystem[] getAllFileSystemSnapshot() {
+        synchronized (fileSystemMap) {
+            return fileSystemMap.values().toArray(EMPTY_ABSTRACT_FILE_SYSTEMS);
+        }
+    }
+
     /**
      * Gets the FileSystemConfigBuilder.
      *

commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http/HttpFileProvider.java

@@ -19,6 +19,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.stream.Stream;
 
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.vfs2.Capability;
@@ -28,6 +29,7 @@
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.UserAuthenticationData;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
 import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
 import org.apache.commons.vfs2.provider.GenericFileName;
 import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
@@ -93,4 +95,17 @@ public Collection<Capability> getCapabilities() {
     public FileSystemConfigBuilder getConfigBuilder() {
         return HttpFileSystemConfigBuilder.getInstance();
     }
+
+    /**
+     * Frees unused resources and close HttpFileSystem.
+     */
+    @Override
+    public void freeUnusedResources() {
+        // process snapshot outside lock
+        Stream.of(getAllFileSystemSnapshot()).filter(AbstractFileSystem::isReleaseable)
+                .forEach(fileSystem -> {
+                    fileSystem.closeCommunicationLink();
+                    closeFileSystem(fileSystem);
+                });
+    }
 }

commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http4/Http4FileProvider.java

@@ -43,6 +43,7 @@
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.UserAuthenticationData;
 import org.apache.commons.vfs2.UserAuthenticator;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
 import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
 import org.apache.commons.vfs2.provider.GenericFileName;
 import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
@@ -378,4 +379,17 @@ private HttpHost getProxyHttpHost(final Http4FileSystemConfigBuilder builder,
         return null;
     }
 
+
+    /**
+     * Frees unused resources and close Http4FileSystem.
+     */
+    @Override
+    public void freeUnusedResources() {
+        // process snapshot outside lock
+        Stream.of(getAllFileSystemSnapshot()).filter(AbstractFileSystem::isReleaseable)
+                .forEach(fileSystem -> {
+                    fileSystem.closeCommunicationLink();
+                    closeFileSystem(fileSystem);
+                });
+    }
 }

commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http5/Http5FileProvider.java

@@ -43,6 +43,7 @@
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.UserAuthenticationData;
 import org.apache.commons.vfs2.UserAuthenticator;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
 import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
 import org.apache.commons.vfs2.provider.GenericFileName;
 import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
@@ -375,4 +376,18 @@ private HttpHost getProxyHttpHost(final Http5FileSystemConfigBuilder builder,
         return null;
     }
 
+
+    /**
+     * Frees unused resources and close Http5FileSystem.
+     */
+    @Override
+    public void freeUnusedResources() {
+        // process snapshot outside lock
+        Stream.of(getAllFileSystemSnapshot()).filter(AbstractFileSystem::isReleaseable)
+                .forEach(fileSystem -> {
+                    fileSystem.closeCommunicationLink();
+                    closeFileSystem(fileSystem);
+                });
+    }
+
 }

commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/AbstractFileSystemTestUtil.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.vfs2.provider;
+
+import org.apache.commons.vfs2.FileObject;
+
+public class AbstractFileSystemTestUtil {
+
+    /**
+     * call {@link AbstractFileSystem#fileObjectDestroyed(FileObject)}
+     */
+    public static void fileObjectDestroyed(final AbstractFileSystem fileSystem, final FileObject fileObject) {
+        fileSystem.fileObjectDestroyed(fileObject);
+    }
+}

commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http/HttpProviderTestCase.java

@@ -20,6 +20,7 @@
 
 import java.io.File;
 import java.time.Duration;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.vfs2.AbstractProviderTestConfig;
 import org.apache.commons.vfs2.FileNotFolderException;
@@ -29,7 +30,11 @@
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.ProviderTestSuite;
 import org.apache.commons.vfs2.VFS;
+import org.apache.commons.vfs2.cache.WeakRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.commons.vfs2.provider.AbstractFileSystemTestUtil;
 import org.apache.commons.vfs2.util.NHttpFileServer;
 import org.junit.Test;
 import org.junit.jupiter.api.Assertions;
@@ -212,4 +217,35 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
         testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
     }
 
+    @Test
+    public void testHttpFileSystemFreeUnusedResources() throws Exception {
+        try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
+            fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
+            // use WeakRef
+            fileSystemManager.setFilesCache(new WeakRefFilesCache());
+            fileSystemManager.init();
+
+            String path = ConnectionUri + "/read-tests/";
+            AbstractFileSystem httpFileSystem = getFile(fileSystemManager, path);
+
+            // make FileSystem.isReleaseable is true through GC will break the build randomly.
+            // It is better to decrease AbstractFileSystem.useCount directly.
+            AbstractFileSystemTestUtil.fileObjectDestroyed(httpFileSystem, null);
+            assertTrue(httpFileSystem.isReleaseable());
+            // free resource
+            // httpFileSystem.httpClient is closed
+            fileSystemManager.freeUnusedResources();
+
+            // get file again
+            getFile(fileSystemManager, path);
+        }
+    }
+
+    private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
+        FileObject fileObject = fileSystemManager.resolveFile(path);
+        // send
+        fileObject.getType();
+        return (AbstractFileSystem) fileObject.getFileSystem();
+    }
+
 }

commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http4/Http4ProviderTestCase.java

@@ -30,7 +30,11 @@
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.ProviderTestSuite;
 import org.apache.commons.vfs2.VFS;
+import org.apache.commons.vfs2.cache.WeakRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.commons.vfs2.provider.AbstractFileSystemTestUtil;
 import org.apache.commons.vfs2.util.NHttpFileServer;
 import org.junit.jupiter.api.Assertions;
 import org.junit.Test;
@@ -216,4 +220,33 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
         testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
     }
 
+    @Test
+    public void testHttp4FileSystemFreeUnusedResources() throws Exception {
+        try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
+            fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
+            // use WeakRef
+            fileSystemManager.setFilesCache(new WeakRefFilesCache());
+            fileSystemManager.init();
+
+            String path = ConnectionUri + "/read-tests/";
+            AbstractFileSystem http4FileSystem = getFile(fileSystemManager, path);
+            // make FileSystem.isReleaseable is true through GC will break the build randomly.
+            // It is better to decrease AbstractFileSystem.useCount directly.
+            AbstractFileSystemTestUtil.fileObjectDestroyed(http4FileSystem, null);
+            assertTrue(http4FileSystem.isReleaseable());
+            // free resource
+            // http4FileSystem.httpClient is closed
+            fileSystemManager.freeUnusedResources();
+
+            // get file again
+            getFile(fileSystemManager, path);
+        }
+    }
+
+    private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
+        FileObject fileObject = fileSystemManager.resolveFile(path);
+        // send
+        fileObject.getType();
+        return (AbstractFileSystem) fileObject.getFileSystem();
+    }
 }

commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http5/Http5ProviderTestCase.java

@@ -30,7 +30,11 @@
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.ProviderTestSuite;
 import org.apache.commons.vfs2.VFS;
+import org.apache.commons.vfs2.cache.WeakRefFilesCache;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.commons.vfs2.provider.AbstractFileSystemTestUtil;
 import org.apache.commons.vfs2.util.NHttpFileServer;
 import org.junit.Test;
 import org.junit.jupiter.api.Assertions;
@@ -210,4 +214,35 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
         testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
     }
 
+    @Test
+    public void testHttp5FileSystemFreeUnusedResources() throws Exception {
+        try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
+            fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
+            // use WeakRef
+            fileSystemManager.setFilesCache(new WeakRefFilesCache());
+            fileSystemManager.init();
+
+            String path = ConnectionUri + "/read-tests/";
+            AbstractFileSystem http5FileSystem = getFile(fileSystemManager, path);
+
+            // make FileSystem.isReleaseable is true through GC will break the build randomly.
+            // It is better to decrease AbstractFileSystem.useCount directly.
+            AbstractFileSystemTestUtil.fileObjectDestroyed(http5FileSystem, null);
+            assertTrue(http5FileSystem.isReleaseable());
+            // free resource
+            // http5FileSystem.httpClient is closed
+            fileSystemManager.freeUnusedResources();
+
+            // get file again
+            getFile(fileSystemManager, path);
+        }
+    }
+
+    private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
+        FileObject fileObject = fileSystemManager.resolveFile(path);
+        // send
+        fileObject.getType();
+        return (AbstractFileSystem) fileObject.getFileSystem();
+    }
+
 }