Avoid infinite loops when registering test classes for reflection

Author: dnestoro

common/junit-platform-native/src/main/java/org/graalvm/junit/platform/JUnitPlatformFeature.java

@@ -66,9 +66,11 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.ServiceLoader;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
@@ -168,7 +170,28 @@ private void registerTestClassesForReflection(List<? extends DiscoverySelector>
                 .forEach(this::registerTestClassForReflection);
     }
 
+    private final Set<Class<?>> registeredClasses = new HashSet<>();
+
+    private boolean shouldRegisterClass(Class<?> clazz) {
+        /* avoid registering java internal classes */
+        if (ModuleLayer.boot().modules().contains(clazz.getModule())) {
+            return false;
+        }
+
+        /* avoid loops (possible case: class B is inner class of A, and B extends A) */
+        if (registeredClasses.contains(clazz)) {
+            return false;
+        }
+        registeredClasses.add(clazz);
+
+        return true;
+    }
+
     private void registerTestClassForReflection(Class<?> clazz) {
+        if (!shouldRegisterClass(clazz)) {
+            return;
+        }
+
         debug("Registering test class for reflection: %s", clazz.getName());
         nativeImageConfigImpl.registerAllClassMembersForReflection(clazz);
         forEachProvider(p -> p.onTestClassRegistered(clazz, nativeImageConfigImpl));

common/junit-platform-native/src/test/java/org/graalvm/junit/jupiter/AbstractParentClassTests.java

@@ -43,6 +43,7 @@
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -52,6 +53,32 @@
 
 public class AbstractParentClassTests {
 
+    public static class OuterClass {
+        @Test
+        protected void test() {
+            Assertions.assertTrue(true, "Just a dummy test that should be executed in outer class");
+        }
+
+        /* Since at org.graalvm.junit.platform.JUnitPlatformFeature#registerTestClassForReflection we register all
+         * declared classes and superclass of the test class, and we do so recursively, we want to avoid infinite loop.
+         * This inheritance shows that we won't call registration of these classes indefinitely (call registration of
+         * all declared classes of AbstractParentClassTests, then recursively call superclass of InfiniteLoopTest and
+         * repeat the process indefinitely) */
+        private class InfiniteLoopTest extends OuterClass {
+            @Test
+            protected void test() {
+                Assertions.assertTrue(true, "Just a dummy test that should be executed in inner class");
+            }
+        }
+
+        /* Since enum here is declared class of AbstractParentClassTests, we want to avoid registrations of
+         * enum's internal superclasses and sub-classes at org.graalvm.junit.platform.JUnitPlatformFeature#registerTestClassForReflection */
+        private enum EnumTest {
+            SOME_VALUE,
+            OTHER_VALUE
+        }
+    }
+
     public abstract static class MathPowerTests {
         protected static BiFunction<Integer, Integer, Integer> powFunction;