Fix 91 (#306)

* OUT OF MEMORY when trying to run all tests
add JavaClassProvider which provied java.* classes
add CombinedJarIndex which indexes main jar and libraries
added missing translation for "progress.jar.indexing.libraries"

* make methods that trace back to non-jar methods non-renamable

* checkstyle

* filter out unreferenced classNames for LibrariesJarIndex and CombinedJarIndex

* add ReferenceIndex::getFieldsReferencedBy
keep lib classes whose fields or methods are referenced in the main jar

* exclude java/io/PrintStream from JavaClassProvider
check if method is indexed in EnigmaProject::isLibraryMethodOverride

* update asm (AnalyzerExceptions persist)

* use ClasspathClassProvider instead of JavaClassProvider to avoid AnalyzerExceptions; BREAK #274

* stop excluding Object from InheritanceIndex and update tests; fix #91

* update TODO

* try finding child node instead of assuming it's first

* add clarifying comment to UN_ANALYZABLE_CLASS_NAMES

Co-authored-by: ix0rai <[email protected]>

* add jar index javadocs

---------

Co-authored-by: ix0rai <[email protected]>

Author: supersaiyansubtlety

enigma-server/src/main/java/org/quiltmc/enigma/network/DedicatedEnigmaServer.java

@@ -124,10 +124,10 @@ public static void main(String[] args) {
 
 			EntryRemapper mappings;
 			if (!Files.exists(mappingsFile)) {
-				mappings = EntryRemapper.mapped(enigma, project.getJarIndex(), project.getMappingsIndex(), project.getRemapper().getJarProposedMappings(), new HashEntryTree<>(), enigma.getNameProposalServices());
+				mappings = EntryRemapper.mapped(enigma, project.getCombinedIndex(), project.getMappingsIndex(), project.getRemapper().getJarProposedMappings(), new HashEntryTree<>(), enigma.getNameProposalServices());
 			} else {
 				Logger.info("Reading mappings...");
-				mappings = EntryRemapper.mapped(enigma, project.getJarIndex(), project.getMappingsIndex(), project.getRemapper().getJarProposedMappings(), readWriteService.get().read(mappingsFile), enigma.getNameProposalServices());
+				mappings = EntryRemapper.mapped(enigma, project.getCombinedIndex(), project.getMappingsIndex(), project.getRemapper().getJarProposedMappings(), readWriteService.get().read(mappingsFile), enigma.getNameProposalServices());
 			}
 
 			PrintWriter log = new PrintWriter(Files.newBufferedWriter(logFile));

enigma-swing/src/main/java/org/quiltmc/enigma/gui/GuiController.java

@@ -686,7 +686,7 @@ public void createClient(String username, String ip, int port, char[] password)
 	}
 
 	public void createServer(String username, int port, char[] password) throws IOException {
-		this.server = new IntegratedEnigmaServer(this.project.getJarChecksum(), password, EntryRemapper.mapped(this.project.getEnigma(), this.project.getJarIndex(), this.project.getMappingsIndex(), new HashEntryTree<>(this.project.getRemapper().getJarProposedMappings()), new HashEntryTree<>(this.project.getRemapper().getDeobfMappings()), this.project.getEnigma().getNameProposalServices()), port);
+		this.server = new IntegratedEnigmaServer(this.project.getJarChecksum(), password, EntryRemapper.mapped(this.project.getEnigma(), this.project.getCombinedIndex(), this.project.getMappingsIndex(), new HashEntryTree<>(this.project.getRemapper().getJarProposedMappings()), new HashEntryTree<>(this.project.getRemapper().getDeobfMappings()), this.project.getEnigma().getNameProposalServices()), port);
 		this.server.start();
 		this.client = new IntegratedEnigmaClient(this, "127.0.0.1", port);
 		this.client.connect();

enigma/src/main/java/org/quiltmc/enigma/api/Enigma.java

@@ -1,12 +1,22 @@
 package org.quiltmc.enigma.api;
 
+import com.google.common.base.Predicates;
+import com.google.common.collect.Streams;
 import com.google.common.io.MoreFiles;
-import org.quiltmc.enigma.api.analysis.index.jar.JarIndex;
+import org.objectweb.asm.tree.ClassNode;
+import org.quiltmc.enigma.api.analysis.index.jar.CombinedJarIndex;
+import org.quiltmc.enigma.api.analysis.index.jar.EntryIndex;
+import org.quiltmc.enigma.api.analysis.index.jar.InheritanceIndex;
 import org.quiltmc.enigma.api.analysis.index.jar.LibrariesJarIndex;
 import org.quiltmc.enigma.api.analysis.index.jar.MainJarIndex;
+import org.quiltmc.enigma.api.analysis.index.jar.ReferenceIndex;
 import org.quiltmc.enigma.api.analysis.index.mapping.MappingsIndex;
+import org.quiltmc.enigma.api.class_provider.ClasspathClassProvider;
 import org.quiltmc.enigma.api.class_provider.ProjectClassProvider;
 import org.quiltmc.enigma.api.translation.mapping.serde.MappingParseException;
+import org.quiltmc.enigma.api.translation.representation.entry.ClassEntry;
+import org.quiltmc.enigma.api.translation.representation.entry.FieldEntry;
+import org.quiltmc.enigma.api.translation.representation.entry.MethodEntry;
 import org.quiltmc.enigma.impl.analysis.ClassLoaderClassProvider;
 import org.quiltmc.enigma.api.service.EnigmaService;
 import org.quiltmc.enigma.api.service.EnigmaServiceContext;
@@ -25,6 +35,8 @@
 import org.quiltmc.enigma.api.translation.mapping.tree.EntryTree;
 import org.quiltmc.enigma.api.translation.mapping.tree.HashEntryTree;
 import org.quiltmc.enigma.api.translation.representation.entry.Entry;
+import org.quiltmc.enigma.impl.analysis.index.AbstractJarIndex;
+import org.quiltmc.enigma.impl.analysis.index.IndexClassVisitor;
 import org.quiltmc.enigma.util.Either;
 import org.quiltmc.enigma.util.I18n;
 import org.quiltmc.enigma.util.Utils;
@@ -43,15 +55,20 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.sql.DriverManager;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class Enigma {
 	public static final String NAME = "Enigma";
@@ -80,19 +97,27 @@ public static Builder builder() {
 
 	public EnigmaProject openJar(Path path, ClassProvider libraryClassProvider, ProgressListener progress) throws IOException {
 		JarClassProvider jarClassProvider = new JarClassProvider(path);
-		JarIndex index = MainJarIndex.empty();
-		JarIndex libIndex = LibrariesJarIndex.empty();
+		AbstractJarIndex jarIndex = MainJarIndex.empty();
+		AbstractJarIndex libIndex = LibrariesJarIndex.empty();
+		AbstractJarIndex comboIndex = CombinedJarIndex.empty();
 
 		ClassLoaderClassProvider jreProvider = new ClassLoaderClassProvider(DriverManager.class.getClassLoader());
-		CombiningClassProvider librariesProvider = new CombiningClassProvider(jreProvider, libraryClassProvider);
-		ClassProvider mainProjectProvider = new ObfuscationFixClassProvider(new CachingClassProvider(jarClassProvider), index);
+		ClasspathClassProvider javaClassProvider = new ClasspathClassProvider();
+		CombiningClassProvider librariesProvider = new CombiningClassProvider(jreProvider, javaClassProvider, libraryClassProvider);
+		ClassProvider mainProjectProvider = new ObfuscationFixClassProvider(new CachingClassProvider(jarClassProvider), jarIndex);
 		ProjectClassProvider projectClassProvider = new ProjectClassProvider(mainProjectProvider, librariesProvider);
 
 		// main index
-		this.index(index, projectClassProvider, progress);
+		this.index(jarIndex, projectClassProvider, progress, "jar", false, null);
+
+		// TODO make filtering toggleable with arg once JavaClassProvider is used
+		final Predicate<String> mainReferencedPredicate = this.createMainReferencedPredicate(jarIndex, projectClassProvider);
 
 		// lib index
-		this.index(libIndex, projectClassProvider, progress);
+		this.index(libIndex, projectClassProvider, progress, "jar", true, mainReferencedPredicate);
+
+		// combined main and lib index
+		this.index(comboIndex, projectClassProvider, progress, "combined", true, mainReferencedPredicate);
 
 		// name proposal
 		var nameProposalServices = this.getNameProposalServices();
@@ -103,7 +128,7 @@ public EnigmaProject openJar(Path path, ClassProvider libraryClassProvider, Prog
 		int j = 1;
 		for (var service : nameProposalServices) {
 			progress.step(j++, I18n.translateFormatted("progress.jar.name_proposal.proposer", service.getId()));
-			Map<Entry<?>, EntryMapping> proposed = service.getProposedNames(this, index);
+			Map<Entry<?>, EntryMapping> proposed = service.getProposedNames(this, jarIndex);
 
 			if (proposed != null) {
 				for (var entry : proposed.entrySet()) {
@@ -118,23 +143,89 @@ public EnigmaProject openJar(Path path, ClassProvider libraryClassProvider, Prog
 		MappingsIndex mappingsIndex = MappingsIndex.empty();
 		mappingsIndex.indexMappings(proposedNames, progress);
 
-		return new EnigmaProject(this, path, mainProjectProvider, index, libIndex, mappingsIndex, proposedNames, Utils.zipSha1(path));
+		return new EnigmaProject(this, path, mainProjectProvider, jarIndex, libIndex, comboIndex, mappingsIndex, proposedNames, Utils.zipSha1(path));
 	}
 
-	private void index(JarIndex index, ProjectClassProvider classProvider, ProgressListener progress) {
-		boolean libraries = index instanceof LibrariesJarIndex;
-		String progressKey = libraries ? "libs" : "jar";
-		index.indexJar(classProvider, progress);
+	private Predicate<String> createMainReferencedPredicate(AbstractJarIndex mainIndex, ProjectClassProvider classProvider) {
+		final EntryIndex mainEntryIndex = mainIndex.getIndex(EntryIndex.class);
+
+		final EntryIndex entryIndex = new EntryIndex();
+		final ReferenceIndex referenceIndex = new ReferenceIndex();
+		final InheritanceIndex inheritanceIndex = new InheritanceIndex(entryIndex);
+
+		final Collection<String> allClassNames = classProvider.getClassNames();
+		for (final String className : allClassNames) {
+			final ClassNode classNode = Objects.requireNonNull(classProvider.get(className));
+			classNode.accept(new IndexClassVisitor(entryIndex, Enigma.ASM_VERSION));
+			classNode.accept(new IndexClassVisitor(referenceIndex, Enigma.ASM_VERSION));
+			classNode.accept(new IndexClassVisitor(inheritanceIndex, Enigma.ASM_VERSION));
+		}
+
+		return className -> {
+			final ClassEntry classEntry = new ClassEntry(className);
+			if (mainEntryIndex.hasClass(classEntry)) {
+				return true;
+			}
+
+			if (inheritanceIndex.getChildren(classEntry).stream().anyMatch(mainEntryIndex::hasClass)) {
+				return true;
+			}
+
+			final boolean typeReferenced = Streams
+					.concat(
+						referenceIndex.getReferencesToClass(classEntry).stream(),
+						referenceIndex.getMethodTypeReferencesToClass(classEntry).stream(),
+						referenceIndex.getFieldTypeReferencesToClass(classEntry).stream()
+					)
+					.anyMatch(reference ->
+						mainEntryIndex.hasClass(reference.entry) || mainEntryIndex.hasEntry(reference.context)
+					);
+
+			if (typeReferenced) {
+				return true;
+			}
+
+			final List<MethodEntry> mainMethods = mainIndex.getChildrenByClass().values().stream()
+					.flatMap(entry -> entry instanceof MethodEntry method ? Stream.of(method) : Stream.empty())
+					.toList();
+
+			final boolean methodReferenced = mainMethods.stream()
+					.flatMap(method -> referenceIndex.getMethodsReferencedBy(method).stream())
+					.map(MethodEntry::getParent)
+					.anyMatch(classEntry::equals);
+			if (methodReferenced) {
+				return true;
+			}
+
+			// field referenced
+			return mainMethods.stream()
+				.flatMap(method -> referenceIndex.getFieldsReferencedBy(method).stream())
+				.map(FieldEntry::getParent)
+				.anyMatch(classEntry::equals);
+		};
+	}
+
+	private void index(
+			AbstractJarIndex index, ProjectClassProvider classProvider, ProgressListener progress, String progressKey,
+			boolean includesLibraries, @Nullable Predicate<String> classNameFilter
+	) {
+		if (classNameFilter == null) {
+			index.indexJar(classProvider, progress);
+			classNameFilter = Predicates.alwaysTrue();
+		} else {
+			index.indexJar(classProvider, progress, classNameFilter);
+		}
 
 		List<JarIndexerService> indexers = this.services.get(JarIndexerService.TYPE);
 		progress.init(indexers.size(), I18n.translate("progress." + progressKey + ".custom_indexing"));
 
 		int i = 1;
 		for (var service : indexers) {
-			if (!(libraries && !service.shouldIndexLibraries())) {
+			if (!(includesLibraries && !service.shouldIndexLibraries())) {
 				progress.step(i++, I18n.translateFormatted("progress." + progressKey + ".custom_indexing.indexer", service.getId()));
-				var names = libraries ? classProvider.getLibraryClassNames() : classProvider.getMainClassNames();
-				Set<String> scope = new HashSet<>(names);
+				Set<String> scope = index.getIndexableClassNames(classProvider).stream()
+						.filter(classNameFilter)
+						.collect(Collectors.toCollection(HashSet::new));
 				service.acceptJar(scope, classProvider, index);
 			}
 		}

enigma/src/main/java/org/quiltmc/enigma/api/EnigmaProject.java

@@ -9,6 +9,8 @@
 import org.quiltmc.enigma.api.analysis.index.mapping.MappingsIndex;
 import org.quiltmc.enigma.api.service.ObfuscationTestService;
 import org.quiltmc.enigma.api.source.TokenType;
+import org.quiltmc.enigma.api.translation.mapping.EntryResolver;
+import org.quiltmc.enigma.api.translation.mapping.ResolutionStrategy;
 import org.quiltmc.enigma.api.translation.mapping.tree.EntryTreeUtil;
 import org.quiltmc.enigma.api.translation.mapping.tree.HashEntryTree;
 import org.quiltmc.enigma.impl.bytecode.translator.TranslationClassVisitor;
@@ -41,9 +43,11 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
@@ -56,22 +60,25 @@ public class EnigmaProject {
 	private final ClassProvider classProvider;
 	private final JarIndex jarIndex;
 	private final JarIndex libIndex;
+	private final JarIndex combinedIndex;
 	private final byte[] jarChecksum;
+	private final Map<MethodEntry, Boolean> libraryMethodOverrideCache = new HashMap<>();
 
 	private EntryRemapper remapper;
 	private MappingsIndex mappingsIndex;
 
-	public EnigmaProject(Enigma enigma, Path jarPath, ClassProvider classProvider, JarIndex jarIndex, JarIndex libIndex, MappingsIndex mappingsIndex, EntryTree<EntryMapping> proposedNames, byte[] jarChecksum) {
+	public EnigmaProject(Enigma enigma, Path jarPath, ClassProvider classProvider, JarIndex jarIndex, JarIndex libIndex, JarIndex combinedIndex, MappingsIndex mappingsIndex, EntryTree<EntryMapping> proposedNames, byte[] jarChecksum) {
 		Preconditions.checkArgument(jarChecksum.length == 20);
 		this.enigma = enigma;
 		this.jarPath = jarPath;
 		this.classProvider = classProvider;
 		this.jarIndex = jarIndex;
 		this.libIndex = libIndex;
+		this.combinedIndex = combinedIndex;
 		this.jarChecksum = jarChecksum;
 
 		this.mappingsIndex = mappingsIndex;
-		this.remapper = EntryRemapper.mapped(enigma, jarIndex, this.mappingsIndex, proposedNames, new HashEntryTree<>(), this.enigma.getNameProposalServices());
+		this.remapper = EntryRemapper.mapped(this.enigma, this.combinedIndex, this.mappingsIndex, proposedNames, new HashEntryTree<>(), this.enigma.getNameProposalServices());
 	}
 
 	/**
@@ -90,12 +97,12 @@ public void setMappings(@Nullable EntryTree<EntryMapping> mappings, ProgressList
 			EntryTree<EntryMapping> mergedTree = EntryTreeUtil.merge(jarProposedMappings, mappings);
 
 			this.mappingsIndex.indexMappings(mergedTree, progress);
-			this.remapper = EntryRemapper.mapped(this.enigma, this.jarIndex, this.mappingsIndex, jarProposedMappings, mappings, this.enigma.getNameProposalServices());
+			this.remapper = EntryRemapper.mapped(this.enigma, this.combinedIndex, this.mappingsIndex, jarProposedMappings, mappings, this.enigma.getNameProposalServices());
 		} else if (!jarProposedMappings.isEmpty()) {
 			this.mappingsIndex.indexMappings(jarProposedMappings, progress);
-			this.remapper = EntryRemapper.mapped(this.enigma, this.jarIndex, this.mappingsIndex, jarProposedMappings, new HashEntryTree<>(), this.enigma.getNameProposalServices());
+			this.remapper = EntryRemapper.mapped(this.enigma, this.combinedIndex, this.mappingsIndex, jarProposedMappings, new HashEntryTree<>(), this.enigma.getNameProposalServices());
 		} else {
-			this.remapper = EntryRemapper.empty(this.enigma, this.jarIndex, this.enigma.getNameProposalServices());
+			this.remapper = EntryRemapper.empty(this.enigma, this.combinedIndex, this.enigma.getNameProposalServices());
 		}
 
 		// update dynamically proposed names
@@ -114,10 +121,27 @@ public ClassProvider getClassProvider() {
 		return this.classProvider;
 	}
 
+	/**
+	 * Gets the index of the main jar of this project; the jar being mapped.
+	 */
 	public JarIndex getJarIndex() {
 		return this.jarIndex;
 	}
 
+	/**
+	 * Gets the index of the library jars of this project.
+	 */
+	public JarIndex getLibIndex() {
+		return this.libIndex;
+	}
+
+	/**
+	 * Gets the index of the main jar <em>and</em> library jars of this project.
+	 */
+	public JarIndex getCombinedIndex() {
+		return this.combinedIndex;
+	}
+
 	public MappingsIndex getMappingsIndex() {
 		return this.mappingsIndex;
 	}
@@ -199,6 +223,10 @@ public boolean isRenamable(Entry<?> obfEntry) {
 					|| isEnumValueOfMethod(parent, obfMethodEntry))) {
 				return false;
 			}
+
+			if (this.isLibraryMethodOverride(obfMethodEntry)) {
+				return false;
+			}
 		} else if (obfEntry instanceof LocalVariableEntry localEntry && !localEntry.isArgument()) {
 			return false;
 		} else if (obfEntry instanceof LocalVariableEntry localEntry && localEntry.isArgument()) {
@@ -216,6 +244,36 @@ public boolean isRenamable(Entry<?> obfEntry) {
 		return this.jarIndex.getIndex(EntryIndex.class).hasEntry(obfEntry);
 	}
 
+	private boolean isLibraryMethodOverride(MethodEntry methodEntry) {
+		final Boolean cached = this.libraryMethodOverrideCache.get(methodEntry);
+		if (cached != null) {
+			return cached;
+		} else {
+			if (this.combinedIndex.getIndex(EntryIndex.class).hasMethod(methodEntry)) {
+				final EntryResolver combinedResolver = this.combinedIndex.getEntryResolver();
+				final Set<MethodEntry> equivalents = combinedResolver.resolveEquivalentMethods(methodEntry);
+				final Set<MethodEntry> roots = equivalents.stream()
+						.flatMap(equivalent -> combinedResolver.resolveEntry(equivalent, ResolutionStrategy.RESOLVE_ROOT).stream())
+						.collect(Collectors.toSet());
+
+				final Set<MethodEntry> equivalentsAndRoots = Stream
+						.concat(equivalents.stream(), roots.stream())
+						.collect(Collectors.toSet());
+
+				final EntryIndex jarEntryIndex = this.jarIndex.getIndex(EntryIndex.class);
+				final boolean anyNonJar = equivalentsAndRoots.stream().anyMatch(method -> !jarEntryIndex.hasMethod(method));
+
+				equivalentsAndRoots.forEach(method -> this.libraryMethodOverrideCache.put(method, anyNonJar));
+
+				return anyNonJar;
+			} else {
+				this.libraryMethodOverrideCache.put(methodEntry, false);
+
+				return false;
+			}
+		}
+	}
+
 	private static boolean isEnumValueOfMethod(ClassDefEntry parent, MethodEntry method) {
 		return parent != null && parent.isEnum() && method.getName().equals("valueOf") && method.getDesc().toString().equals("(Ljava/lang/String;)L" + parent.getFullName() + ";");
 	}