Merge pull request #138 from codellm-devkit/issue136-duplicate-key-error

Fixes for duplicate key and stack overflow errors, and updates in call site model and symbol table construction

Author: rahlk

build.gradle

@@ -17,6 +17,7 @@ plugins {
     id 'application'
     id 'org.graalvm.buildtools.native' version '0.10.4'
     id 'org.jetbrains.kotlin.jvm'
+    id 'com.diffplug.spotless' version '6.25.0'
 }
 
 // Get the version from the property file first
@@ -145,6 +146,20 @@ test {
     systemProperty 'testcontainers.reuse.enable', 'true'
 }
 
+spotless {
+    java {
+        target 'src/**/*.java'
+        trimTrailingWhitespace()
+        endWithNewline()
+        importOrder()
+    }
+}
+
+compileJava.dependsOn spotlessApply
+
+// Optionally, automatically format before compilation
+// compileJava.dependsOn googleJavaFormat
+
 task fatJar(type: Jar) {
     archiveBaseName = 'codeanalyzer'
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE

gradle.properties

@@ -1 +1 @@
-version=2.3.4
+version=2.3.5

src/main/java/com/ibm/cldk/CodeAnalyzer.java

@@ -12,6 +12,17 @@
  */
 package com.ibm.cldk;
 
+import com.github.javaparser.Problem;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.ibm.cldk.entities.JavaCompilationUnit;
+import com.ibm.cldk.utils.BuildProject;
+import com.ibm.cldk.utils.Log;
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
@@ -23,21 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-
 import org.apache.commons.lang3.tuple.Pair;
-
-import com.github.javaparser.Problem;
-import com.google.common.reflect.TypeToken;
-import com.google.gson.FieldNamingPolicy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.ibm.cldk.entities.JavaCompilationUnit;
-import com.ibm.cldk.utils.BuildProject;
-import com.ibm.cldk.utils.Log;
-
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;

src/main/java/com/ibm/cldk/SymbolTable.java

@@ -1,38 +1,19 @@
 package com.ibm.cldk;
 
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
 import com.github.javaparser.*;
 import com.github.javaparser.ast.*;
 import com.github.javaparser.ast.body.*;
 import com.github.javaparser.ast.comments.Comment;
 import com.github.javaparser.ast.comments.JavadocComment;
 import com.github.javaparser.ast.expr.*;
 import com.github.javaparser.ast.nodeTypes.NodeWithJavadoc;
-import com.github.javaparser.ast.stmt.*;
-import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
-import com.ibm.cldk.entities.*;
-import com.ibm.cldk.javaee.EntrypointsFinderFactory;
-import org.apache.commons.lang3.tuple.Pair;
-
 import com.github.javaparser.ast.nodeTypes.NodeWithName;
+import com.github.javaparser.ast.stmt.*;
 import com.github.javaparser.ast.type.ReferenceType;
 import com.github.javaparser.ast.type.Type;
+import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
 import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration;
+import com.github.javaparser.resolution.declarations.ResolvedMethodLikeDeclaration;
 import com.github.javaparser.resolution.types.ResolvedType;
 import com.github.javaparser.symbolsolver.JavaSymbolSolver;
 import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
@@ -42,10 +23,20 @@
 import com.github.javaparser.utils.SourceRoot;
 import com.google.common.collect.Table;
 import com.google.common.collect.Tables;
+import com.ibm.cldk.entities.*;
 import com.ibm.cldk.javaee.CRUDFinderFactory;
+import com.ibm.cldk.javaee.EntrypointsFinderFactory;
 import com.ibm.cldk.javaee.utils.enums.CRUDOperationType;
 import com.ibm.cldk.javaee.utils.enums.CRUDQueryType;
 import com.ibm.cldk.utils.Log;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.commons.lang3.tuple.Pair;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class SymbolTable {
@@ -501,7 +492,7 @@ private static Pair<String, Callable> processCallableDeclaration(CallableDeclara
         callableNode.setFilePath(filePath);
 
         // add callable signature
-        callableNode.setSignature(callableDecl.getSignature().asString());
+        callableNode.setSignature(getTypeErasureSignature(callableDecl));
 
         // add comment associated with method/constructor
         callableNode.setComments(
@@ -581,11 +572,62 @@ private static Pair<String, Callable> processCallableDeclaration(CallableDeclara
         callableNode.setVariableDeclarations(getVariableDeclarations(body));
         callableNode.setCyclomaticComplexity(getCyclomaticComplexity(callableDecl));
 
-        String callableSignature = (callableDecl instanceof MethodDeclaration) ? callableDecl.getSignature().asString()
-                : callableDecl.getSignature().asString().replace(callableDecl.getSignature().getName(), "<init>");
+        String callableSignature = getTypeErasureSignature(callableDecl);
         return Pair.of(callableSignature, callableNode);
     }
 
+    /**
+     * Returns type erasure signature for the given callable. Returns regular signature if an
+     * error occurs in getting erased types.
+     *
+     * @param callableDecl: Callable to compute type erasure signature for
+     * @return String representing type erasure or regular signature
+     */
+    private static String getTypeErasureSignature(CallableDeclaration callableDecl) {
+        try {
+            StringBuilder signature = new StringBuilder(
+                    (callableDecl instanceof MethodDeclaration) ? callableDecl.getNameAsString() : "<init>"
+            );
+            List<String> erasureParameterTypes = new ArrayList<>();
+            for (Object param : callableDecl.getParameters()) {
+                Parameter parameter = (Parameter) param;
+                ResolvedType resolvedType = parameter.getType().resolve();
+                if (parameter.isVarArgs()) {
+                    erasureParameterTypes.add(resolvedType.describe() + "[]");
+                } else {
+                    erasureParameterTypes.add(resolvedType.erasure().describe());
+                }
+            }
+            signature.append("(");
+            signature.append(String.join(", ", erasureParameterTypes));
+            signature.append(")");
+            return signature.toString();
+        } catch (Throwable e) {
+            Log.warn("Could not compute type erasure signature for "+callableDecl.getSignature().asString()+
+                    "; computing regular signature");
+            return callableDecl.getSignature().asString();
+        }
+    }
+
+    /**
+     * Returns type erasure signature for the given method or constructor declaration
+     * resolved for a call site.
+     *
+     * @param methodDecl: Resolved method/constructor to compute type erasure signature for
+     * @return String representing type erasure signature
+     */
+    private static String getTypeErasureSignature(ResolvedMethodLikeDeclaration methodDecl) {
+        StringBuilder signature = new StringBuilder(methodDecl.getName());
+        List<String> erasureParameterTypes = new ArrayList<>();
+        for (int i = 0; i < methodDecl.getNumberOfParams(); i++) {
+            erasureParameterTypes.add(methodDecl.getParam(i).getType().erasure().describe());
+        }
+        signature.append("(");
+        signature.append(String.join(", ", erasureParameterTypes));
+        signature.append(")");
+        return signature.toString();
+    }
+
     private static boolean isEntryPointMethod(CallableDeclaration callableDecl) {
         return EntrypointsFinderFactory.getEntrypointFinders()
                 .anyMatch(finder -> finder.isEntrypointMethod(callableDecl));
@@ -835,8 +877,8 @@ private static List<CallSite> getCallSites(Optional<BlockStmt> callableBody) {
             // resolve callee and get signature
             String calleeSignature = "";
             try {
-                calleeSignature = methodCallExpr.resolve().getSignature();
-            } catch (RuntimeException exception) {
+                calleeSignature = getTypeErasureSignature(methodCallExpr.resolve());
+            } catch (Throwable exception) {
                 Log.debug("Could not resolve method call: " + methodCallExpr + ": " + exception.getMessage());
             }
 
@@ -845,12 +887,12 @@ private static List<CallSite> getCallSites(Optional<BlockStmt> callableBody) {
             try {
                 ResolvedMethodDeclaration resolvedMethodDeclaration = methodCallExpr.resolve();
                 accessSpecifier = resolvedMethodDeclaration.accessSpecifier();
-            } catch (RuntimeException exception) {
+            } catch (Throwable exception) {
                 Log.debug("Could not resolve access specifier for method call: " + methodCallExpr + ": "
                         + exception.getMessage());
             }
             // resolve arguments of the method call to types
-            List<String> arguments = methodCallExpr.getArguments().stream().map(SymbolTable::resolveExpression)
+            List<String> argumentTypes = methodCallExpr.getArguments().stream().map(SymbolTable::resolveExpression)
                     .collect(Collectors.toList());
             // Get argument string from the callsite
             List<String> listOfArgumentStrings = methodCallExpr.getArguments().stream().map(Expression::toString)
@@ -886,7 +928,7 @@ private static List<CallSite> getCallSites(Optional<BlockStmt> callableBody) {
 
 
             callSites.add(createCallSite(methodCallExpr, methodCallExpr.getNameAsString(), receiverName, declaringType,
-                    arguments, returnType, calleeSignature, isStaticCall, false, crudOperation, crudQuery,
+                    argumentTypes, listOfArgumentStrings, returnType, calleeSignature, isStaticCall, false, crudOperation, crudQuery,
                     accessSpecifier));
         }
 
@@ -895,14 +937,18 @@ private static List<CallSite> getCallSites(Optional<BlockStmt> callableBody) {
             String instantiatedType = resolveType(objectCreationExpr.getType());
 
             // resolve arguments of the constructor call to types
-            List<String> arguments = objectCreationExpr.getArguments().stream().map(SymbolTable::resolveExpression)
+            List<String> argumentTypes = objectCreationExpr.getArguments().stream().map(SymbolTable::resolveExpression)
+                    .collect(Collectors.toList());
+
+            // get argument expressions for constructor call
+            List<String> argumentExpressions = objectCreationExpr.getArguments().stream().map(Expression::toString)
                     .collect(Collectors.toList());
 
             // resolve callee and get signature
             String calleeSignature = "";
             try {
-                calleeSignature = objectCreationExpr.resolve().getSignature();
-            } catch (RuntimeException exception) {
+                calleeSignature = getTypeErasureSignature(objectCreationExpr.resolve());
+            } catch (Throwable exception) {
                 Log.debug("Could not resolve constructor call: " + objectCreationExpr + ": " + exception.getMessage());
             }
 
@@ -911,7 +957,7 @@ private static List<CallSite> getCallSites(Optional<BlockStmt> callableBody) {
                     .add(createCallSite(objectCreationExpr, "<init>",
                             objectCreationExpr.getScope().isPresent() ? objectCreationExpr.getScope().get().toString()
                                     : "",
-                            instantiatedType, arguments, instantiatedType, calleeSignature, false, true, null, null,
+                            instantiatedType, argumentTypes, argumentExpressions, instantiatedType, calleeSignature, false, true, null, null,
                             AccessSpecifier.NONE));
         }
 
@@ -962,17 +1008,24 @@ private static Optional<CRUDOperationType> findCRUDOperation(String declaringTyp
      * @param calleeName
      * @param receiverExpr
      * @param receiverType
-     * @param arguments
+     * @param argumentTypes
+     * @param argumentExpr
+     * @param returnType
+     * @param calleeSignature
      * @param isStaticCall
      * @param isConstructorCall
+     * @param crudOperation,
+     * @param crudQuery,
+     * @param accessSpecifier
      * @return
      */
     private static CallSite createCallSite(
             Expression callExpr,
             String calleeName,
             String receiverExpr,
             String receiverType,
-            List<String> arguments,
+            List<String> argumentTypes,
+            List<String> argumentExpr,
             String returnType,
             String calleeSignature,
             boolean isStaticCall,
@@ -998,7 +1051,8 @@ private static CallSite createCallSite(
         callSite.setMethodName(calleeName);
         callSite.setReceiverExpr(receiverExpr);
         callSite.setReceiverType(receiverType);
-        callSite.setArgumentTypes(arguments);
+        callSite.setArgumentTypes(argumentTypes);
+        callSite.setArgumentExpr(argumentExpr);
         callSite.setReturnType(returnType);
         callSite.setCalleeSignature(calleeSignature);
         callSite.setStaticCall(isStaticCall);
@@ -1039,7 +1093,7 @@ private static String resolveExpression(Expression expression) {
                 if (resolvedType.isReferenceType() || resolvedType.isUnionType()) {
                     return resolvedType.describe();
                 }
-            } catch (RuntimeException exception) {
+            } catch (Throwable exception) {
                 Log.debug("Could not resolve expression: " + expression + ": " + exception.getMessage());
                 unresolvedExpressions.add(expression.toString());
             }
@@ -1060,7 +1114,7 @@ private static String resolveType(Type type) {
         if (!unresolvedTypes.contains(type.asString())) {
             try {
                 return type.resolve().describe();
-            } catch (RuntimeException e) {
+            } catch (Throwable e) {
                 Log.warn("Could not resolve type: " + type.asString() + ": " + e.getMessage());
                 unresolvedTypes.add(type.asString());
             }
@@ -1079,6 +1133,20 @@ private static String resolveType(Type type) {
      *         project
      * @throws IOException
      */
+    private static final String[] EXCLUDED_SOURCE_ROOTS = {
+            Paths.get("src", "test", "resources").toString(),
+            Paths.get("src", "it", "resources").toString(),
+            Paths.get("src", "xdocs-examples").toString()
+    };
+    private static boolean excludeSourceRoot(Path sourceRoot) {
+        for (String excludedSrcRoot : EXCLUDED_SOURCE_ROOTS) {
+            if (Pattern.compile(excludedSrcRoot).matcher(sourceRoot.toString()).find()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> extractAll(Path projectRootPath)
             throws IOException {
         SymbolSolverCollectionStrategy symbolSolverCollectionStrategy = new SymbolSolverCollectionStrategy();
@@ -1088,6 +1156,9 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
         Map<String, JavaCompilationUnit> symbolTable = new LinkedHashMap<>();
         Map<String, List<Problem>> parseProblems = new HashMap<>();
         for (SourceRoot sourceRoot : projectRoot.getSourceRoots()) {
+            if (excludeSourceRoot(sourceRoot.getRoot())) {
+                continue;
+            }
             for (ParseResult<CompilationUnit> parseResult : sourceRoot.tryToParse()) {
                 if (parseResult.isSuccessful()) {
                     CompilationUnit compilationUnit = LexicalPreservingPrinter.setup(parseResult.getResult().get());