v1.11 - Improving the "subdomains" scanner and other minor edits.

README.md

@@ -52,6 +52,15 @@ I'm open for ideas/suggestions to help improve or optimize this tool.
 ### Contributors; thanks to
 - [Stan K (@zuh4n)](https://twitter.com/zuh4n): For suggesting the dependency confusion feature, besides helping with testing and improving the functionality. 
 
+### Build from source
+```
+git clone https://github.com/minamo7sen/burp-JS-Miner.git
+cd burp-JS-Miner
+gradle fatJar
+```
+Then, the jar file can be found at `build/libs/burp-JS-Miner-all.jar`.
+
+
 ## Disclaimer
 It is the user's responsibility to obey all applicable local, state and federal laws. The author assumes no liability and is not responsible for any misuse or damage caused by this tool.
 

build.gradle

@@ -2,6 +2,9 @@ plugins {
     id 'java'
 }
 
+targetCompatibility = '1.8'
+sourceCompatibility = '1.8'
+
 repositories {
     mavenCentral()
 }

src/main/java/burp/BurpExtender.java

@@ -16,7 +16,7 @@ public class BurpExtender implements IBurpExtender, IContextMenuFactory, IExtens
     static PrintWriter mStdOut;
     static PrintWriter mStdErr;
     public static final String EXTENSION_NAME = "JS Miner";
-    private static final String EXTENSION_VERSION = "1.1";
+    private static final String EXTENSION_VERSION = "1.11";
 
     // Exposing callbacks for use in other classes
     public static IBurpExtenderCallbacks getCallbacks() {

src/main/java/burp/InterestingStuffFinder.java

@@ -175,8 +175,16 @@ private void findSecrets(IHttpRequestResponse baseRequestResponse, String respon
      * Scan function 2 - Get all subdomains
      */
     private void findSubDomains(IHttpRequestResponse baseRequestResponse, String responseBodyString) throws InterruptedException {
+        String domainFromReferer = Utilities.getDomainFromReferer(baseRequestResponse);
         String requestDomain = helpers.analyzeRequest(baseRequestResponse).getUrl().getHost();
-        String rootDomain = Utilities.getRootDomain(requestDomain);
+        String rootDomain;
+        // Try to get caller domain from Referer header (to avoid matching cdn subdomains, ..etc.)
+        if (domainFromReferer != null) {
+            rootDomain = domainFromReferer;
+        } else {
+            // If the above failed, then use the domain from the HTTP request
+            rootDomain = Utilities.getRootDomain(requestDomain);
+        }
 
         Runnable runnable = () -> {
             if (rootDomain != null) {
@@ -185,7 +193,7 @@ private void findSubDomains(IHttpRequestResponse baseRequestResponse, String res
                 Matcher matcherSubDomains = subDomainsRegex.matcher(new InterruptibleCharSequence(responseBodyString));
                 while (matcherSubDomains.find() && BurpExtender.isLoaded()) {
                     if (
-                            isMatchedDomainValid(matcherSubDomains.group(), rootDomain, requestDomain)
+                            Utilities.isMatchedDomainValid(matcherSubDomains.group(), rootDomain, requestDomain)
                     ) {
                         // Get markers of found subdomains
                         List<int[]> subDomainsMatches = Utilities.getMatches(baseRequestResponse.getResponse(), matcherSubDomains.group().getBytes());
@@ -205,14 +213,6 @@ private void findSubDomains(IHttpRequestResponse baseRequestResponse, String res
         Utilities.regexRunnerWithTimeOut(runnable);
     }
 
-    /**
-     * Make sure the found subdomain does not match (www.'request domain') or request domain or root domain
-     */
-    private static boolean isMatchedDomainValid(String matchedDomain, String rootDomain, String requestDomain) {
-        return !matchedDomain.equals("www." + requestDomain)
-                && !matchedDomain.equals(requestDomain)
-                && !matchedDomain.equals("www." + rootDomain);
-    }
 
     /**
      * Scan function 3 - Get Cloud URLs

src/main/java/burp/Utilities.java

@@ -1,6 +1,8 @@
 package burp;
 
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -47,10 +49,39 @@ public static String getRootDomain(String requestDomain) {
         // Get root Domain (e.g.: example.com instead of sub.example.com)
         Pattern rootDomainRegex = Pattern.compile("[a-z0-9]+.[a-z0-9]+$", Pattern.CASE_INSENSITIVE);
         Matcher matcherRootDomain = rootDomainRegex.matcher(requestDomain);
-            if (matcherRootDomain.find() && BurpExtender.isLoaded()) {
-                return matcherRootDomain.group();
+        if (matcherRootDomain.find() && BurpExtender.isLoaded()) {
+            return matcherRootDomain.group();
+        }
+        return null;
+    }
+
+    /**
+     * Get domain from "Referer" header to search the caller domain instead of a cdn for example
+     */
+    public static String getDomainFromReferer(IHttpRequestResponse baseRequestResponse) {
+        List<String> requestHeadersList = helpers.analyzeRequest(baseRequestResponse).getHeaders();
+        String domain;
+        for (String header : requestHeadersList) {
+            if (header.startsWith("Referer:")) {
+                domain = header.replaceAll("^Referer: ", "");
+                try {
+                    URI domainURI = new URI(domain);
+                    return getRootDomain(domainURI.getHost());
+                } catch (URISyntaxException e) {
+                    mStdErr.println("[-] URI syntax error.");
+                }
             }
-            return null;
+        }
+        return null;
+    }
+
+    /**
+     * Make sure the found subdomain does not match (www.'request domain') or request domain or root domain
+     */
+    public static boolean isMatchedDomainValid(String matchedDomain, String rootDomain, String requestDomain) {
+        return !matchedDomain.equals("www." + requestDomain)
+                && !matchedDomain.equals(requestDomain)
+                && !matchedDomain.equals("www." + rootDomain);
     }
 
     // Source: https://rosettacode.org/wiki/Entropy#Java
@@ -224,7 +255,7 @@ public static String b64Decode(String encodedString) {
 
     /**
      * A dirty method to timeout Regexes that takes so long
-     *
+     * <p>
      * Due to the fact we are using big complex Regexes,
      * sometimes with big files, this can take a long time.
      * This method mitigates this problem by killing the thread before it "kills" your CPU =)