diff --git a/plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist.java b/plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist.java
index 9cea6cf65..453891011 100644
--- a/plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist.java
+++ b/plugin/src/main/java/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist.java
@@ -36,10 +36,11 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import jenkins.util.SystemProperties;
@@ -135,12 +136,7 @@ public Enumeration<URL> getResources(String name) throws IOException {
 
         private static boolean isAllowed(URL url) {
             String urlString = url.toString();
-            for (GroovySourceFileAllowlist allowlist : GroovySourceFileAllowlist.all()) {
-                if (allowlist.isAllowed(urlString)) {
-                    return true;
-                }
-            }
-            return false;
+            return ExtensionList.lookupSingleton(AllowedGroovyResourcesCache.class).isAllowed(urlString);
         }
 
         private static boolean endsWithIgnoreCase(String value, String suffix) {
@@ -149,6 +145,27 @@ private static boolean endsWithIgnoreCase(String value, String suffix) {
         }
     }
 
+    @Extension
+    @SuppressFBWarnings(value = "NP_BOOLEAN_RETURN_NULL", justification = "intentionally not caching negative results")
+    public static class AllowedGroovyResourcesCache {
+        private final Map<String, Boolean> cache = new ConcurrentHashMap<>();
+
+        public boolean isAllowed(String groovySourceFileUrl) {
+            Boolean cachedResult = cache.computeIfAbsent(groovySourceFileUrl, url -> {
+                for (GroovySourceFileAllowlist allowlist : GroovySourceFileAllowlist.all()) {
+                    if (allowlist.isAllowed(url)) {
+                        return true;
+                    }
+                }
+                // In practice we should only get here with files that are allowed, so we don't cache negative
+                // results in case it would cause problems with unusual Pipelines that reference Groovy source
+                // files directly in combination with dynamically installed plugins.
+                return null;
+            });
+            return Boolean.TRUE.equals(cachedResult);
+        }
+    }
+
     /**
      * Allows Groovy source files used to implement DSLs in plugins that were created before
      * {@link GroovySourceFileAllowlist} was introduced.
diff --git a/plugin/src/main/resources/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist/default-allowlist b/plugin/src/main/resources/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist/default-allowlist
index 132684793..78619ea0f 100644
--- a/plugin/src/main/resources/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist/default-allowlist
+++ b/plugin/src/main/resources/org/jenkinsci/plugins/workflow/cps/GroovySourceFileAllowlist/default-allowlist
@@ -1,10 +1,8 @@
 # This list is ordered from most popular to least popular plugin to minimize performance impact.
 # pipeline-model-definition
-/org/jenkinsci/plugins/pipeline/modeldefinition/ModelInterpreter.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/agent/impl/AnyScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/agent/impl/LabelScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/agent/impl/NoneScript.groovy
-/org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/AbstractChangelogConditionalScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/AllOfConditionalScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/AnyOfConditionalScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/BranchConditionalScript.groovy
@@ -18,11 +16,7 @@
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/NotConditionalScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/TagConditionalScript.groovy
 /org/jenkinsci/plugins/pipeline/modeldefinition/when/impl/TriggeredByConditionalScript.groovy
-# pipeline-model-extensions
-/org/jenkinsci/plugins/pipeline/modeldefinition/agent/CheckoutScript.groovy
 # docker-workflow
-/org/jenkinsci/plugins/docker/workflow/Docker.groovy
-/org/jenkinsci/plugins/docker/workflow/declarative/AbstractDockerPipelineScript.groovy
 /org/jenkinsci/plugins/docker/workflow/declarative/DockerPipelineFromDockerfileScript.groovy
 /org/jenkinsci/plugins/docker/workflow/declarative/DockerPipelineScript.groovy
 # kubernetes