diff --git a/src/core/xref.js b/src/core/xref.js
index da7491075b709..af185c6057979 100644
--- a/src/core/xref.js
+++ b/src/core/xref.js
@@ -31,6 +31,7 @@ import {
 } from "./core_utils.js";
 import { BaseStream } from "./base_stream.js";
 import { CipherTransformFactory } from "./crypto.js";
+import { NameTree } from "./name_number_tree.js";
 
 class XRef {
   #firstXRefStmPos = null;
@@ -118,22 +119,6 @@ class XRef {
       }
       warn(`XRef.parse - Invalid "Encrypt" reference: "${ex}".`);
     }
-    if (encrypt instanceof Dict) {
-      const ids = trailerDict.get("ID");
-      const fileId = ids?.length ? ids[0] : "";
-      // The 'Encrypt' dictionary itself should not be encrypted, and by
-      // setting `suppressEncryption` we can prevent an infinite loop inside
-      // of `XRef_fetchUncompressed` if the dictionary contains indirect
-      // objects (fixes issue7665.pdf).
-      encrypt.suppressEncryption = true;
-      this.encrypt = new CipherTransformFactory(
-        encrypt,
-        fileId,
-        this.pdfManager.password
-      );
-    }
-
-    // Get the root dictionary (catalog) object, and do some basic validation.
     let root;
     try {
       root = trailerDict.get("Root");
@@ -143,6 +128,43 @@ class XRef {
       }
       warn(`XRef.parse - Invalid "Root" reference: "${ex}".`);
     }
+
+    if (encrypt instanceof Dict) {
+      // Check if only the file attachments are encrypted.
+      if (
+        encrypt.get("CF")?.get("StdCF")?.get("AuthEvent")?.name === "EFOpen"
+      ) {
+        let hasEncryptedAttachments = false;
+        if (root instanceof Dict) {
+          const names = root.get("Names");
+          if (names instanceof Dict && names.has("EmbeddedFiles")) {
+            const nameTree = new NameTree(names.getRaw("EmbeddedFiles"), this);
+            const attachments = nameTree.getAll();
+            if (attachments.size > 0) {
+              hasEncryptedAttachments = true;
+            }
+          }
+        }
+        if (!hasEncryptedAttachments) {
+          // If there are no encrypted attachments, encrypt dictionary is
+          // not needed.
+          encrypt = null;
+        }
+      } else {
+        const ids = trailerDict.get("ID");
+        const fileId = ids?.length ? ids[0] : "";
+        // The 'Encrypt' dictionary itself should not be encrypted, and by
+        // setting `suppressEncryption` we can prevent an infinite loop inside
+        // of `XRef_fetchUncompressed` if the dictionary contains indirect
+        // objects (fixes issue7665.pdf).
+        encrypt.suppressEncryption = true;
+        this.encrypt = new CipherTransformFactory(
+          encrypt,
+          fileId,
+          this.pdfManager.password
+        );
+      }
+    }
     if (root instanceof Dict) {
       try {
         const pages = root.get("Pages");
diff --git a/test/pdfs/.gitignore b/test/pdfs/.gitignore
index 233649a1f1df1..582e32a584fc3 100644
--- a/test/pdfs/.gitignore
+++ b/test/pdfs/.gitignore
@@ -385,6 +385,7 @@
 !bug1020226.pdf
 !issue9534_reduced.pdf
 !attachment.pdf
+!issue20049.pdf
 !basicapi.pdf
 !issue15590.pdf
 !issue15594_reduced.pdf
diff --git a/test/pdfs/issue20049.pdf b/test/pdfs/issue20049.pdf
new file mode 100644
index 0000000000000..ec0ed82e0f421
Binary files /dev/null and b/test/pdfs/issue20049.pdf differ
diff --git a/test/test_manifest.json b/test/test_manifest.json
index 2e28f992fc89a..8c3b37bb6aea0 100644
--- a/test/test_manifest.json
+++ b/test/test_manifest.json
@@ -4930,6 +4930,13 @@
     "rounds": 1,
     "type": "eq"
   },
+  {
+    "id": "issue20049",
+    "file": "pdfs/issue20049.pdf",
+    "md5": "1cdfde56be6b070e0c18aafc487d92ff",
+    "rounds": 1,
+    "type": "eq"
+  },
   {
     "id": "issue8117",
     "file": "pdfs/issue8117.pdf",
diff --git a/test/unit/api_spec.js b/test/unit/api_spec.js
index e5fdf5d3003a3..52714028b8abe 100644
--- a/test/unit/api_spec.js
+++ b/test/unit/api_spec.js
@@ -878,6 +878,21 @@ describe("api", function () {
 
       await loadingTask.destroy();
     });
+
+    it("should not prompt for password if only attachments are encrypted and there are none", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue20049.pdf"));
+      expect(loadingTask instanceof PDFDocumentLoadingTask).toEqual(true);
+
+      loadingTask.onPassword = function (callback, reason) {
+        if (reason === PasswordResponses.NEED_PASSWORD) {
+          expect(false).toEqual(true);
+          throw new Error("Should not prompt for password.");
+        }
+      };
+
+      const pdfDocument = await loadingTask.promise;
+      expect(pdfDocument.numPages).toBeGreaterThan(0);
+    });
   });
 
   describe("PDFWorker", function () {