Skip to content

Add AsyncPCK support (ResourceLoader.load_threaded_* version) #116673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adamscott wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Add AsyncPCK support (ResourceLoader.load_threaded_* version) #116673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cce6bd2
Add AsyncPCK support
adamscott Nov 13, 2025
ba67dc0
Apply dsnopek suggestions
adamscott Apr 9, 2026
ee98422
Apply some of KoBeWi's suggestions
adamscott Apr 9, 2026
e7af7f1
Remove incompatible Closure compiler
adamscott Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/web_builds.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ env:
SCONS_FLAGS: >-
dev_mode=yes
debug_symbols=no
use_closure_compiler=yes
EM_VERSION: 4.0.11

jobs:
Expand Down
13 changes: 10 additions & 3 deletions core/config/project_settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,11 +587,18 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f
return false;
}

if (p_pack == "res://") {
String pack = p_pack;
if (pack == "res://") {
// Loading the resource directory as a pack source is reserved for internal use only.
return false;
}

pack = pack.trim_suffix("/");

if (pack.ends_with(".asyncpck")) {
PackedData::get_singleton()->add_pack_source(memnew(PackedSourceAsyncPCK));
}

if (!p_main_pack && !using_datapack && !OS::get_singleton()->get_resource_dir().is_empty()) {
// Add the project's resource file system to PackedData so directory access keeps working when
// the game is running without a main pack, like in the editor or on Android.
Expand All @@ -601,7 +608,7 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f
using_datapack = true;
}

bool ok = PackedData::get_singleton()->add_pack(p_pack, p_replace_files, p_offset) == OK;
bool ok = PackedData::get_singleton()->add_pack(pack, p_replace_files, p_offset) == OK;
if (!ok) {
return false;
}
Expand Down Expand Up @@ -761,7 +768,7 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
}
}

#ifdef ANDROID_ENABLED
#if defined(ANDROID_ENABLED) || defined(WEB_ENABLED)
// Attempt to load sparse PCK assets.
_load_resource_pack("res://assets.sparsepck", false, 0, true);
#endif
Expand Down
123 changes: 92 additions & 31 deletions core/io/file_access_pack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,55 +46,58 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
return ERR_FILE_UNRECOGNIZED;
}

void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle, bool p_delta, const String &p_salt) {
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, BitField<PackedFile::PackedFileProperty> p_properties, const String &p_salt) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());

bool exists = files.has(pmd5);

PackedFile pf;
pf.encrypted = p_encrypted;
pf.bundle = p_bundle;
pf.delta = p_delta;
pf.properties = p_properties;
pf.pack = p_pkg_path;
pf.salt = p_salt;
pf.offset = p_ofs;
pf.size = p_size;
for (int i = 0; i < 16; i++) {
pf.md5[i] = p_md5[i];
}
memcpy(pf.md5, p_md5, 16);
pf.src = p_src;

if (p_delta) {
if (pf.properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA)) {
delta_patches[pmd5].push_back(pf);
} else if (!exists || p_replace_files) {
files[pmd5] = pf;
delta_patches[pmd5].clear();
}

if (!exists) {
// Search for directory.
PackedDir *cd = root;
if (exists) {
return;
}

if (simplified_path.contains_char('/')) { // In a subdirectory.
Vector<String> ds = simplified_path.get_base_dir().split("/");
// Search for directory.
PackedDir *cd = root;

for (int j = 0; j < ds.size(); j++) {
if (!cd->subdirs.has(ds[j])) {
PackedDir *pd = memnew(PackedDir);
pd->name = ds[j];
pd->parent = cd;
cd->subdirs[pd->name] = pd;
cd = pd;
} else {
cd = cd->subdirs[ds[j]];
}
if (simplified_path.contains_char('/')) { // In a subdirectory.
for (const String &dir : simplified_path.get_base_dir().split("/")) {
if (!cd->subdirs.has(dir)) {
PackedDir *pd = memnew(PackedDir);
pd->name = dir;
pd->parent = cd;
cd->subdirs[pd->name] = pd;
cd = pd;
} else {
cd = cd->subdirs[dir];
}
}
String filename = simplified_path.get_file();
// Don't add as a file if the path points to a directory.
if (!filename.is_empty()) {
cd->files.insert(filename);
}
String filename = simplified_path.get_file();
// Don't add as a file if the path points to a directory.
if (!filename.is_empty()) {
cd->files.insert(filename);
}

if (p_properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC)) {
Ref<FileAccess> file = FileAccess::create_for_path(p_path);
if (!file->file_exists(p_path)) {
async_files[pmd5] = pf;
}
}
}
Expand Down Expand Up @@ -165,6 +168,26 @@ bool PackedData::has_delta_patches(const String &p_path) const {
return !E->value.is_empty();
}

String PackedData::get_file_pack_path(const String &p_path) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, PackedFile, PathMD5>::Iterator file_iterator = files.find(pmd5);
if (!file_iterator) {
return "";
}
return file_iterator->value.pack;
}

String PackedData::get_file_async_pack_path(const String &p_path) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, PackedFile, PathMD5>::Iterator file_iterator = async_files.find(pmd5);
if (!file_iterator) {
return "";
}
return file_iterator->value.pack;
}

HashSet<String> PackedData::get_file_paths() const {
HashSet<String> file_paths;
_get_file_paths(root, root->name, file_paths);
Expand Down Expand Up @@ -299,6 +322,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3.
bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE);
bool async_pck = (pack_flags & PACK_ASYNC);
String salt;

uint64_t file_base = f->get_64();
Expand Down Expand Up @@ -366,7 +390,20 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
if (flags & PACK_FILE_REMOVAL) { // The file was removed.
PackedData::get_singleton()->remove_path(path);
} else {
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle, (flags & PACK_FILE_DELTA), salt);
BitField<PackedData::PackedFile::PackedFileProperty> properties = PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_NONE;
if (flags & PACK_FILE_ENCRYPTED) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED);
}
if (sparse_bundle) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED);
}
if (flags & PACK_FILE_DELTA) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA);
}
if (async_pck) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC);
}
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, properties, salt);
}
}

Expand All @@ -389,6 +426,29 @@ Ref<FileAccess> PackedSourcePCK::get_file(const String &p_path, PackedData::Pack

//////////////////////////////////////////////////////////////////

bool PackedSourceAsyncPCK::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector<uint8_t> &p_decryption_key) {
String path = p_path.simplify_path();
if (path.ends_with("/")) {
path = path.trim_suffix("/");
}
if (!path.ends_with(".asyncpck")) {
return false;
}
path = path.path_join("assets").path_join("assets.sparsepck");
return PackedSourcePCK::try_open_pack(path, p_replace_files, p_offset, p_decryption_key);
}

Ref<FileAccess> PackedSourceAsyncPCK::get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector<uint8_t> &p_decryption_key) {
String pack = p_file->pack.get_base_dir();
String path = p_path.simplify_path().trim_prefix("res://");
String file_path = pack.path_join(path);
Ref<FileAccessPack> file_access_pack;
file_access_pack.instantiate(file_path, *p_file, p_decryption_key);
return file_access_pack;
}

//////////////////////////////////////////////////////////////////

bool PackedSourceDirectory::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector<uint8_t> &p_decryption_key) {
// Load with offset feature only supported for PCK files.
ERR_FAIL_COND_V_MSG(p_offset != 0, false, "Invalid PCK data. Note that loading files with a non-zero offset isn't supported with directories.");
Expand Down Expand Up @@ -416,7 +476,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
for (const String &file_name : da->get_files()) {
String file_path = p_path.path_join(file_name);
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false, false);
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files);
}

for (const String &sub_dir_name : da->get_directories()) {
Expand Down Expand Up @@ -526,7 +586,8 @@ void FileAccessPack::close() {
FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file, const Vector<uint8_t> &p_decryption_key) {
path = p_path;
pf = p_file;
if (pf.bundle) {

if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED)) {
String simplified_path = p_path.simplify_path();
String path_to_load = simplified_path;
#ifdef TOOLS_ENABLED
Expand All @@ -550,7 +611,7 @@ FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFil
off = pf.offset;
}

if (pf.encrypted) {
if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED)) {
Ref<FileAccessEncrypted> fae;
fae.instantiate();
ERR_FAIL_COND_MSG(fae.is_null(), vformat(R"(Can't open encrypted pack-referenced file "%s" from pack "%s".)", p_path, pf.pack));
Expand Down
55 changes: 51 additions & 4 deletions core/io/file_access_pack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ enum PackFlags {
PACK_DIR_ENCRYPTED = 1 << 0,
PACK_REL_FILEBASE = 1 << 1,
PACK_SPARSE_BUNDLE = 1 << 2,
PACK_ASYNC = 1 << 3,
};

enum PackFileFlags {
Expand All @@ -68,15 +69,22 @@ class PackedData {

public:
struct PackedFile {
enum PackedFileProperty {
PACKED_FILE_PROPERTY_NONE = 0,
PACKED_FILE_PROPERTY_ENCRYPTED = 1 << 0,
PACKED_FILE_PROPERTY_BUNDLED = 1 << 1,
PACKED_FILE_PROPERTY_DELTA = 1 << 2,
PACKED_FILE_PROPERTY_ASYNC = 1 << 3,
PACKED_FILE_PROPERTY_ALL = (PACKED_FILE_PROPERTY_ASYNC << 1) - 1,
};

String pack;
uint64_t offset; //if offset is ZERO, the file was ERASED
uint64_t size;
uint8_t md5[16];
PackSource *src = nullptr;
bool encrypted;
bool bundle;
bool delta;
String salt;
BitField<PackedFileProperty> properties;
};

private:
Expand Down Expand Up @@ -109,6 +117,7 @@ class PackedData {

HashMap<PathMD5, PackedFile, PathMD5> files;
HashMap<PathMD5, Vector<PackedFile>, PathMD5> delta_patches;
HashMap<PathMD5, PackedFile, PathMD5> async_files;

Vector<PackSource *> sources;

Expand All @@ -131,11 +140,13 @@ class PackedData {

public:
void add_pack_source(PackSource *p_source);
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false, bool p_delta = false, const String &p_salt = String()); // for PackSource
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, BitField<PackedFile::PackedFileProperty> p_properties = PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_NONE, const String &p_salt = String()); // for PackSource
void remove_path(const String &p_path);
uint8_t *get_file_hash(const String &p_path);
Vector<PackedFile> get_delta_patches(const String &p_path) const;
bool has_delta_patches(const String &p_path) const;
String get_file_pack_path(const String &p_path);
String get_file_async_pack_path(const String &p_path);
HashSet<String> get_file_paths() const;

void set_disabled(bool p_disabled) { disabled = p_disabled; }
Expand All @@ -148,6 +159,8 @@ class PackedData {

_FORCE_INLINE_ Ref<FileAccess> try_open_path(const String &p_path, const Vector<uint8_t> &p_decryption_key = Vector<uint8_t>());
_FORCE_INLINE_ bool has_path(const String &p_path);
_FORCE_INLINE_ bool has_async_path(const String &p_path);
_FORCE_INLINE_ String get_async_path(const String &p_path);

_FORCE_INLINE_ int64_t get_size(const String &p_path);

Expand All @@ -171,6 +184,12 @@ class PackedSourcePCK : public PackSource {
virtual Ref<FileAccess> get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector<uint8_t> &p_decryption_key = Vector<uint8_t>()) override;
};

class PackedSourceAsyncPCK : public PackedSourcePCK {
public:
virtual bool try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector<uint8_t> &p_decryption_key = Vector<uint8_t>()) override;
virtual Ref<FileAccess> get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector<uint8_t> &p_decryption_key = Vector<uint8_t>()) override;
};

class PackedSourceDirectory : public PackSource {
void add_directory(const String &p_path, bool p_replace_files);

Expand Down Expand Up @@ -255,6 +274,34 @@ bool PackedData::has_path(const String &p_path) {
return files.has(_get_simplified_path(p_path));
}

bool PackedData::has_async_path(const String &p_path) {
return !PackedData::get_async_path(p_path).is_empty();
}

String PackedData::get_async_path(const String &p_path) {
const String PREFIX_RES = "res://";

const String md5_path = p_path.simplify_path().trim_prefix(PREFIX_RES);
PathMD5 md5_data(md5_path.md5_buffer());
if (async_files.has(md5_data)) {
return PREFIX_RES + md5_path;
}

String md5_remap_path = md5_path + ".remap";
PathMD5 md5_remap_data(md5_remap_path.md5_buffer());
if (async_files.has(md5_remap_data)) {
return PREFIX_RES + md5_remap_path;
}

String md5_import_path = md5_path + ".import";
PathMD5 md5_import_data(md5_import_path.md5_buffer());
if (async_files.has(md5_import_data)) {
return PREFIX_RES + md5_import_path;
}

return String();
}

bool PackedData::has_directory(const String &p_path) {
Ref<DirAccess> da = try_open_directory(p_path);
if (da.is_valid()) {
Expand Down
2 changes: 1 addition & 1 deletion core/io/file_access_patched.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ Error FileAccessPatched::_apply_patch() const {

for (int i = 0; i < delta_patches.size(); ++i) {
const PackedData::PackedFile &delta_patch = delta_patches[i];
ERR_FAIL_COND_V(delta_patch.bundle, FAILED);
ERR_FAIL_COND_V(delta_patch.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED), FAILED);

Error err = OK;

Expand Down
Loading