diff --git a/.github/workflows/web_builds.yml b/.github/workflows/web_builds.yml index dba7896b4c70..c6766339dfee 100644 --- a/.github/workflows/web_builds.yml +++ b/.github/workflows/web_builds.yml @@ -8,7 +8,6 @@ env: SCONS_FLAGS: >- dev_mode=yes debug_symbols=no - use_closure_compiler=yes EM_VERSION: 4.0.11 jobs: diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index 5992b462495f..c3274062baa7 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -587,11 +587,18 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f return false; } - if (p_pack == "res://") { + String pack = p_pack; + if (pack == "res://") { // Loading the resource directory as a pack source is reserved for internal use only. return false; } + pack = pack.trim_suffix("/"); + + if (pack.ends_with(".asyncpck")) { + PackedData::get_singleton()->add_pack_source(memnew(PackedSourceAsyncPCK)); + } + if (!p_main_pack && !using_datapack && !OS::get_singleton()->get_resource_dir().is_empty()) { // Add the project's resource file system to PackedData so directory access keeps working when // the game is running without a main pack, like in the editor or on Android. @@ -601,7 +608,7 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f using_datapack = true; } - bool ok = PackedData::get_singleton()->add_pack(p_pack, p_replace_files, p_offset) == OK; + bool ok = PackedData::get_singleton()->add_pack(pack, p_replace_files, p_offset) == OK; if (!ok) { return false; } @@ -761,7 +768,7 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b } } -#ifdef ANDROID_ENABLED +#if defined(ANDROID_ENABLED) || defined(WEB_ENABLED) // Attempt to load sparse PCK assets. _load_resource_pack("res://assets.sparsepck", false, 0, true); #endif diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index f1124fd539d7..c41905e334cd 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -46,55 +46,58 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t return ERR_FILE_UNRECOGNIZED; } -void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle, bool p_delta, const String &p_salt) { +void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, BitField p_properties, const String &p_salt) { String simplified_path = p_path.simplify_path().trim_prefix("res://"); PathMD5 pmd5(simplified_path.md5_buffer()); bool exists = files.has(pmd5); PackedFile pf; - pf.encrypted = p_encrypted; - pf.bundle = p_bundle; - pf.delta = p_delta; + pf.properties = p_properties; pf.pack = p_pkg_path; pf.salt = p_salt; pf.offset = p_ofs; pf.size = p_size; - for (int i = 0; i < 16; i++) { - pf.md5[i] = p_md5[i]; - } + memcpy(pf.md5, p_md5, 16); pf.src = p_src; - if (p_delta) { + if (pf.properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA)) { delta_patches[pmd5].push_back(pf); } else if (!exists || p_replace_files) { files[pmd5] = pf; delta_patches[pmd5].clear(); } - if (!exists) { - // Search for directory. - PackedDir *cd = root; + if (exists) { + return; + } - if (simplified_path.contains_char('/')) { // In a subdirectory. - Vector ds = simplified_path.get_base_dir().split("/"); + // Search for directory. + PackedDir *cd = root; - for (int j = 0; j < ds.size(); j++) { - if (!cd->subdirs.has(ds[j])) { - PackedDir *pd = memnew(PackedDir); - pd->name = ds[j]; - pd->parent = cd; - cd->subdirs[pd->name] = pd; - cd = pd; - } else { - cd = cd->subdirs[ds[j]]; - } + if (simplified_path.contains_char('/')) { // In a subdirectory. + for (const String &dir : simplified_path.get_base_dir().split("/")) { + if (!cd->subdirs.has(dir)) { + PackedDir *pd = memnew(PackedDir); + pd->name = dir; + pd->parent = cd; + cd->subdirs[pd->name] = pd; + cd = pd; + } else { + cd = cd->subdirs[dir]; } } - String filename = simplified_path.get_file(); - // Don't add as a file if the path points to a directory. - if (!filename.is_empty()) { - cd->files.insert(filename); + } + String filename = simplified_path.get_file(); + // Don't add as a file if the path points to a directory. + if (!filename.is_empty()) { + cd->files.insert(filename); + } + + if (p_properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC)) { + Ref file = FileAccess::create_for_path(p_path); + if (!file->file_exists(p_path)) { + async_files[pmd5] = pf; } } } @@ -165,6 +168,26 @@ bool PackedData::has_delta_patches(const String &p_path) const { return !E->value.is_empty(); } +String PackedData::get_file_pack_path(const String &p_path) { + String simplified_path = p_path.simplify_path().trim_prefix("res://"); + PathMD5 pmd5(simplified_path.md5_buffer()); + HashMap::Iterator file_iterator = files.find(pmd5); + if (!file_iterator) { + return ""; + } + return file_iterator->value.pack; +} + +String PackedData::get_file_async_pack_path(const String &p_path) { + String simplified_path = p_path.simplify_path().trim_prefix("res://"); + PathMD5 pmd5(simplified_path.md5_buffer()); + HashMap::Iterator file_iterator = async_files.find(pmd5); + if (!file_iterator) { + return ""; + } + return file_iterator->value.pack; +} + HashSet PackedData::get_file_paths() const { HashSet file_paths; _get_file_paths(root, root->name, file_paths); @@ -299,6 +322,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED); bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3. bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE); + bool async_pck = (pack_flags & PACK_ASYNC); String salt; uint64_t file_base = f->get_64(); @@ -366,7 +390,20 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, if (flags & PACK_FILE_REMOVAL) { // The file was removed. PackedData::get_singleton()->remove_path(path); } else { - PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle, (flags & PACK_FILE_DELTA), salt); + BitField properties = PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_NONE; + if (flags & PACK_FILE_ENCRYPTED) { + properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED); + } + if (sparse_bundle) { + properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED); + } + if (flags & PACK_FILE_DELTA) { + properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA); + } + if (async_pck) { + properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC); + } + PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, properties, salt); } } @@ -389,6 +426,29 @@ Ref PackedSourcePCK::get_file(const String &p_path, PackedData::Pack ////////////////////////////////////////////////////////////////// +bool PackedSourceAsyncPCK::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector &p_decryption_key) { + String path = p_path.simplify_path(); + if (path.ends_with("/")) { + path = path.trim_suffix("/"); + } + if (!path.ends_with(".asyncpck")) { + return false; + } + path = path.path_join("assets").path_join("assets.sparsepck"); + return PackedSourcePCK::try_open_pack(path, p_replace_files, p_offset, p_decryption_key); +} + +Ref PackedSourceAsyncPCK::get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector &p_decryption_key) { + String pack = p_file->pack.get_base_dir(); + String path = p_path.simplify_path().trim_prefix("res://"); + String file_path = pack.path_join(path); + Ref file_access_pack; + file_access_pack.instantiate(file_path, *p_file, p_decryption_key); + return file_access_pack; +} + +////////////////////////////////////////////////////////////////// + bool PackedSourceDirectory::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector &p_decryption_key) { // Load with offset feature only supported for PCK files. ERR_FAIL_COND_V_MSG(p_offset != 0, false, "Invalid PCK data. Note that loading files with a non-zero offset isn't supported with directories."); @@ -416,7 +476,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f for (const String &file_name : da->get_files()) { String file_path = p_path.path_join(file_name); uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false, false); + PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files); } for (const String &sub_dir_name : da->get_directories()) { @@ -526,7 +586,8 @@ void FileAccessPack::close() { FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file, const Vector &p_decryption_key) { path = p_path; pf = p_file; - if (pf.bundle) { + + if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED)) { String simplified_path = p_path.simplify_path(); String path_to_load = simplified_path; #ifdef TOOLS_ENABLED @@ -550,7 +611,7 @@ FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFil off = pf.offset; } - if (pf.encrypted) { + if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED)) { Ref fae; fae.instantiate(); ERR_FAIL_COND_MSG(fae.is_null(), vformat(R"(Can't open encrypted pack-referenced file "%s" from pack "%s".)", p_path, pf.pack)); diff --git a/core/io/file_access_pack.h b/core/io/file_access_pack.h index 490778f203c3..4a53a8c04165 100644 --- a/core/io/file_access_pack.h +++ b/core/io/file_access_pack.h @@ -51,6 +51,7 @@ enum PackFlags { PACK_DIR_ENCRYPTED = 1 << 0, PACK_REL_FILEBASE = 1 << 1, PACK_SPARSE_BUNDLE = 1 << 2, + PACK_ASYNC = 1 << 3, }; enum PackFileFlags { @@ -68,15 +69,22 @@ class PackedData { public: struct PackedFile { + enum PackedFileProperty { + PACKED_FILE_PROPERTY_NONE = 0, + PACKED_FILE_PROPERTY_ENCRYPTED = 1 << 0, + PACKED_FILE_PROPERTY_BUNDLED = 1 << 1, + PACKED_FILE_PROPERTY_DELTA = 1 << 2, + PACKED_FILE_PROPERTY_ASYNC = 1 << 3, + PACKED_FILE_PROPERTY_ALL = (PACKED_FILE_PROPERTY_ASYNC << 1) - 1, + }; + String pack; uint64_t offset; //if offset is ZERO, the file was ERASED uint64_t size; uint8_t md5[16]; PackSource *src = nullptr; - bool encrypted; - bool bundle; - bool delta; String salt; + BitField properties; }; private: @@ -109,6 +117,7 @@ class PackedData { HashMap files; HashMap, PathMD5> delta_patches; + HashMap async_files; Vector sources; @@ -131,11 +140,13 @@ class PackedData { public: void add_pack_source(PackSource *p_source); - void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false, bool p_delta = false, const String &p_salt = String()); // for PackSource + void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, BitField p_properties = PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_NONE, const String &p_salt = String()); // for PackSource void remove_path(const String &p_path); uint8_t *get_file_hash(const String &p_path); Vector get_delta_patches(const String &p_path) const; bool has_delta_patches(const String &p_path) const; + String get_file_pack_path(const String &p_path); + String get_file_async_pack_path(const String &p_path); HashSet get_file_paths() const; void set_disabled(bool p_disabled) { disabled = p_disabled; } @@ -148,6 +159,8 @@ class PackedData { _FORCE_INLINE_ Ref try_open_path(const String &p_path, const Vector &p_decryption_key = Vector()); _FORCE_INLINE_ bool has_path(const String &p_path); + _FORCE_INLINE_ bool has_async_path(const String &p_path); + _FORCE_INLINE_ String get_async_path(const String &p_path); _FORCE_INLINE_ int64_t get_size(const String &p_path); @@ -171,6 +184,12 @@ class PackedSourcePCK : public PackSource { virtual Ref get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector &p_decryption_key = Vector()) override; }; +class PackedSourceAsyncPCK : public PackedSourcePCK { +public: + virtual bool try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset, const Vector &p_decryption_key = Vector()) override; + virtual Ref get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector &p_decryption_key = Vector()) override; +}; + class PackedSourceDirectory : public PackSource { void add_directory(const String &p_path, bool p_replace_files); @@ -255,6 +274,34 @@ bool PackedData::has_path(const String &p_path) { return files.has(_get_simplified_path(p_path)); } +bool PackedData::has_async_path(const String &p_path) { + return !PackedData::get_async_path(p_path).is_empty(); +} + +String PackedData::get_async_path(const String &p_path) { + const String PREFIX_RES = "res://"; + + const String md5_path = p_path.simplify_path().trim_prefix(PREFIX_RES); + PathMD5 md5_data(md5_path.md5_buffer()); + if (async_files.has(md5_data)) { + return PREFIX_RES + md5_path; + } + + String md5_remap_path = md5_path + ".remap"; + PathMD5 md5_remap_data(md5_remap_path.md5_buffer()); + if (async_files.has(md5_remap_data)) { + return PREFIX_RES + md5_remap_path; + } + + String md5_import_path = md5_path + ".import"; + PathMD5 md5_import_data(md5_import_path.md5_buffer()); + if (async_files.has(md5_import_data)) { + return PREFIX_RES + md5_import_path; + } + + return String(); +} + bool PackedData::has_directory(const String &p_path) { Ref da = try_open_directory(p_path); if (da.is_valid()) { diff --git a/core/io/file_access_patched.cpp b/core/io/file_access_patched.cpp index 4657491a9d13..cb5cd57e5d16 100644 --- a/core/io/file_access_patched.cpp +++ b/core/io/file_access_patched.cpp @@ -43,7 +43,7 @@ Error FileAccessPatched::_apply_patch() const { for (int i = 0; i < delta_patches.size(); ++i) { const PackedData::PackedFile &delta_patch = delta_patches[i]; - ERR_FAIL_COND_V(delta_patch.bundle, FAILED); + ERR_FAIL_COND_V(delta_patch.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED), FAILED); Error err = OK; diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp index a2085cdd395e..7d1c64698bea 100644 --- a/core/io/resource_loader.cpp +++ b/core/io/resource_loader.cpp @@ -274,9 +274,12 @@ ResourceLoader::LoadToken::~LoadToken() { Ref ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, CacheMode p_cache_mode, Error *r_error, bool p_use_sub_threads, float *r_progress) { const String &original_path = p_original_path.is_empty() ? p_path : p_original_path; - load_nesting++; + ERR_FAIL_COND_V_MSG( + OS::get_singleton()->async_pck_is_supported() && OS::get_singleton()->async_pck_is_file_installable(p_original_path) && !OS::get_singleton()->async_pck_is_file_installed(p_original_path), + Ref(), + vformat(R"*("%s" is an AsyncPCK file that needs to be downloaded first before use. You need to call `ResourceLoader.load_threaded_request()` and wait for the file to load first.)*", p_original_path)); - print_verbose(vformat("Loading resource: %s", p_path)); + load_nesting++; // Try all loaders and pick the first match for the type hint bool found = false; @@ -541,6 +544,12 @@ Ref ResourceLoader::_load_start(const String &p_path, bool ignoring_cache = p_cache_mode == CACHE_MODE_IGNORE || p_cache_mode == CACHE_MODE_IGNORE_DEEP; + LoadThreadMode thread_mode = +#ifdef THREADS_ENABLED + p_thread_mode; +#else + LOAD_THREAD_FROM_CURRENT; +#endif // THREADS_ENABLED Ref load_token; bool must_not_register = false; ThreadLoadTask *load_task_ptr = nullptr; @@ -584,8 +593,8 @@ Ref ResourceLoader::_load_start(const String &p_path, load_task.local_path = local_path; load_task.type_hint = p_type_hint; load_task.cache_mode = p_cache_mode; - load_task.use_sub_threads = p_thread_mode == LOAD_THREAD_DISTRIBUTE; - if (p_cache_mode == CACHE_MODE_REUSE) { + load_task.use_sub_threads = thread_mode == LOAD_THREAD_DISTRIBUTE; + if (p_cache_mode == ResourceFormatLoader::CACHE_MODE_REUSE) { Ref existing = ResourceCache::get_ref(local_path); if (existing.is_valid()) { //referencing is fine @@ -621,7 +630,7 @@ Ref ResourceLoader::_load_start(const String &p_path, // the token anymore so it's released. load_task_ptr->load_token->reference(); - if (p_thread_mode == LOAD_THREAD_FROM_CURRENT) { + if (thread_mode == LOAD_THREAD_FROM_CURRENT) { // The current thread may happen to be a thread from the pool. WorkerThreadPool::TaskID tid = WorkerThreadPool::get_singleton()->get_caller_task_id(); if (tid != WorkerThreadPool::INVALID_TASK_ID) { @@ -634,7 +643,15 @@ Ref ResourceLoader::_load_start(const String &p_path, } } // MutexLock(thread_load_mutex). - if (p_thread_mode == LOAD_THREAD_FROM_CURRENT) { + if (thread_mode == LOAD_THREAD_FROM_CURRENT) { + if (OS::get_singleton()->async_pck_is_supported() && OS::get_singleton()->async_pck_is_file_installable(p_path) && !OS::get_singleton()->async_pck_is_file_installed(p_path)) { + OS::get_singleton()->async_pck_install_file(p_path); + load_task_ptr->is_async_pck = true; + load_task_ptr->is_async_pck_installing = true; + load_task_ptr->status = THREAD_LOAD_IN_PROGRESS; + return load_token; + } + _run_load_task(load_task_ptr); } @@ -696,7 +713,20 @@ ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const status = load_task_ptr->status; if (r_progress) { - *r_progress = _dependency_get_progress(local_path); + if (load_task_ptr->is_async_pck) { +#ifdef THREADS_ENABLED + if (load_task_ptr->is_async_pck_installing) { + *r_progress = (float)load_task_ptr->progress_async_pck_install / 2.0f; + } else { + *r_progress = 0.5 + _dependency_get_progress(local_path); + } +#else + // There's no dependency progress possible, as the actual load must be done is sync. + *r_progress = load_task_ptr->progress_async_pck_install; +#endif // THREADS_ENABLED + } else { + *r_progress = _dependency_get_progress(local_path); + } } // Support userland polling in a loop on the main thread. @@ -722,6 +752,11 @@ Ref ResourceLoader::load_threaded_get(const String &p_path, Error *r_e *r_error = OK; } + if (OS::get_singleton()->async_pck_is_supported() && OS::get_singleton()->async_pck_is_file_installable(p_path) && !OS::get_singleton()->async_pck_is_file_installed(p_path)) { + *r_error = FAILED; + ERR_FAIL_V_MSG(Ref(), vformat(R"*("%s" is an AsyncPCK file that needs to be downloaded first before use. You need to call `ResourceLoader.load_threaded_request()` and wait for the file to load first.)*", p_path)); + } + Ref res; { MutexLock thread_load_lock(thread_load_mutex); @@ -737,6 +772,7 @@ Ref ResourceLoader::load_threaded_get(const String &p_path, Error *r_e LoadToken *load_token = user_load_tokens[p_path]; DEV_ASSERT(load_token->user_rc >= 1); +#ifdef THREADS_ENABLED // Support userland requesting on the main thread before the load is reported to be complete. if (Thread::is_main_thread() && !load_token->local_path.is_empty()) { ThreadLoadTask *load_task_ptr; @@ -765,6 +801,7 @@ Ref ResourceLoader::load_threaded_get(const String &p_path, Error *r_e } } } +#endif // THREADS_ENABLED res = _load_complete_inner(*load_token, r_error, thread_load_lock); @@ -946,6 +983,26 @@ Ref ResourceLoader::_load_complete_inner(LoadToken &p_load_token, Erro return resource; } +void ResourceLoader::poll_async_pck_install() { + for (KeyValue &KV : thread_load_tasks) { + if (!KV.value.is_async_pck || !KV.value.is_async_pck_installing) { + continue; + } + + Dictionary status = OS::get_singleton()->async_pck_install_file_get_status(KV.value.local_path); + KV.value.progress_async_pck_install = status["progress_ratio"]; + + if (status["status"] == "STATUS_ERROR") { + KV.value.is_async_pck_installing = false; + KV.value.resource = Ref(); + KV.value.status = THREAD_LOAD_FAILED; + } else if (status["status"] == "STATUS_INSTALLED") { + KV.value.is_async_pck_installing = false; + _run_load_task(&KV.value); + } + } +} + bool ResourceLoader::_ensure_load_progress() { // Some servers may need a new engine iteration to allow the load to progress. // Since the only known one is the rendering server (in single thread mode), let's keep it simple and just sync it. diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h index ec738adca99d..28d0ac464ce6 100644 --- a/core/io/resource_loader.h +++ b/core/io/resource_loader.h @@ -187,6 +187,7 @@ class ResourceLoader { String local_path; String type_hint; float progress = 0.0f; + float progress_async_pck_install = 0.0f; float max_reported_progress = 0.0f; uint64_t last_progress_check_main_thread_frame = UINT64_MAX; ThreadLoadStatus status = THREAD_LOAD_IN_PROGRESS; @@ -200,6 +201,8 @@ class ResourceLoader { bool need_wait : 1; bool in_progress_check : 1; // Measure against recursion cycles in progress reporting. Cycles are not expected, but can happen due to how it's currently implemented. bool use_sub_threads : 1; + bool is_async_pck : 1; + bool is_async_pck_installing : 1; struct ResourceChangedConnection { Resource *source = nullptr; @@ -212,7 +215,9 @@ class ResourceLoader { awaited(false), need_wait(true), in_progress_check(false), - use_sub_threads(false) {} + use_sub_threads(false), + is_async_pck(false), + is_async_pck_installing(false) {} }; static void _run_load_task(void *p_userdata); @@ -311,6 +316,8 @@ class ResourceLoader { static Vector list_directory(const String &p_directory); + static void poll_async_pck_install(); + static void initialize(); static void finalize(); }; diff --git a/core/object/object.h b/core/object/object.h index 0b88aa2f7e7a..140a50ae9824 100644 --- a/core/object/object.h +++ b/core/object/object.h @@ -66,6 +66,9 @@ // Helper macro to use with PROPERTY_HINT_ARRAY_TYPE for arrays of specific resources: // PropertyInfo(Variant::ARRAY, "fallbacks", PROPERTY_HINT_ARRAY_TYPE, MAKE_RESOURCE_TYPE_HINT("Font") #define MAKE_RESOURCE_TYPE_HINT(m_type) vformat("%s/%s:%s", Variant::OBJECT, PROPERTY_HINT_RESOURCE_TYPE, m_type) +// Helper macro to use with PROPERTY_HINT_ARRAY_TYPE for arrays of file paths: +// PropertyInfo(Variant::PACKED_STRING_ARRAY, "async/initial_load_forced_files", PROPERTY_HINT_ARRAY_TYPE, MAKE_FILE_ARRAY_TYPE_HINT("*")) +#define MAKE_FILE_ARRAY_TYPE_HINT(m_type) vformat("%s/%s:%s", Variant::STRING, PROPERTY_HINT_FILE, m_type) // API used to extend in GDExtension and other C compatible compiled languages. class MethodBind; diff --git a/core/os/os.cpp b/core/os/os.cpp index ffc0dff1c747..8795e5acc999 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -826,6 +826,39 @@ void OS::benchmark_dump() { #endif } +String OS::async_pck_get_async_pck_path(const String &p_path, Error *r_error) const { + Error err = OK; + String async_path; + String pck_path; + +#define RETURN_ERROR \ + if (r_error != nullptr) { \ + *r_error = err; \ + } \ + return pck_path + +#define _ERR_FAIL_COND(m_cond, m_err) \ + if (unlikely(m_cond)) { \ + err = m_err; \ + RETURN_ERROR; \ + } \ + (void)0 + + _ERR_FAIL_COND(p_path.is_empty(), ERR_INVALID_PARAMETER); + _ERR_FAIL_COND(FileAccess::exists(p_path), ERR_INVALID_PARAMETER); + + async_path = PackedData::get_singleton()->get_async_path(p_path); + _ERR_FAIL_COND(async_path.is_empty(), ERR_CANT_RESOLVE); + + pck_path = PackedData::get_singleton()->get_file_async_pack_path(async_path); + _ERR_FAIL_COND(pck_path.is_empty(), ERR_CANT_RESOLVE); + + RETURN_ERROR; + +#undef _ERR_FAIL_COND +#undef RETURN_ERROR +} + OS::OS() { singleton = this; diff --git a/core/os/os.h b/core/os/os.h index fd7b0c896a96..a3e8480f12f9 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -31,6 +31,7 @@ #pragma once #include "core/config/engine.h" +#include "core/io/file_access_pack.h" #include "core/io/logger.h" #include "core/io/remote_filesystem_client.h" #include "core/os/process_id.h" @@ -142,6 +143,8 @@ class OS { virtual bool _check_internal_feature_support(const String &p_feature) = 0; + virtual String async_pck_get_async_pck_path(const String &p_path, Error *r_error) const; + public: static OS *get_singleton(); @@ -394,6 +397,24 @@ class OS { virtual PreferredTextureFormat get_preferred_texture_format() const; + virtual bool async_pck_is_supported() const { return false; } + virtual bool async_pck_is_file_installable(const String &p_path) const { + return PackedData::get_singleton() && !PackedData::get_singleton()->is_disabled() && PackedData::get_singleton()->has_async_path(p_path); + } + virtual bool async_pck_is_file_installed(const String &p_path) const { return false; } + virtual Error async_pck_install_file(const String &p_path) const { + return FAILED; + } + virtual Dictionary async_pck_install_file_get_status(const String &p_path) const { + Dictionary status; + status["files"] = Dictionary(); + status["size"] = 0; + status["progress"] = 0; + status["progress_ratio"] = 0; + status["status"] = "STATUS_IDLE"; + return status; + } + // Load GDExtensions specific to this platform. // This is invoked by the GDExtensionManager after loading GDExtensions specified by the project. virtual void load_platform_gdextensions() const {} diff --git a/doc/classes/ResourceLoader.xml b/doc/classes/ResourceLoader.xml index c3f9de48bce7..23adc0bcda25 100644 --- a/doc/classes/ResourceLoader.xml +++ b/doc/classes/ResourceLoader.xml @@ -106,6 +106,7 @@ GDScript has a simplified [method @GDScript.load] built-in method which can be used in most situations, leaving the use of [ResourceLoader] for more advanced scenarios. [b]Note:[/b] If [member ProjectSettings.editor/export/convert_text_resources_to_binary] is [code]true[/code], [method @GDScript.load] will not be able to read converted files in an exported project. If you rely on run-time loading of files present within the PCK, set [member ProjectSettings.editor/export/convert_text_resources_to_binary] to [code]false[/code]. [b]Note:[/b] Relative paths will be prefixed with [code]"res://"[/code] before loading, to avoid unexpected results make sure your paths are absolute. + [b]Note:[/b] On Web, if [member EditorExportPlatformWeb.async/initial_install_mode] is set to "Only Install Required Resources", [method load_threaded_request] needs to be called first for non-initially installed resources. Otherwise, [method @GDScript.load] will fail and return an invalid resource. @@ -114,6 +115,7 @@ Returns the resource loaded by [method load_threaded_request]. If this is called before the loading thread is done (i.e. [method load_threaded_get_status] is not [constant THREAD_LOAD_LOADED]), the calling thread will be blocked until the resource has finished loading. However, it's recommended to use [method load_threaded_get_status] to known when the load has actually completed. + [b]Note:[/b] On Web, if [member EditorExportPlatformWeb.async/initial_install_mode] is set to "Only Install Required Resources" and [member EditorExportPlatformWeb.variant/thread_support] is [code]false[/code], [method load_threaded_get_status] will return an invalid resource if the file is not downloaded and installed. [method load_threaded_request] needs to be called first for non-initially installed resources. As usual, [method load_threaded_get_status] is recommended to use to know when the load has actually completed. @@ -124,6 +126,8 @@ Returns the status of a threaded loading operation started with [method load_threaded_request] for the resource at [param path]. An array variable can optionally be passed via [param progress], and will return a one-element array containing the ratio of completion of the threaded loading (between [code]0.0[/code] and [code]1.0[/code]). [b]Note:[/b] The recommended way of using this method is to call it during different frames (e.g., in [method Node._process], instead of a loop). + [b]Note:[/b] On Web, if [member EditorExportPlatformWeb.async/initial_install_mode] is set to "Only Install Required Resources" and [member EditorExportPlatformWeb.variant/thread_support] is [code]false[/code], [method load_threaded_get_status] returns the status of the download operation started with [method load_threaded_request] for the resource at [param path]. The array variable that can optionally be passed via [param progress] will return a one-element array containing the ratio of the download completion of resource, including all its dependencies (between [code]0.0[/code] and [code]1.0[/code]). + [b]Note:[/b] On Web, if [member EditorExportPlatformWeb.async/initial_install_mode] is set to "Only Install Required Resources" and [member EditorExportPlatformWeb.variant/thread_support] is [code]true[/code], the array variable that can optionally be passed via [param progress] will return a one-element array containing the ratio of the download progress of resource, including all its dependencies, and the completion of the threaded loading, both combined (between [code]0.0[/code] and [code]0.5[/code] for the download phase, and between [code]0.5[/code] and [code]1.0[/code] for the completion of the threaded loading). @@ -135,6 +139,7 @@ Loads the resource using threads. If [param use_sub_threads] is [code]true[/code], multiple threads will be used to load the resource, which makes loading faster, but may affect the main thread (and thus cause game slowdowns). The [param cache_mode] parameter defines whether and how the cache should be used or updated when loading the resource. + [b]Note:[/b] On Web, if [member EditorExportPlatformWeb.async/initial_install_mode] is set to "Only Install Required Resources", calling this method will start downloading and installing the resource, before loading it as usual. If [member EditorExportPlatformWeb.variant/thread_support] is [code]false[/code], the [param use_sub_threads] parameter is ignored as threads aren't used. diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index db99a028432e..37a7e27223ba 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -1401,7 +1401,7 @@ void EditorNode::_fs_changed() { } if (err != OK) { export_error = vformat("Project export for preset \"%s\" failed.", preset_name); - } else if (platform->get_worst_message_type() >= EditorExportPlatform::EXPORT_MESSAGE_WARNING) { + } else if (platform->get_worst_message_type() >= EditorExportPlatformData::EXPORT_MESSAGE_WARNING) { export_error = vformat("Project export for preset \"%s\" completed with warnings.", preset_name); } } diff --git a/editor/editor_node.h b/editor/editor_node.h index 337c7484a5f7..ef998b212659 100644 --- a/editor/editor_node.h +++ b/editor/editor_node.h @@ -1006,6 +1006,7 @@ class EditorNode : public Node { Dictionary drag_files_and_dirs(const Vector &p_paths, Control *p_from); EditorQuickOpenDialog *get_quick_open_dialog() { return quick_open_dialog; } + ProjectExportDialog *get_project_export_dialog() { return project_export; } void add_tool_menu_item(const String &p_name, const Callable &p_callback); void add_tool_submenu_item(const String &p_name, PopupMenu *p_submenu); diff --git a/editor/export/editor_export.cpp b/editor/export/editor_export.cpp index d4d02788fe20..2abcb9761992 100644 --- a/editor/export/editor_export.cpp +++ b/editor/export/editor_export.cpp @@ -109,10 +109,12 @@ void EditorExport::_save() { for (const KeyValue &E : preset->values) { PropertyInfo *prop = preset->properties.getptr(E.key); - if (prop && prop->usage & PROPERTY_USAGE_SECRET) { - credentials->set_value(option_section, E.key, E.value); - } else { - config->set_value(option_section, E.key, E.value); + if (prop && !(prop->usage & PROPERTY_USAGE_NO_INSTANCE_STATE)) { + if (prop->usage & PROPERTY_USAGE_SECRET) { + credentials->set_value(option_section, E.key, E.value); + } else { + config->set_value(option_section, E.key, E.value); + } } } } diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index ceaf61df46bc..3ccdb5746a1e 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -36,7 +36,6 @@ #include "core/extension/gdextension.h" #include "core/io/delta_encoding.h" #include "core/io/dir_access.h" -#include "core/io/file_access_encrypted.h" #include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION #include "core/io/image.h" #include "core/io/image_loader.h" @@ -49,11 +48,12 @@ #include "core/os/os.h" #include "core/os/shared_object.h" #include "core/string/translation.h" +#include "core/templates/bit_field.h" #include "core/version.h" #include "editor/editor_node.h" #include "editor/editor_string_names.h" #include "editor/export/editor_export.h" -#include "editor/export/editor_export_plugin.h" +#include "editor/export/editor_export_platform_utils.h" #include "editor/file_system/editor_file_system.h" #include "editor/file_system/editor_paths.h" #include "editor/script/script_editor_plugin.h" @@ -64,38 +64,6 @@ #include "scene/resources/packed_scene.h" #include "scene/resources/texture.h" -class EditorExportSaveProxy { - HashSet saved_paths; - EditorExportPlatform::EditorExportSaveFunction save_func; - bool tracking_saves = false; - -public: - bool has_saved(const String &p_path) const { return saved_paths.has(p_path); } - - Error save_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { - if (tracking_saves) { - saved_paths.insert(p_path.simplify_path().trim_prefix("res://")); - } - - return save_func(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta); - } - - EditorExportSaveProxy(EditorExportPlatform::EditorExportSaveFunction p_save_func, bool p_track_saves) : - save_func(p_save_func), tracking_saves(p_track_saves) {} -}; - -static int _get_pad(int p_alignment, int p_n) { - int rest = p_n % p_alignment; - int pad = 0; - if (rest > 0) { - pad = p_alignment - rest; - }; - - return pad; -} - -static constexpr int PCK_PADDING = 16; - Ref EditorExportPlatform::_load_icon_or_splash_image(const String &p_path, Error *r_error) const { Ref image; @@ -128,8 +96,8 @@ bool EditorExportPlatform::fill_log_messages(RichTextLabel *p_log, Error p_err) p_log->add_text(" "); p_log->add_text(get_name()); p_log->add_text(" - "); - if (p_err == OK && get_worst_message_type() < EditorExportPlatform::EXPORT_MESSAGE_ERROR) { - if (get_worst_message_type() >= EditorExportPlatform::EXPORT_MESSAGE_WARNING) { + if (p_err == OK && get_worst_message_type() < EditorExportPlatformData::EXPORT_MESSAGE_ERROR) { + if (get_worst_message_type() >= EditorExportPlatformData::EXPORT_MESSAGE_WARNING) { p_log->add_image(p_log->get_editor_theme_icon(SNAME("StatusWarning")), 16 * EDSCALE, 16 * EDSCALE, Color(1.0, 1.0, 1.0), INLINE_ALIGNMENT_CENTER); p_log->add_text(" "); p_log->add_text(TTR("Completed with warnings.")); @@ -160,14 +128,14 @@ bool EditorExportPlatform::fill_log_messages(RichTextLabel *p_log, Error p_err) Ref icon; switch (msg.msg_type) { - case EditorExportPlatform::EXPORT_MESSAGE_INFO: { + case EditorExportPlatformData::EXPORT_MESSAGE_INFO: { color = p_log->get_theme_color(SceneStringName(font_color), EditorStringName(Editor)) * Color(1, 1, 1, 0.6); } break; - case EditorExportPlatform::EXPORT_MESSAGE_WARNING: { + case EditorExportPlatformData::EXPORT_MESSAGE_WARNING: { icon = p_log->get_editor_theme_icon(SNAME("Warning")); color = p_log->get_theme_color(SNAME("warning_color"), EditorStringName(Editor)); } break; - case EditorExportPlatform::EXPORT_MESSAGE_ERROR: { + case EditorExportPlatformData::EXPORT_MESSAGE_ERROR: { icon = p_log->get_editor_theme_icon(SNAME("Error")); color = p_log->get_theme_color(SNAME("error_color"), EditorStringName(Editor)); } break; @@ -341,17 +309,17 @@ Error EditorExportPlatform::_load_patches(const Ref &p_prese Error err = _extract_android_assets(path, pck_path, temp_dir); if (err != OK) { _unload_patches(); - add_message(EXPORT_MESSAGE_ERROR, TTR("Patch Creation"), vformat(TTR("Could not extract assets from Android bundle \"%s\", due to error \"%s\"."), path, error_names[err])); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Patch Creation"), vformat(TTR("Could not extract assets from Android bundle \"%s\", due to error \"%s\"."), path, error_names[err])); return err; } patch_temp_dirs.push_back(temp_dir); } - Error err = PackedData::get_singleton()->add_pack(pck_path, true, 0, _get_script_encryption_key_bytes(p_preset)); + Error err = PackedData::get_singleton()->add_pack(pck_path, true, 0, EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(EditorExportPlatformUtils::get_script_encryption_key(p_preset))); if (err != OK) { _unload_patches(); - add_message(EXPORT_MESSAGE_ERROR, TTR("Patch Creation"), vformat(TTR("Could not load patch pack with path \"%s\"."), pck_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Patch Creation"), vformat(TTR("Could not load patch pack with path \"%s\"."), pck_path)); return err; } } @@ -374,64 +342,10 @@ void EditorExportPlatform::_unload_patches() { patch_temp_dirs.clear(); } -Error EditorExportPlatform::_encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt) { - r_encrypt = false; - for (int i = 0; i < p_enc_in_filters.size(); ++i) { - if (p_path.matchn(p_enc_in_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_in_filters[i])) { - r_encrypt = true; - break; - } - } - - for (int i = 0; i < p_enc_ex_filters.size(); ++i) { - if (p_path.matchn(p_enc_ex_filters[i]) || p_path.trim_prefix("res://").matchn(p_enc_ex_filters[i])) { - r_encrypt = false; - break; - } - } - - Ref fae; - Ref ftmp = p_fd; - if (r_encrypt) { - Vector iv; - if (p_seed != 0) { - uint64_t seed = p_seed; - - const uint8_t *ptr = p_data.ptr(); - int64_t len = p_data.size(); - for (int64_t i = 0; i < len; i++) { - seed = ((seed << 5) + seed) ^ ptr[i]; - } - - RandomPCG rng = RandomPCG(seed); - iv.resize(16); - for (int i = 0; i < 16; i++) { - iv.write[i] = rng.rand() % 256; - } - } - - fae.instantiate(); - ERR_FAIL_COND_V(fae.is_null(), ERR_FILE_CANT_OPEN); - - Error err = fae->open_and_parse(ftmp, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); - ERR_FAIL_COND_V(err != OK, ERR_FILE_CANT_OPEN); - ftmp = fae; - } - - // Store file content. - ftmp->store_buffer(p_data.ptr(), p_data.size()); - - if (fae.is_valid()) { - ftmp.unref(); - fae.unref(); - } - return OK; -} - Error EditorExportPlatform::_save_pack_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { ERR_FAIL_COND_V_MSG(p_total < 1, ERR_PARAMETER_RANGE_ERROR, "Must select at least one file to export."); - PackData *pd = (PackData *)p_userdata; + EditorExportPlatformData::PackData *pd = (EditorExportPlatformData::PackData *)p_userdata; const String simplified_path = simplify_path(p_path); @@ -442,12 +356,12 @@ Error EditorExportPlatform::_save_pack_file(const Ref &p_pre ftmp = pd->f; } - SavedData sd; + EditorExportPlatformData::SavedData sd; sd.path_utf8 = simplified_path.trim_prefix("res://").utf8(); sd.ofs = (pd->use_sparse_pck) ? 0 : pd->f->get_position(); sd.size = p_data.size(); sd.delta = p_delta; - Error err = _encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted); + Error err = EditorExportPlatformUtils::encrypt_and_store_data(ftmp, simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, sd.encrypted); if (err != OK) { return err; } @@ -456,7 +370,7 @@ Error EditorExportPlatform::_save_pack_file(const Ref &p_pre } if (!pd->use_sparse_pck) { - int pad = _get_pad(PCK_PADDING, pd->f->get_position()); + int pad = EditorExportPlatformUtils::get_pad(EditorExportPlatformData::PCK_PADDING, pd->f->get_position()); for (int i = 0; i < pad; i++) { pd->f->store_8(0); } @@ -483,7 +397,7 @@ Error EditorExportPlatform::_save_pack_file(const Ref &p_pre } Error EditorExportPlatform::_save_pack_patch_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { - Ref old_file = PackedData::get_singleton()->try_open_path(p_path, _get_script_encryption_key_bytes(p_preset)); + Ref old_file = PackedData::get_singleton()->try_open_path(p_path, EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(EditorExportPlatformUtils::get_script_encryption_key(p_preset))); if (old_file.is_null()) { return _save_pack_file(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, false); } @@ -547,7 +461,7 @@ Error EditorExportPlatform::_save_zip_file(const Ref &p_pres const String path = simplify_path(p_path).replace_first("res://", ""); - ZipData *zd = (ZipData *)p_userdata; + EditorExportPlatformData::ZipData *zd = (EditorExportPlatformData::ZipData *)p_userdata; zipFile zip = (zipFile)zd->zip; @@ -636,125 +550,6 @@ Ref EditorExportPlatform::create_preset() { return preset; } -void EditorExportPlatform::_export_find_resources(EditorFileSystemDirectory *p_dir, HashSet &p_paths) { - for (int i = 0; i < p_dir->get_subdir_count(); i++) { - _export_find_resources(p_dir->get_subdir(i), p_paths); - } - - for (int i = 0; i < p_dir->get_file_count(); i++) { - if (p_dir->get_file_type(i) == "TextFile") { - continue; - } - p_paths.insert(p_dir->get_file_path(i)); - } -} - -void EditorExportPlatform::_export_find_customized_resources(const Ref &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet &p_paths) { - for (int i = 0; i < p_dir->get_subdir_count(); i++) { - EditorFileSystemDirectory *subdir = p_dir->get_subdir(i); - _export_find_customized_resources(p_preset, subdir, p_preset->get_file_export_mode(subdir->get_path(), p_mode), p_paths); - } - - for (int i = 0; i < p_dir->get_file_count(); i++) { - if (p_dir->get_file_type(i) == "TextFile") { - continue; - } - String path = p_dir->get_file_path(i); - EditorExportPreset::FileExportMode file_mode = p_preset->get_file_export_mode(path, p_mode); - if (file_mode != EditorExportPreset::MODE_FILE_REMOVE) { - p_paths.insert(path); - } - } -} - -void EditorExportPlatform::_export_find_dependencies(const String &p_path, HashSet &p_paths) { - if (p_paths.has(p_path)) { - return; - } - - p_paths.insert(p_path); - - EditorFileSystemDirectory *dir; - int file_idx; - dir = EditorFileSystem::get_singleton()->find_file(p_path, &file_idx); - if (!dir) { - return; - } - - Vector deps = dir->get_file_deps(file_idx); - - for (int i = 0; i < deps.size(); i++) { - _export_find_dependencies(deps[i], p_paths); - } -} - -void EditorExportPlatform::_edit_files_with_filter(Ref &da, const Vector &p_filters, HashSet &r_list, bool exclude) { - da->list_dir_begin(); - String cur_dir = da->get_current_dir().replace_char('\\', '/'); - if (!cur_dir.ends_with("/")) { - cur_dir += "/"; - } - String cur_dir_no_prefix = cur_dir.replace("res://", ""); - - Vector dirs; - String f = da->get_next(); - while (!f.is_empty()) { - if (da->current_is_dir()) { - dirs.push_back(f); - } else { - String fullpath = cur_dir + f; - // Test also against path without res:// so that filters like `file.txt` can work. - String fullpath_no_prefix = cur_dir_no_prefix + f; - for (int i = 0; i < p_filters.size(); ++i) { - if (fullpath.matchn(p_filters[i]) || fullpath_no_prefix.matchn(p_filters[i])) { - if (!exclude) { - r_list.insert(fullpath); - } else { - r_list.erase(fullpath); - } - } - } - } - f = da->get_next(); - } - - da->list_dir_end(); - - for (int i = 0; i < dirs.size(); ++i) { - const String &dir = dirs[i]; - if (dir.begins_with(".")) { - continue; - } - - if (EditorFileSystem::_should_skip_directory(cur_dir + dir)) { - continue; - } - - da->change_dir(dir); - _edit_files_with_filter(da, p_filters, r_list, exclude); - da->change_dir(".."); - } -} - -void EditorExportPlatform::_edit_filter_list(HashSet &r_list, const String &p_filter, bool exclude) { - if (p_filter.is_empty()) { - return; - } - Vector split = p_filter.split(","); - Vector filters; - for (int i = 0; i < split.size(); i++) { - String f = split[i].strip_edges(); - if (f.is_empty()) { - continue; - } - filters.push_back(f); - } - - Ref da = DirAccess::create(DirAccess::ACCESS_RESOURCES); - ERR_FAIL_COND(da.is_null()); - _edit_files_with_filter(da, filters, r_list, exclude); -} - HashSet EditorExportPlatform::get_features(const Ref &p_preset, bool p_debug) const { Ref platform = p_preset->get_platform(); List feature_list; @@ -1129,133 +924,61 @@ String EditorExportPlatform::_export_customize(const String &p_path, LocalVector return save_path.is_empty() ? p_path : save_path; } -String EditorExportPlatform::_get_script_encryption_key(const Ref &p_preset) { - const String from_env = OS::get_singleton()->get_environment(ENV_SCRIPT_ENCRYPTION_KEY); - if (!from_env.is_empty()) { - return from_env.to_lower(); +Error EditorExportPlatform::_generate_sparse_pck_metadata(const Ref &p_preset, EditorExportPlatformData::PackData &p_pack_data, Vector &r_data, bool p_async) { + Error err; + Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export_index", "tmp", false, &err); + if (err != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not create temporary file!")); + return err; + } + int64_t pck_start_pos = ftmp->get_position(); + uint64_t file_base_ofs = 0; + uint64_t dir_base_ofs = 0; + + BitField pack_flags = PACK_REL_FILEBASE | PACK_SPARSE_BUNDLE; + if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { + pack_flags.set_flag(PACK_DIR_ENCRYPTED); } - return p_preset->get_script_encryption_key().to_lower(); -} + if (p_async) { + pack_flags.set_flag(PACK_ASYNC); + } + EditorExportPlatform::_store_header(ftmp, pack_flags, file_base_ofs, dir_base_ofs, p_pack_data.salt); + + // Write directory. + uint64_t dir_offset = ftmp->get_position(); + ftmp->seek(dir_base_ofs); + ftmp->store_64(dir_offset - pck_start_pos); + ftmp->seek(dir_offset); -Vector EditorExportPlatform::_get_script_encryption_key_bytes(const Ref &p_preset) { Vector key; - String script_key = _get_script_encryption_key(p_preset); - if (script_key.length() == 64) { - key.resize(32); - for (int i = 0; i < 32; i++) { - int v = 0; - if (i * 2 < script_key.length()) { - char32_t ct = script_key[i * 2]; - if (is_digit(ct)) { - ct = ct - '0'; - } else if (ct >= 'a' && ct <= 'f') { - ct = 10 + ct - 'a'; - } - v |= ct << 4; - } + if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { + key = EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(EditorExportPlatformUtils::get_script_encryption_key(p_preset)); + } - if (i * 2 + 1 < script_key.length()) { - char32_t ct = script_key[i * 2 + 1]; - if (is_digit(ct)) { - ct = ct - '0'; - } else if (ct >= 'a' && ct <= 'f') { - ct = 10 + ct - 'a'; - } - v |= ct; - } - key.write[i] = v; - } + if (!EditorExportPlatformUtils::encrypt_and_store_directory(ftmp, p_pack_data, key, p_preset->get_seed(), 0)) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); + return ERR_CANT_CREATE; } - return key; + r_data.resize(ftmp->get_length()); + ftmp->seek(0); + ftmp->get_buffer(r_data.ptrw(), r_data.size()); + ftmp.unref(); + + return OK; } Dictionary EditorExportPlatform::get_internal_export_files(const Ref &p_preset, bool p_debug) { - Dictionary files; - - // Text server support data. - if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { - bool include_data = (bool)get_project_setting(p_preset, "internationalization/locale/include_text_server_data"); - if (!include_data) { - Vector translations = get_project_setting(p_preset, "internationalization/locale/translations"); - for (const String &t : translations) { - Ref tr = ResourceLoader::load(t); - if (tr.is_valid() && TS->is_locale_using_support_data(tr->get_locale())) { - include_data = true; - break; - } - } - if (TS->is_locale_using_support_data(get_project_setting(p_preset, "internationalization/locale/fallback"))) { - include_data = true; - } - } - if (include_data) { - String ts_name = TS->get_support_data_filename(); - String ts_target = "res://" + ts_name; - if (!ts_name.is_empty()) { - bool export_ok = false; - if (FileAccess::exists(ts_target)) { // Include user supplied data file. - const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(ts_target); - if (!ts_data.is_empty()) { - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using user provided text server data, text display in the exported project might be broken if export template was built with different ICU version!")); - files[ts_target] = ts_data; - export_ok = true; - } - } else { - String current_version = GODOT_VERSION_FULL_CONFIG; - String template_path = EditorPaths::get_singleton()->get_export_templates_dir().path_join(current_version); - if (p_debug && p_preset->has("custom_template/debug") && p_preset->get("custom_template/debug") != "") { - template_path = p_preset->get("custom_template/debug").operator String().get_base_dir(); - } else if (!p_debug && p_preset->has("custom_template/release") && p_preset->get("custom_template/release") != "") { - template_path = p_preset->get("custom_template/release").operator String().get_base_dir(); - } - String data_file_name = template_path.path_join(ts_name); - if (FileAccess::exists(data_file_name)) { - const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(data_file_name); - if (!ts_data.is_empty()) { - print_line("Using text server data from export templates."); - files[ts_target] = ts_data; - export_ok = true; - } - } else { - const PackedByteArray &ts_data = TS->get_support_data(); - if (!ts_data.is_empty()) { - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using editor embedded text server data, text display in the exported project might be broken if export template was built with different ICU version!")); - files[ts_target] = ts_data; - export_ok = true; - } - } - } - if (!export_ok) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Export"), TTR("Missing text server data, text display in the exported project might be broken!")); - } - } - } + Dictionary internal_export_files; + HashMap files = EditorExportPlatformUtils::get_internal_export_files(this, p_preset, p_debug); + for (const KeyValue &key_value : files) { + internal_export_files[key_value.key] = key_value.value; } - - return files; + return internal_export_files; } Vector EditorExportPlatform::get_forced_export_files(const Ref &p_preset) { - Vector files; - - files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); - - String icon = ResourceUID::ensure_path(get_project_setting(p_preset, "application/config/icon")); - String splash = ResourceUID::ensure_path(get_project_setting(p_preset, "application/boot_splash/image")); - if (!icon.is_empty() && FileAccess::exists(icon)) { - files.push_back(icon); - } - if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { - files.push_back(splash); - } - - String extension_list_config_file = GDExtension::get_extension_list_config_file(); - if (FileAccess::exists(extension_list_config_file)) { - files.push_back(extension_list_config_file); - } - - return files; + return EditorExportPlatformUtils::get_forced_export_files(p_preset); } Error EditorExportPlatform::_script_save_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { @@ -1314,57 +1037,7 @@ Error EditorExportPlatform::export_project_files(const Ref & HashSet paths; Vector path_remaps; - if (p_preset->get_export_filter() == EditorExportPreset::EXPORT_ALL_RESOURCES) { - //find stuff - _export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), paths); - } else if (p_preset->get_export_filter() == EditorExportPreset::EXCLUDE_SELECTED_RESOURCES) { - _export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), paths); - Vector files = p_preset->get_files_to_export(); - for (int i = 0; i < files.size(); i++) { - paths.erase(files[i]); - } - } else if (p_preset->get_export_filter() == EditorExportPreset::EXPORT_CUSTOMIZED) { - _export_find_customized_resources(p_preset, EditorFileSystem::get_singleton()->get_filesystem(), p_preset->get_file_export_mode("res://"), paths); - } else { - bool scenes_only = p_preset->get_export_filter() == EditorExportPreset::EXPORT_SELECTED_SCENES; - - Vector files = p_preset->get_files_to_export(); - for (int i = 0; i < files.size(); i++) { - if (scenes_only && ResourceLoader::get_resource_type(files[i]) != "PackedScene") { - continue; - } - - _export_find_dependencies(files[i], paths); - } - - // Add autoload resources and their dependencies - List props; - ProjectSettings::get_singleton()->get_property_list(&props); - - for (const PropertyInfo &pi : props) { - if (!pi.name.begins_with("autoload/")) { - continue; - } - - String autoload_path = get_project_setting(p_preset, pi.name); - - if (autoload_path.begins_with("*")) { - autoload_path = autoload_path.substr(1); - } - - _export_find_dependencies(autoload_path, paths); - } - } - - //add native icons to non-resource include list - _edit_filter_list(paths, String("*.icns"), false); - _edit_filter_list(paths, String("*.ico"), false); - - _edit_filter_list(paths, p_preset->get_include_filter(), false); - _edit_filter_list(paths, p_preset->get_exclude_filter(), true); - - // Ignore import files, since these are automatically added to the jar later with the resources - _edit_filter_list(paths, String("*.import"), true); + EditorExportPlatformUtils::export_find_preset_resources(p_preset, paths); // Get encryption filters. bool enc_pck = p_preset->get_enc_pck(); @@ -1394,10 +1067,10 @@ Error EditorExportPlatform::export_project_files(const Ref & } // Get encryption key. - key = _get_script_encryption_key_bytes(p_preset); + key = EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(EditorExportPlatformUtils::get_script_encryption_key(p_preset)); } - EditorExportSaveProxy save_proxy(p_save_func, p_remove_func != nullptr); + EditorExportPlatformData::EditorExportSaveProxy save_proxy(p_save_func, p_remove_func != nullptr); Error err = OK; Vector> export_plugins = EditorExport::get_singleton()->get_export_plugins(); @@ -1786,11 +1459,11 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - const FilteredCache filtered_cache = _get_filtered_cache(paths); + const EditorExportPlatformData::FilteredCache filtered_cache = EditorExportPlatformUtils::get_filtered_cache(paths); - Vector forced_export = get_forced_export_files(p_preset); + Vector forced_export = EditorExportPlatformUtils::get_forced_export_files(p_preset); for (const String &file : forced_export) { - Vector array; + PackedByteArray array; if (file == GDExtension::get_extension_list_config_file()) { array = filtered_cache.extension_list; @@ -1852,73 +1525,8 @@ Error EditorExportPlatform::export_project_files(const Ref & return OK; } -// Used by the main export function to filter excluded global classes, extensions -// and UIDs based on excluded resources configured in the export preset. -EditorExportPlatform::FilteredCache EditorExportPlatform::_get_filtered_cache(const HashSet &p_paths) { - FilteredCache result; - - HashSet extension_list_lines; - Ref ext_file = FileAccess::open(GDExtension::get_extension_list_config_file(), FileAccess::READ); - if (ext_file.is_valid()) { - while (!ext_file->eof_reached()) { - String line = ext_file->get_line().strip_edges(); - extension_list_lines.insert(line); - } - } - - HashMap class_by_path; - Ref global_class_cf; - global_class_cf.instantiate(); - if (global_class_cf->load(ProjectSettings::get_singleton()->get_global_class_list_path()) == OK) { - Array original_list = global_class_cf->get_value("", "list", Array()); - class_by_path.reserve(original_list.size()); - for (const Variant &item : original_list) { - const Dictionary &class_dict = item; - ERR_CONTINUE(!class_dict.has("path")); - class_by_path[class_dict["path"]] = class_dict; - } - } - - Vector extension_lines; - Array global_class_list; - Vector> uid_entries; - extension_lines.reserve(extension_list_lines.size()); - global_class_list.reserve(class_by_path.size()); - uid_entries.reserve(p_paths.size()); - - for (const String &path : p_paths) { - if (extension_list_lines.has(path)) { - extension_lines.push_back(path); - } - if (class_by_path.has(path)) { - global_class_list.push_back(class_by_path[path]); - } - ResourceUID::ID uid = EditorFileSystem::get_singleton()->get_file_uid(path); - if (uid != ResourceUID::INVALID_ID) { - uid_entries.push_back(Pair(uid, path)); - } - } - - // Encode extensions. - for (const String &line : extension_lines) { - result.extension_list.append_array(line.to_utf8_buffer()); - result.extension_list.append('\n'); - } - - // Encode global classes. - if (!global_class_list.is_empty()) { - global_class_cf->set_value("", "list", global_class_list); - result.global_class_list = global_class_cf->encode_to_text().to_utf8_buffer(); - } - - // Encode UIDs. - result.uids = ResourceUID::encode_binary_cache(uid_entries); - - return result; -} - Error EditorExportPlatform::_pack_add_shared_object(const Ref &p_preset, void *p_userdata, const SharedObject &p_so) { - PackData *pack_data = (PackData *)p_userdata; + EditorExportPlatformData::PackData *pack_data = (EditorExportPlatformData::PackData *)p_userdata; if (pack_data->so_files) { pack_data->so_files->push_back(p_so); } @@ -1927,16 +1535,15 @@ Error EditorExportPlatform::_pack_add_shared_object(const Ref &p_preset, void *p_userdata, const String &p_path) { - PackData *pd = (PackData *)p_userdata; - - SavedData sd; + EditorExportPlatformData::PackData *pd = (EditorExportPlatformData::PackData *)p_userdata; + EditorExportPlatformData::SavedData sd; sd.path_utf8 = p_path.utf8(); sd.ofs = pd->f->get_position(); sd.size = 0; sd.removal = true; // This padding will likely never be added, as we should already be aligned when removals are added. - int pad = _get_pad(PCK_PADDING, pd->f->get_position()); + int pad = EditorExportPlatformUtils::get_pad(EditorExportPlatformData::PCK_PADDING, pd->f->get_position()); for (int i = 0; i < pad; i++) { pd->f->store_8(0); } @@ -1949,7 +1556,7 @@ Error EditorExportPlatform::_remove_pack_file(const Ref &p_p } Error EditorExportPlatform::_zip_add_shared_object(const Ref &p_preset, void *p_userdata, const SharedObject &p_so) { - ZipData *zip_data = (ZipData *)p_userdata; + EditorExportPlatformData::ZipData *zip_data = (EditorExportPlatformData::ZipData *)p_userdata; if (zip_data->so_files) { zip_data->so_files->push_back(p_so); } @@ -2053,7 +1660,7 @@ void EditorExportPlatform::zip_folder_recursive(zipFile &p_zip, const String &p_ Ref fa = FileAccess::open(dir.path_join(f), FileAccess::READ); if (fa.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("ZIP Creation"), vformat(TTR("Could not open file to read from path \"%s\"."), dir.path_join(f))); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("ZIP Creation"), vformat(TTR("Could not open file to read from path \"%s\"."), dir.path_join(f))); return; } const int bufsize = 16384; @@ -2164,20 +1771,16 @@ Dictionary EditorExportPlatform::_save_zip_patch(const Ref & return ret; } -bool EditorExportPlatform::_store_header(Ref p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs, const String &p_salt) { +bool EditorExportPlatform::_store_header(Ref p_fd, BitField p_pack_flags, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs, const String &p_salt) { p_fd->store_32(PACK_HEADER_MAGIC); p_fd->store_32(PACK_FORMAT_VERSION); p_fd->store_32(GODOT_VERSION_MAJOR); p_fd->store_32(GODOT_VERSION_MINOR); p_fd->store_32(GODOT_VERSION_PATCH); - uint32_t pack_flags = PACK_REL_FILEBASE; - if (p_enc) { - pack_flags |= PACK_DIR_ENCRYPTED; - } - if (p_sparse) { - pack_flags |= PACK_SPARSE_BUNDLE; - } + // Ensure that PACK_REL_FILEBASE is set. + p_pack_flags.set_flag(PACK_REL_FILEBASE); + uint32_t pack_flags = p_pack_flags; p_fd->store_32(pack_flags); // Flags. r_file_base_ofs = p_fd->get_position(); @@ -2186,7 +1789,7 @@ bool EditorExportPlatform::_store_header(Ref p_fd, bool p_enc, bool r_dir_base_ofs = p_fd->get_position(); p_fd->store_64(0); // Directory offset. - if (p_enc && p_sparse && p_salt.length() == 32) { + if (p_pack_flags.has_flag(PACK_DIR_ENCRYPTED) && p_pack_flags.has_flag(PACK_SPARSE_BUNDLE) && p_salt.length() == 32) { CharString cs = p_salt.latin1(); p_fd->store_buffer((const uint8_t *)cs.ptr(), 32); } else { @@ -2203,79 +1806,6 @@ bool EditorExportPlatform::_store_header(Ref p_fd, bool p_enc, bool return true; } -bool EditorExportPlatform::_encrypt_and_store_directory(Ref p_fd, PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base) { - Ref fae; - Ref fhead = p_fd; - - fhead->store_32(p_pack_data.file_ofs.size()); //amount of files - - if (!p_key.is_empty()) { - uint64_t seed = p_seed; - fae.instantiate(); - if (fae.is_null()) { - return false; - } - - Vector iv; - if (seed != 0) { - for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { - for (int64_t j = 0; j < p_pack_data.file_ofs[i].path_utf8.length(); j++) { - seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].path_utf8.get_data()[j]; - } - for (int64_t j = 0; j < p_pack_data.file_ofs[i].md5.size(); j++) { - seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].md5[j]; - } - seed = ((seed << 5) + seed) ^ (p_pack_data.file_ofs[i].ofs - p_file_base); - seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].size; - } - - RandomPCG rng = RandomPCG(seed); - iv.resize(16); - for (int i = 0; i < 16; i++) { - iv.write[i] = rng.rand() % 256; - } - } - - Error err = fae->open_and_parse(fhead, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); - if (err != OK) { - return false; - } - - fhead = fae; - } - for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { - uint32_t string_len = p_pack_data.file_ofs[i].path_utf8.length(); - uint32_t pad = _get_pad(4, string_len); - - fhead->store_32(string_len + pad); - fhead->store_buffer((const uint8_t *)p_pack_data.file_ofs[i].path_utf8.get_data(), string_len); - for (uint32_t j = 0; j < pad; j++) { - fhead->store_8(0); - } - - fhead->store_64(p_pack_data.file_ofs[i].ofs - p_file_base); - fhead->store_64(p_pack_data.file_ofs[i].size); // pay attention here, this is where file is - fhead->store_buffer(p_pack_data.file_ofs[i].md5.ptr(), 16); //also save md5 for file - uint32_t flags = 0; - if (p_pack_data.file_ofs[i].encrypted) { - flags |= PACK_FILE_ENCRYPTED; - } - if (p_pack_data.file_ofs[i].removal) { - flags |= PACK_FILE_REMOVAL; - } - if (p_pack_data.file_ofs[i].delta) { - flags |= PACK_FILE_DELTA; - } - fhead->store_32(flags); - } - - if (fae.is_valid()) { - fhead.unref(); - fae.unref(); - } - return true; -} - Error EditorExportPlatform::save_pack(const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files, EditorExportSaveFunction p_save_func, EditorExportRemoveFunction p_remove_func, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) { EditorProgress ep("savepack", TTR("Packing"), 102, true); @@ -2293,14 +1823,14 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b // Regular output to separate PCK file. f = FileAccess::open(p_path, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for writing at path \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for writing at path \"%s\"."), p_path)); return ERR_CANT_CREATE; } } else { // Append to executable. f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for reading-writing at path \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), vformat(TTR("Can't open file for reading-writing at path \"%s\"."), p_path)); return ERR_FILE_CANT_OPEN; } @@ -2322,10 +1852,14 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b uint64_t file_base_ofs = 0; uint64_t dir_base_ofs = 0; - _store_header(f, p_preset->get_enc_pck() && p_preset->get_enc_directory(), false, file_base_ofs, dir_base_ofs, String()); + BitField pack_flags = PACK_REL_FILEBASE; + if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { + pack_flags.set_flag(PACK_DIR_ENCRYPTED); + } + _store_header(f, pack_flags, file_base_ofs, dir_base_ofs); // Align for first file. - int file_padding = _get_pad(PCK_PADDING, f->get_position()); + int file_padding = EditorExportPlatformUtils::get_pad(EditorExportPlatformData::PCK_PADDING, f->get_position()); for (int i = 0; i < file_padding; i++) { f->store_8(0); } @@ -2336,7 +1870,7 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b f->seek(file_base); // Write files. - PackData pd; + EditorExportPlatformData::PackData pd; pd.ep = &ep; pd.f = f; pd.so_files = p_so_files; @@ -2345,18 +1879,18 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b Error err = export_project_files(p_preset, p_debug, p_save_func, p_remove_func, &pd, _pack_add_shared_object); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Failed to export project files.")); return err; } if (pd.file_ofs.is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); return FAILED; } pd.file_ofs.sort(); // Do sort, so we can do binary search later (where ?). - int dir_padding = _get_pad(PCK_PADDING, f->get_position()); + int dir_padding = EditorExportPlatformUtils::get_pad(EditorExportPlatformData::PCK_PADDING, f->get_position()); for (int i = 0; i < dir_padding; i++) { f->store_8(0); } @@ -2369,11 +1903,11 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b Vector key; if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { - key = _get_script_encryption_key_bytes(p_preset); + key = EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(EditorExportPlatformUtils::get_script_encryption_key(p_preset)); } - if (!_encrypt_and_store_directory(f, pd, key, p_preset->get_seed(), file_base)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); + if (!EditorExportPlatformUtils::encrypt_and_store_directory(f, pd, key, p_preset->get_seed(), file_base)) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); return ERR_CANT_CREATE; } @@ -2415,14 +1949,14 @@ Error EditorExportPlatform::save_zip(const Ref &p_preset, bo zlib_filefunc_def io = zipio_create_io(&io_fa); zipFile zip = zipOpen2(tmppath.utf8().get_data(), APPEND_STATUS_CREATE, nullptr, &io); - ZipData zd; + EditorExportPlatformData::ZipData zd; zd.ep = &ep; zd.zip = zip; zd.so_files = p_so_files; Error err = export_project_files(p_preset, p_debug, p_save_func, nullptr, &zd, _zip_add_shared_object); if (err != OK && err != ERR_SKIP) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save ZIP"), TTR("Failed to export project files.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save ZIP"), TTR("Failed to export project files.")); zipClose(zip, nullptr); return err; } @@ -2433,14 +1967,14 @@ Error EditorExportPlatform::save_zip(const Ref &p_preset, bo if (zd.file_count == 0) { da->remove(tmppath); - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("No files or changes to export.")); return FAILED; } err = da->rename(tmppath, p_path); if (err != OK) { da->remove(tmppath); - add_message(EXPORT_MESSAGE_ERROR, TTR("Save ZIP"), vformat(TTR("Failed to move temporary file \"%s\" to \"%s\"."), tmppath, p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save ZIP"), vformat(TTR("Failed to move temporary file \"%s\" to \"%s\"."), tmppath, p_path)); return err; } @@ -2490,11 +2024,11 @@ Vector EditorExportPlatform::gen_export_flags(BitFieldhas_setting("export/android/use_wifi_for_remote_debug") && EDITOR_GET("export/android/use_wifi_for_remote_debug")) { host = EDITOR_GET("export/android/wifi_remote_debug_host"); - } else if (p_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST)) { + } else if (p_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST)) { host = "localhost"; } - if (p_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT)) { + if (p_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT)) { int port = EDITOR_GET("filesystem/file_server/port"); String passwd = EDITOR_GET("filesystem/file_server/password"); ret.push_back("--remote-fs"); @@ -2505,7 +2039,7 @@ Vector EditorExportPlatform::gen_export_flags(BitField EditorExportPlatform::gen_export_flags(BitField &p_preset, const StringName &p_name) { - if (p_preset.is_valid()) { - return p_preset->get_project_setting(p_name); - } else { - return GLOBAL_GET(p_name); - } + return EditorExportPlatformUtils::get_project_setting(p_preset, p_name); } void EditorExportPlatform::_bind_methods() { @@ -2783,14 +2313,14 @@ void EditorExportPlatform::_bind_methods() { ClassDB::bind_static_method("EditorExportPlatform", D_METHOD("get_forced_export_files", "preset"), &EditorExportPlatform::get_forced_export_files, DEFVAL(Ref())); - BIND_ENUM_CONSTANT(EXPORT_MESSAGE_NONE); - BIND_ENUM_CONSTANT(EXPORT_MESSAGE_INFO); - BIND_ENUM_CONSTANT(EXPORT_MESSAGE_WARNING); - BIND_ENUM_CONSTANT(EXPORT_MESSAGE_ERROR); + BIND_ENUM_CONSTANT(EditorExportPlatformData::EXPORT_MESSAGE_NONE); + BIND_ENUM_CONSTANT(EditorExportPlatformData::EXPORT_MESSAGE_INFO); + BIND_ENUM_CONSTANT(EditorExportPlatformData::EXPORT_MESSAGE_WARNING); + BIND_ENUM_CONSTANT(EditorExportPlatformData::EXPORT_MESSAGE_ERROR); - BIND_BITFIELD_FLAG(DEBUG_FLAG_DUMB_CLIENT); - BIND_BITFIELD_FLAG(DEBUG_FLAG_REMOTE_DEBUG); - BIND_BITFIELD_FLAG(DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST); - BIND_BITFIELD_FLAG(DEBUG_FLAG_VIEW_COLLISIONS); - BIND_BITFIELD_FLAG(DEBUG_FLAG_VIEW_NAVIGATION); + BIND_BITFIELD_FLAG(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); + BIND_BITFIELD_FLAG(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG); + BIND_BITFIELD_FLAG(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST); + BIND_BITFIELD_FLAG(EditorExportPlatformData::DEBUG_FLAG_VIEW_COLLISIONS); + BIND_BITFIELD_FLAG(EditorExportPlatformData::DEBUG_FLAG_VIEW_NAVIGATION); } diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 4d21da01bbdd..1d6c04af35e1 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -30,7 +30,12 @@ #pragma once +#include "core/io/file_access_pack.h" +#include "core/io/zip_io.h" +#include "core/os/os.h" #include "core/os/process_id.h" +#include "editor/export/editor_export_platform_data.h" +#include "editor/export/editor_export_platform_utils.h" #include "editor/export/editor_export_preset.h" class DirAccess; @@ -51,83 +56,34 @@ const String ENV_SCRIPT_ENCRYPTION_KEY = "GODOT_SCRIPT_ENCRYPTION_KEY"; class EditorExportPlatform : public RefCounted { GDCLASS(EditorExportPlatform, RefCounted); + friend class EditorExportPlatformUtils; + protected: static void _bind_methods(); public: - typedef Error (*EditorExportSaveFunction)(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); - typedef Error (*EditorExportRemoveFunction)(const Ref &p_preset, void *p_userdata, const String &p_path); - typedef Error (*EditorExportSaveSharedObject)(const Ref &p_preset, void *p_userdata, const SharedObject &p_so); - - enum DebugFlags { - DEBUG_FLAG_DUMB_CLIENT = 1, - DEBUG_FLAG_REMOTE_DEBUG = 2, - DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST = 4, - DEBUG_FLAG_VIEW_COLLISIONS = 8, - DEBUG_FLAG_VIEW_NAVIGATION = 16, - }; - - enum ExportMessageType { - EXPORT_MESSAGE_NONE, - EXPORT_MESSAGE_INFO, - EXPORT_MESSAGE_WARNING, - EXPORT_MESSAGE_ERROR, - }; + typedef EditorExportPlatformData::EditorExportSaveFunction EditorExportSaveFunction; + typedef EditorExportPlatformData::EditorExportRemoveFunction EditorExportRemoveFunction; + typedef EditorExportPlatformData::EditorExportSaveSharedObject EditorExportSaveSharedObject; - struct ExportMessage { - ExportMessageType msg_type; - String category; - String text; - }; - - struct SavedData { - uint64_t ofs = 0; - uint64_t size = 0; - bool encrypted = false; - bool removal = false; - bool delta = false; - Vector md5; - CharString path_utf8; - - bool operator<(const SavedData &p_data) const { - return path_utf8 < p_data.path_utf8; - } - }; + using DebugFlags = EditorExportPlatformData::DebugFlags; + using ExportMessageType = EditorExportPlatformData::ExportMessageType; + using ExportMessage = EditorExportPlatformData::ExportMessage; - struct PackData { - String path; - String salt; - Ref f; - Vector file_ofs; - EditorProgress *ep = nullptr; - Vector *so_files = nullptr; - bool use_sparse_pck = false; - }; + friend bool EditorExportPlatformUtils::encrypt_and_store_directory(Ref p_fd, EditorExportPlatformData::PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base); + friend Error EditorExportPlatformUtils::encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt); - static bool _store_header(Ref p_fd, bool p_enc, bool p_sparse, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs, const String &p_salt); - static bool _encrypt_and_store_directory(Ref p_fd, PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base); - static Error _encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt); - static String _get_script_encryption_key(const Ref &p_preset); - static Vector _get_script_encryption_key_bytes(const Ref &p_preset); + static bool _store_header(Ref p_fd, BitField p_pack_flags, uint64_t &r_file_base_ofs, uint64_t &r_dir_base_ofs, const String &p_salt = ""); + Error _generate_sparse_pck_metadata(const Ref &p_preset, EditorExportPlatformData::PackData &p_pack_data, Vector &r_data, bool p_async = false); private: - struct ZipData { - void *zip = nullptr; - EditorProgress *ep = nullptr; - Vector *so_files = nullptr; - int file_count = 0; - }; - Vector messages; Vector patch_temp_dirs; - void _export_find_resources(EditorFileSystemDirectory *p_dir, HashSet &p_paths); - void _export_find_customized_resources(const Ref &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet &p_paths); - void _export_find_dependencies(const String &p_path, HashSet &p_paths); - static Error _save_pack_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); static Error _save_pack_patch_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); static Error _pack_add_shared_object(const Ref &p_preset, void *p_userdata, const SharedObject &p_so); + static bool _check_hash(const uint8_t *p_hash, const Vector &p_data); static Error _remove_pack_file(const Ref &p_preset, void *p_userdata, const String &p_path); @@ -143,17 +99,6 @@ class EditorExportPlatform : public RefCounted { static Error _script_save_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); static Error _script_add_shared_object(const Ref &p_preset, void *p_userdata, const SharedObject &p_so); - void _edit_files_with_filter(Ref &da, const Vector &p_filters, HashSet &r_list, bool exclude); - void _edit_filter_list(HashSet &r_list, const String &p_filter, bool exclude); - - struct FilteredCache { - Vector extension_list; - Vector global_class_list; - Vector uids; - }; - - static FilteredCache _get_filtered_cache(const HashSet &p_paths); - struct FileExportCache { uint64_t source_modified_time = 0; String source_md5; @@ -257,13 +202,13 @@ class EditorExportPlatform : public RefCounted { msg.msg_type = p_type; messages.push_back(msg); switch (p_type) { - case EXPORT_MESSAGE_INFO: { + case EditorExportPlatformData::EXPORT_MESSAGE_INFO: { print_line(vformat("%s: %s", msg.category, msg.text)); } break; - case EXPORT_MESSAGE_WARNING: { + case EditorExportPlatformData::EXPORT_MESSAGE_WARNING: { WARN_PRINT(vformat("%s: %s", msg.category, msg.text)); } break; - case EXPORT_MESSAGE_ERROR: { + case EditorExportPlatformData::EXPORT_MESSAGE_ERROR: { ERR_PRINT(vformat("%s: %s", msg.category, msg.text)); } break; default: @@ -281,7 +226,7 @@ class EditorExportPlatform : public RefCounted { } virtual ExportMessageType _get_message_type(int p_index) const { - ERR_FAIL_INDEX_V(p_index, messages.size(), EXPORT_MESSAGE_NONE); + ERR_FAIL_INDEX_V(p_index, messages.size(), EditorExportPlatformData::EXPORT_MESSAGE_NONE); return messages[p_index].msg_type; } @@ -296,7 +241,7 @@ class EditorExportPlatform : public RefCounted { } virtual ExportMessageType get_worst_message_type() const { - ExportMessageType worst_type = EXPORT_MESSAGE_NONE; + ExportMessageType worst_type = EditorExportPlatformData::EXPORT_MESSAGE_NONE; for (int i = 0; i < messages.size(); i++) { worst_type = MAX(worst_type, messages[i].msg_type); } diff --git a/editor/export/editor_export_platform_apple_embedded.cpp b/editor/export/editor_export_platform_apple_embedded.cpp index 0f701c74765f..b3c47781ffa1 100644 --- a/editor/export/editor_export_platform_apple_embedded.cpp +++ b/editor/export/editor_export_platform_apple_embedded.cpp @@ -39,6 +39,7 @@ #include "editor/editor_node.h" #include "editor/editor_string_names.h" #include "editor/export/editor_export.h" +#include "editor/export/editor_export_platform_data.h" #include "editor/import/resource_importer_texture_settings.h" #include "editor/themes/editor_scale.h" #include "main/main.h" @@ -1660,7 +1661,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref= Math::floor(total_files * 0.8))) { da->erase_contents_recursive(); } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Unexpected files found in the export destination directory \"%s.xcodeproj\", delete it manually or select another destination."), binary_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Unexpected files found in the export destination directory \"%s.xcodeproj\", delete it manually or select another destination."), binary_dir)); return ERR_CANT_CREATE; } } @@ -1753,7 +1754,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref= Math::floor(total_files * 0.8))) { da->erase_contents_recursive(); } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Unexpected files found in the export destination directory \"%s\", delete it manually or select another destination."), binary_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Unexpected files found in the export destination directory \"%s\", delete it manually or select another destination."), binary_dir)); return ERR_CANT_CREATE; } } @@ -1762,7 +1763,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Refdir_exists(binary_dir)) { Error err = da->make_dir(binary_dir); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Failed to create the directory: \"%s\""), binary_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Failed to create the directory: \"%s\""), binary_dir)); return err; } } @@ -1832,7 +1833,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref tmp_app_path = DirAccess::create_for_path(dest_dir); if (tmp_app_path.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), dest_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), dest_dir)); return ERR_CANT_CREATE; } @@ -1841,7 +1842,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Refmake_dir_recursive(dir_name); if (dir_err) { unzClose(src_pkg_zip); - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create a directory at path \"%s\"."), dir_name)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create a directory at path \"%s\"."), dir_name)); return ERR_CANT_CREATE; } } @@ -1944,7 +1945,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref f = FileAccess::open(file, FileAccess::WRITE); if (f.is_null()) { unzClose(src_pkg_zip); - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write to a file at path \"%s\"."), file)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write to a file at path \"%s\"."), file)); return ERR_CANT_CREATE; }; f->store_buffer(data.ptr(), data.size()); @@ -1965,7 +1966,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Refcopy(static_lib_path, dest_lib_file_path); if (lib_copy_err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not copy a file at path \"%s\" to \"%s\"."), static_lib_path, dest_lib_file_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not copy a file at path \"%s\" to \"%s\"."), static_lib_path, dest_lib_file_path)); return lib_copy_err; } } @@ -2047,7 +2048,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Refmake_dir_recursive(iconset_dir); } if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create a directory at path \"%s\"."), iconset_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create a directory at path \"%s\"."), iconset_dir)); return err; } @@ -2062,7 +2063,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref launch_screen_da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); if (launch_screen_da.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not access the filesystem.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not access the filesystem.")); return ERR_CANT_CREATE; } @@ -2070,7 +2071,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref f = FileAccess::open(project_file_name, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write to a file at path \"%s\"."), project_file_name)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write to a file at path \"%s\"."), project_file_name)); return ERR_CANT_CREATE; }; f->store_buffer(project_file_data.ptr(), project_file_data.size()); @@ -2101,7 +2102,7 @@ Error EditorExportPlatformAppleEmbedded::_export_project_helper(const Ref &p_pr String can_export_error; bool can_export_missing_templates; if (!can_export(p_preset, can_export_error, can_export_missing_templates)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), can_export_error); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), can_export_error); return ERR_UNCONFIGURED; } @@ -2704,11 +2705,11 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr String host = EDITOR_GET("network/debug/remote_host"); int remote_port = (int)EDITOR_GET("network/debug/remote_port"); - if (p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST)) { host = "localhost"; } - if (p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT)) { int port = EDITOR_GET("filesystem/file_server/port"); String passwd = EDITOR_GET("filesystem/file_server/password"); cmd_args_list.push_back("--remote-fs"); @@ -2719,7 +2720,7 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr } } - if (p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG)) { cmd_args_list.push_back("--remote-debug"); cmd_args_list.push_back(get_debug_protocol() + host + ":" + String::num_int64(remote_port)); @@ -2741,11 +2742,11 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr } } - if (p_debug_flags.has_flag(DEBUG_FLAG_VIEW_COLLISIONS)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_VIEW_COLLISIONS)) { cmd_args_list.push_back("--debug-collisions"); } - if (p_debug_flags.has_flag(DEBUG_FLAG_VIEW_NAVIGATION)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_VIEW_NAVIGATION)) { cmd_args_list.push_back("--debug-navigation"); } @@ -2779,12 +2780,12 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr int ec; err = OS::get_singleton()->execute(idepl, args, &log, &ec, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Run"), TTR("Could not start ios-deploy executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Run"), TTR("Could not start ios-deploy executable.")); CLEANUP_AND_RETURN(err); } if (ec != 0) { print_line("ios-deploy:\n" + log); - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Installation/running failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Installation/running failed, see editor log for details.")); CLEANUP_AND_RETURN(ERR_UNCONFIGURED); } } @@ -2808,7 +2809,7 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr }); if (ec != 0) { print_line("device install:\n" + log); - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Installation failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Installation failed, see editor log for details.")); CLEANUP_AND_RETURN(ERR_UNCONFIGURED); } } @@ -2835,7 +2836,7 @@ Error EditorExportPlatformAppleEmbedded::run(const Ref &p_pr }); if (ec != 0) { print_line("devicectl launch:\n" + log); - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Running failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Running failed, see editor log for details.")); } } } diff --git a/editor/export/editor_export_platform_data.cpp b/editor/export/editor_export_platform_data.cpp new file mode 100644 index 000000000000..6dc4ecc3a917 --- /dev/null +++ b/editor/export/editor_export_platform_data.cpp @@ -0,0 +1,39 @@ +/**************************************************************************/ +/* editor_export_platform_data.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "editor_export_platform_data.h" + +Error EditorExportPlatformData::EditorExportSaveProxy::save_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { + if (tracking_saves) { + saved_paths.insert(p_path.simplify_path().trim_prefix("res://")); + } + + return save_func(p_preset, p_userdata, p_path, p_data, p_file, p_total, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta); +} diff --git a/editor/export/editor_export_platform_data.h b/editor/export/editor_export_platform_data.h new file mode 100644 index 000000000000..763ef27022b3 --- /dev/null +++ b/editor/export/editor_export_platform_data.h @@ -0,0 +1,119 @@ +/**************************************************************************/ +/* editor_export_platform_data.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#pragma once + +#include "core/io/file_access.h" +#include "core/os/shared_object.h" +#include "core/variant/variant.h" +#include "editor/editor_node.h" + +class EditorFileSystemDirectory; + +class EditorExportPlatformData { +public: + static constexpr int PCK_PADDING = 16; + + enum ExportMessageType { + EXPORT_MESSAGE_NONE, + EXPORT_MESSAGE_INFO, + EXPORT_MESSAGE_WARNING, + EXPORT_MESSAGE_ERROR, + }; + + struct ExportMessage { + ExportMessageType msg_type; + String category; + String text; + }; + + struct SavedData { + uint64_t ofs = 0; + uint64_t size = 0; + bool encrypted = false; + bool removal = false; + bool delta = false; + Vector md5; + CharString path_utf8; + + bool operator<(const SavedData &p_data) const { + return path_utf8 < p_data.path_utf8; + } + }; + + struct PackData { + String path; + String salt; + Ref f; + Vector file_ofs; + EditorProgress *ep = nullptr; + Vector *so_files = nullptr; + bool use_sparse_pck = false; + }; + + enum DebugFlags { + DEBUG_FLAG_DUMB_CLIENT = 1, + DEBUG_FLAG_REMOTE_DEBUG = 2, + DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST = 4, + DEBUG_FLAG_VIEW_COLLISIONS = 8, + DEBUG_FLAG_VIEW_NAVIGATION = 16, + }; + + struct ZipData { + void *zip = nullptr; + EditorProgress *ep = nullptr; + Vector *so_files = nullptr; + int file_count = 0; + }; + + struct FilteredCache { + PackedByteArray extension_list; + PackedByteArray global_class_list; + PackedByteArray uids; + }; + + typedef Error (*EditorExportSaveFunction)(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); + typedef Error (*EditorExportRemoveFunction)(const Ref &p_preset, void *p_userdata, const String &p_path); + typedef Error (*EditorExportSaveSharedObject)(const Ref &p_preset, void *p_userdata, const SharedObject &p_so); + + class EditorExportSaveProxy { + HashSet saved_paths; + EditorExportSaveFunction save_func; + bool tracking_saves = false; + + public: + bool has_saved(const String &p_path) const { return saved_paths.has(p_path); } + + Error save_file(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta); + + EditorExportSaveProxy(EditorExportSaveFunction p_save_func, bool p_track_saves) : + save_func(p_save_func), tracking_saves(p_track_saves) {} + }; +}; diff --git a/editor/export/editor_export_platform_pc.cpp b/editor/export/editor_export_platform_pc.cpp index b9c7f716c1a8..ff34e3d855bd 100644 --- a/editor/export/editor_export_platform_pc.cpp +++ b/editor/export/editor_export_platform_pc.cpp @@ -154,7 +154,7 @@ Error EditorExportPlatformPC::export_project(const Ref &p_pr Error EditorExportPlatformPC::prepare_template(const Ref &p_preset, bool p_debug, const String &p_path, BitField p_flags) { if (!DirAccess::exists(p_path.get_base_dir())) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), TTR("The given export path doesn't exist.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), TTR("The given export path doesn't exist.")); return ERR_FILE_BAD_PATH; } @@ -170,7 +170,7 @@ Error EditorExportPlatformPC::prepare_template(const Ref &p_ } if (!template_path.is_empty() && !FileAccess::exists(template_path)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), vformat(TTR("Template file not found: \"%s\"."), template_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), vformat(TTR("Template file not found: \"%s\"."), template_path)); return ERR_FILE_NOT_FOUND; } @@ -198,7 +198,7 @@ Error EditorExportPlatformPC::prepare_template(const Ref &p_ } } if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), TTR("Failed to copy export template.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Template"), TTR("Failed to copy export template.")); return err; } @@ -220,7 +220,7 @@ Error EditorExportPlatformPC::export_project_data(const Ref Error err = save_pack(p_preset, p_debug, pck_path, &so_files, nullptr, nullptr, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); if (err == OK && p_preset->get("binary_format/embed_pck")) { if (embedded_size >= 0x100000000 && String(p_preset->get("binary_format/architecture")).contains("32")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); return ERR_INVALID_PARAMETER; } @@ -246,13 +246,13 @@ Error EditorExportPlatformPC::export_project_data(const Ref if (err == OK) { err = da->copy_dir(src_path, target_path, -1, true); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("GDExtension"), vformat(TTR("Failed to copy shared object \"%s\"."), src_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("GDExtension"), vformat(TTR("Failed to copy shared object \"%s\"."), src_path)); } } } else { err = da->copy(src_path, target_path); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("GDExtension"), vformat(TTR("Failed to copy shared object \"%s\"."), src_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("GDExtension"), vformat(TTR("Failed to copy shared object \"%s\"."), src_path)); } if (err == OK) { err = sign_shared_object(p_preset, p_debug, target_path); diff --git a/editor/export/editor_export_platform_utils.cpp b/editor/export/editor_export_platform_utils.cpp new file mode 100644 index 000000000000..63f2d3dc967b --- /dev/null +++ b/editor/export/editor_export_platform_utils.cpp @@ -0,0 +1,730 @@ +/**************************************************************************/ +/* editor_export_platform_utils.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "editor_export_platform_utils.h" + +#include "core/config/project_settings.h" +#include "core/crypto/crypto_core.h" +#include "core/extension/gdextension.h" +#include "core/io/file_access_encrypted.h" +#include "core/io/file_access_pack.h" +#include "core/math/random_pcg.h" +#include "core/version.h" +#include "editor/export/editor_export_platform.h" +#include "editor/file_system/editor_file_system.h" +#include "editor/file_system/editor_paths.h" + +/** + * EditorExportPlatformUtils::AsyncPckFileDependenciesState + */ +void EditorExportPlatformUtils::AsyncPckFileDependenciesState::add_to_file_dependencies(const String &p_file) { + String file = p_file.strip_edges().simplify_path(); + + if (file_dependencies.has(file)) { + return; + } + + List dependencies; + ResourceLoader::get_dependencies(file, &dependencies); + for (const String &dependency : dependencies) { + String dependency_path = EditorExportPlatformUtils::get_path_from_dependency(dependency); + if (!file_dependencies.has(file)) { + HashSet *dependency_list = &file_dependencies_lists.push_back({})->get(); + file_dependencies[file] = dependency_list; + } + file_dependencies[file]->insert(dependency_path); + add_to_file_dependencies(dependency_path); + } +} + +void EditorExportPlatformUtils::AsyncPckFileDependenciesState::add_to_file_dependencies(const HashSet &p_file_set) { + for (const String &file : p_file_set) { + if (file.ends_with("/")) { + continue; + } + add_to_file_dependencies(file); + } +} + +HashMap *> EditorExportPlatformUtils::AsyncPckFileDependenciesState::get_file_dependencies_of(const HashSet &p_file_set) { + HashMap *> dependencies; + for (const String &file : p_file_set) { + _get_file_dependencies_of(file, dependencies); + } + return dependencies; +} + +HashMap *> EditorExportPlatformUtils::AsyncPckFileDependenciesState::get_file_dependencies_of(const String &p_file) { + HashMap *> dependencies; + _get_file_dependencies_of(p_file, dependencies); + return dependencies; +} + +void EditorExportPlatformUtils::AsyncPckFileDependenciesState::_get_file_dependencies_of(const String &p_file, HashMap *> &p_dependencies) { + if (p_dependencies.has(p_file)) { + return; + } + if (!file_dependencies.has(p_file)) { + p_dependencies[p_file] = {}; + return; + } + ERR_FAIL_NULL(file_dependencies[p_file]); + + p_dependencies[p_file] = file_dependencies[p_file]; + for (const String &file_dependency : *file_dependencies[p_file]) { + _get_file_dependencies_of(file_dependency, p_dependencies); + } +} + +/** + * EditorExportPlatformUtils + */ +String EditorExportPlatformUtils::get_path_from_dependency(const String &p_dependency) { + String path = p_dependency; + if (path.contains("::")) { + return path.get_slice("::", 2); + } + if (path.begins_with("uid://")) { + return ResourceUID::get_singleton()->uid_to_path(path); + } + return path.simplify_path(); +} + +int EditorExportPlatformUtils::get_pad(int p_alignment, int p_n) { + int rest = p_n % p_alignment; + int pad = 0; + if (rest > 0) { + pad = p_alignment - rest; + } + + return pad; +} + +Variant EditorExportPlatformUtils::get_project_setting(const Ref &p_preset, const StringName &p_name) { + if (p_preset.is_valid()) { + return p_preset->get_project_setting(p_name); + } else { + return GLOBAL_GET(p_name); + } +} + +bool EditorExportPlatformUtils::encrypt_and_store_directory(Ref p_fd, EditorExportPlatformData::PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base) { + Ref fae; + Ref fhead = p_fd; + + // Amount of files. + fhead->store_32(p_pack_data.file_ofs.size()); + + if (!p_key.is_empty()) { + uint64_t seed = p_seed; + fae.instantiate(); + if (fae.is_null()) { + return false; + } + + Vector iv; + if (seed != 0) { + for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { + for (int64_t j = 0; j < p_pack_data.file_ofs[i].path_utf8.length(); j++) { + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].path_utf8.get_data()[j]; + } + for (int64_t j = 0; j < p_pack_data.file_ofs[i].md5.size(); j++) { + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].md5[j]; + } + seed = ((seed << 5) + seed) ^ (p_pack_data.file_ofs[i].ofs - p_file_base); + seed = ((seed << 5) + seed) ^ p_pack_data.file_ofs[i].size; + } + + RandomPCG rng = RandomPCG(seed); + iv.resize(16); + for (int i = 0; i < 16; i++) { + iv.write[i] = rng.rand() % 256; + } + } + + Error err = fae->open_and_parse(fhead, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); + if (err != OK) { + return false; + } + + fhead = fae; + } + for (int i = 0; i < p_pack_data.file_ofs.size(); i++) { + uint32_t string_len = p_pack_data.file_ofs[i].path_utf8.length(); + uint32_t pad = EditorExportPlatformUtils::get_pad(4, string_len); + + fhead->store_32(string_len + pad); + fhead->store_buffer((const uint8_t *)p_pack_data.file_ofs[i].path_utf8.get_data(), string_len); + for (uint32_t j = 0; j < pad; j++) { + fhead->store_8(0); + } + + fhead->store_64(p_pack_data.file_ofs[i].ofs - p_file_base); + fhead->store_64(p_pack_data.file_ofs[i].size); // Pay attention here, this is where file is. + fhead->store_buffer(p_pack_data.file_ofs[i].md5.ptr(), 16); // Also save md5 for file. + uint32_t flags = 0; + if (p_pack_data.file_ofs[i].encrypted) { + flags |= PACK_FILE_ENCRYPTED; + } + if (p_pack_data.file_ofs[i].removal) { + flags |= PACK_FILE_REMOVAL; + } + if (p_pack_data.file_ofs[i].delta) { + flags |= PACK_FILE_DELTA; + } + fhead->store_32(flags); + } + + if (fae.is_valid()) { + fhead.unref(); + fae.unref(); + } + return true; +} + +Error EditorExportPlatformUtils::encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt) { + r_encrypt = false; + for (const String &enc_in_filter : p_enc_in_filters) { + if (p_path.matchn(enc_in_filter) || p_path.trim_prefix("res://").matchn(enc_in_filter)) { + r_encrypt = true; + break; + } + } + + for (const String &enc_ex_filter : p_enc_ex_filters) { + if (p_path.matchn(enc_ex_filter) || p_path.trim_prefix("res://").matchn(enc_ex_filter)) { + r_encrypt = false; + break; + } + } + + Ref fae; + Ref ftmp = p_fd; + if (r_encrypt) { + Vector iv; + if (p_seed != 0) { + uint64_t seed = p_seed; + + const uint8_t *ptr = p_data.ptr(); + int64_t len = p_data.size(); + for (int64_t i = 0; i < len; i++) { + seed = ((seed << 5) + seed) ^ ptr[i]; + } + + RandomPCG rng = RandomPCG(seed); + iv.resize(16); + for (int i = 0; i < 16; i++) { + iv.write[i] = rng.rand() % 256; + } + } + + fae.instantiate(); + ERR_FAIL_COND_V(fae.is_null(), ERR_FILE_CANT_OPEN); + + Error err = fae->open_and_parse(ftmp, p_key, FileAccessEncrypted::MODE_WRITE_AES256, false, iv); + ERR_FAIL_COND_V(err != OK, ERR_FILE_CANT_OPEN); + ftmp = fae; + } + + // Store file content. + ftmp->store_buffer(p_data.ptr(), p_data.size()); + + if (fae.is_valid()) { + ftmp.unref(); + fae.unref(); + } + return OK; +} + +Error EditorExportPlatformUtils::store_temp_file(const String &p_simplified_path, const PackedByteArray &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const PackedByteArray &p_key, uint64_t p_seed, bool p_delta, PackedByteArray &r_enc_data, EditorExportPlatformData::SavedData &r_sd) { + Error err = OK; + Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err); + if (err != OK) { + return err; + } + r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8(); + r_sd.ofs = 0; + r_sd.size = p_data.size(); + r_sd.delta = p_delta; + err = EditorExportPlatformUtils::encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted); + if (err != OK) { + return err; + } + + r_enc_data.resize(ftmp->get_length()); + ftmp->seek(0); + ftmp->get_buffer(r_enc_data.ptrw(), r_enc_data.size()); + ftmp.unref(); + + // Store MD5 of original file. + { + unsigned char hash[16]; + CryptoCore::md5(p_data.ptr(), p_data.size(), hash); + r_sd.md5.resize(16); + for (int i = 0; i < 16; i++) { + r_sd.md5.write[i] = hash[i]; + } + } + return OK; +} + +// Utility method used to create a directory. +Error EditorExportPlatformUtils::create_directory(const String &p_dir) { + if (DirAccess::exists(p_dir)) { + return OK; + } + Ref filesystem_da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + ERR_FAIL_COND_V_MSG(filesystem_da.is_null(), ERR_CANT_CREATE, "Cannot create directory '" + p_dir + "'."); + Error err = filesystem_da->make_dir_recursive(p_dir); + ERR_FAIL_COND_V_MSG(err, ERR_CANT_CREATE, "Cannot create directory '" + p_dir + "'."); + return OK; +} + +// Writes p_data into a file at p_path, creating directories if necessary. +// Note: this will overwrite the file at p_path if it already exists. +Error EditorExportPlatformUtils::store_file_at_path(const String &p_path, const PackedByteArray &p_data) { + String dir = p_path.get_base_dir(); + Error err = EditorExportPlatformUtils::create_directory(dir); + if (err != OK) { + return err; + } + Ref fa = FileAccess::open(p_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(fa.is_null(), ERR_CANT_CREATE, "Cannot create file '" + p_path + "'."); + fa->store_buffer(p_data.ptr(), p_data.size()); + return OK; +} + +// Writes string p_data into a file at p_path, creating directories if necessary. +// Note: this will overwrite the file at p_path if it already exists. +Error EditorExportPlatformUtils::store_string_at_path(const String &p_path, const String &p_data) { + String dir = p_path.get_base_dir(); + Error err = EditorExportPlatformUtils::create_directory(dir); + if (err != OK) { + if (OS::get_singleton()->is_stdout_verbose()) { + print_error("Unable to write data into " + p_path); + } + return err; + } + Ref fa = FileAccess::open(p_path, FileAccess::WRITE); + ERR_FAIL_COND_V_MSG(fa.is_null(), ERR_CANT_CREATE, "Cannot create file '" + p_path + "'."); + fa->store_string(p_data); + return OK; +} + +String EditorExportPlatformUtils::get_script_encryption_key(const Ref &p_preset) { + const String from_env = OS::get_singleton()->get_environment(ENV_SCRIPT_ENCRYPTION_KEY); + if (!from_env.is_empty()) { + return from_env.to_lower(); + } + return p_preset->get_script_encryption_key().to_lower(); +} + +PackedByteArray EditorExportPlatformUtils::convert_script_encryption_key_to_bytes(const String &p_encryption_key) { + PackedByteArray key; + key.resize_initialized(32); + ERR_FAIL_COND_V(p_encryption_key.length() != 64, key); + + for (int i = 0; i < 32; i++) { + int v = 0; + if (i * 2 < p_encryption_key.length()) { + char32_t ct = p_encryption_key[i * 2]; + if (is_digit(ct)) { + ct = ct - '0'; + } else if (ct >= 'a' && ct <= 'f') { + ct = 10 + ct - 'a'; + } + v |= ct << 4; + } + + if (i * 2 + 1 < p_encryption_key.length()) { + char32_t ct = p_encryption_key[i * 2 + 1]; + if (is_digit(ct)) { + ct = ct - '0'; + } else if (ct >= 'a' && ct <= 'f') { + ct = 10 + ct - 'a'; + } + v |= ct; + } + key.write[i] = v; + } + + return key; +} + +void EditorExportPlatformUtils::export_find_resources(EditorFileSystemDirectory *p_dir, HashSet &p_paths) { + for (int i = 0; i < p_dir->get_subdir_count(); i++) { + EditorExportPlatformUtils::export_find_resources(p_dir->get_subdir(i), p_paths); + } + + for (int i = 0; i < p_dir->get_file_count(); i++) { + if (p_dir->get_file_type(i) == "TextFile") { + continue; + } + p_paths.insert(p_dir->get_file_path(i)); + } +} + +void EditorExportPlatformUtils::export_find_customized_resources(const Ref &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet &p_paths) { + for (int i = 0; i < p_dir->get_subdir_count(); i++) { + EditorFileSystemDirectory *subdir = p_dir->get_subdir(i); + EditorExportPlatformUtils::export_find_customized_resources(p_preset, subdir, p_preset->get_file_export_mode(subdir->get_path(), p_mode), p_paths); + } + + for (int i = 0; i < p_dir->get_file_count(); i++) { + if (p_dir->get_file_type(i) == "TextFile") { + continue; + } + String path = p_dir->get_file_path(i); + EditorExportPreset::FileExportMode file_mode = p_preset->get_file_export_mode(path, p_mode); + if (file_mode != EditorExportPreset::MODE_FILE_REMOVE) { + p_paths.insert(path); + } + } +} + +void EditorExportPlatformUtils::export_find_dependencies(const String &p_path, HashSet &p_paths) { + if (p_paths.has(p_path)) { + return; + } + + p_paths.insert(p_path); + + EditorFileSystemDirectory *dir; + int file_idx; + dir = EditorFileSystem::get_singleton()->find_file(p_path, &file_idx); + if (!dir) { + return; + } + + PackedStringArray deps = dir->get_file_deps(file_idx); + + for (const String &dep : deps) { + EditorExportPlatformUtils::export_find_dependencies(dep, p_paths); + } +} + +void EditorExportPlatformUtils::export_find_preset_resources(const Ref &p_preset, HashSet &p_paths) { + EditorExportPreset::ExportFilter export_filter = p_preset->get_export_filter(); + switch (export_filter) { + case EditorExportPreset::EXPORT_ALL_RESOURCES: + case EditorExportPreset::EXCLUDE_SELECTED_RESOURCES: { + EditorExportPlatformUtils::export_find_resources(EditorFileSystem::get_singleton()->get_filesystem(), p_paths); + + if (export_filter == EditorExportPreset::EXCLUDE_SELECTED_RESOURCES) { + PackedStringArray excluded_resources = p_preset->get_files_to_export(); + for (const String &excluded_resource : excluded_resources) { + p_paths.erase(excluded_resource); + } + } + } break; + + case EditorExportPreset::EXPORT_CUSTOMIZED: { + EditorExportPlatformUtils::export_find_customized_resources(p_preset, EditorFileSystem::get_singleton()->get_filesystem(), p_preset->get_file_export_mode("res://"), p_paths); + } break; + + case EditorExportPreset::EXPORT_SELECTED_SCENES: + case EditorExportPreset::EXPORT_SELECTED_RESOURCES: { + bool scenes_only = export_filter == EditorExportPreset::EXPORT_SELECTED_SCENES; + + PackedStringArray files = p_preset->get_files_to_export(); + for (const String &file : files) { + if (scenes_only && ResourceLoader::get_resource_type(file) != "PackedScene") { + continue; + } + + EditorExportPlatformUtils::export_find_dependencies(file, p_paths); + } + + // Add autoload resources and their dependencies + List props; + ProjectSettings::get_singleton()->get_property_list(&props); + + for (const PropertyInfo &pi : props) { + if (!pi.name.begins_with("autoload/")) { + continue; + } + + String autoload_path = EditorExportPlatformUtils::get_project_setting(p_preset, pi.name); + + if (autoload_path.begins_with("*")) { + autoload_path = autoload_path.substr(1); + } + + EditorExportPlatformUtils::export_find_dependencies(autoload_path, p_paths); + } + } break; + } + + //add native icons to non-resource include list + EditorExportPlatformUtils::edit_filter_list(p_paths, String("*.icns"), false); + EditorExportPlatformUtils::edit_filter_list(p_paths, String("*.ico"), false); + + EditorExportPlatformUtils::edit_filter_list(p_paths, p_preset->get_include_filter(), false); + EditorExportPlatformUtils::edit_filter_list(p_paths, p_preset->get_exclude_filter(), true); + + // Ignore import files, since these are automatically added to the jar later with the resources + EditorExportPlatformUtils::edit_filter_list(p_paths, String("*.import"), true); +} + +void EditorExportPlatformUtils::edit_files_with_filter(Ref &da, const Vector &p_filters, HashSet &r_list, bool exclude) { + da->list_dir_begin(); + String cur_dir = da->get_current_dir().replace_char('\\', '/'); + if (!cur_dir.ends_with("/")) { + cur_dir += "/"; + } + String cur_dir_no_prefix = cur_dir.replace("res://", ""); + + Vector dirs; + String f = da->get_next(); + while (!f.is_empty()) { + if (da->current_is_dir()) { + dirs.push_back(f); + } else { + String fullpath = cur_dir + f; + // Test also against path without res:// so that filters like `file.txt` can work. + String fullpath_no_prefix = cur_dir_no_prefix + f; + for (const String &filter : p_filters) { + if (fullpath.matchn(filter) || fullpath_no_prefix.matchn(filter)) { + if (!exclude) { + r_list.insert(fullpath); + } else { + r_list.erase(fullpath); + } + } + } + } + f = da->get_next(); + } + + da->list_dir_end(); + + for (const String &dir : dirs) { + if (dir.begins_with(".")) { + continue; + } + + if (EditorFileSystem::_should_skip_directory(cur_dir + dir)) { + continue; + } + + da->change_dir(dir); + EditorExportPlatformUtils::edit_files_with_filter(da, p_filters, r_list, exclude); + da->change_dir(".."); + } +} + +void EditorExportPlatformUtils::edit_filter_list(HashSet &r_list, const String &p_filter, bool exclude) { + if (p_filter.is_empty()) { + return; + } + Vector split = p_filter.split(","); + Vector filters; + for (int i = 0; i < split.size(); i++) { + String f = split[i].strip_edges(); + if (f.is_empty()) { + continue; + } + filters.push_back(f); + } + + Ref da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + ERR_FAIL_COND(da.is_null()); + EditorExportPlatformUtils::edit_files_with_filter(da, filters, r_list, exclude); +} + +Vector EditorExportPlatformUtils::filter_extension_list_config_file(const String &p_config_path, const HashSet &p_paths) { + Ref f = FileAccess::open(p_config_path, FileAccess::READ); + if (f.is_null()) { + ERR_FAIL_V_MSG(Vector(), "Can't open file from path '" + String(p_config_path) + "'."); + } + Vector data; + while (!f->eof_reached()) { + String l = f->get_line().strip_edges(); + if (p_paths.has(l)) { + data.append_array(l.to_utf8_buffer()); + data.append('\n'); + } + } + return data; +} + +Vector EditorExportPlatformUtils::get_forced_export_files(const Ref &p_preset) { + Vector files; + + files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); + + String icon = ResourceUID::ensure_path(get_project_setting(p_preset, "application/config/icon")); + String splash = ResourceUID::ensure_path(get_project_setting(p_preset, "application/boot_splash/image")); + if (!icon.is_empty() && FileAccess::exists(icon)) { + files.push_back(icon); + } + if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { + files.push_back(splash); + } + String resource_cache_file = ResourceUID::get_cache_file(); + if (FileAccess::exists(resource_cache_file)) { + files.push_back(resource_cache_file); + } + + String extension_list_config_file = GDExtension::get_extension_list_config_file(); + if (FileAccess::exists(extension_list_config_file)) { + files.push_back(extension_list_config_file); + } + + return files; +} + +HashMap EditorExportPlatformUtils::get_internal_export_files(const Ref &p_editor_export_platform, const Ref &p_preset, bool p_debug) { + HashMap files; + + // Text server support data. + if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { + bool include_data = (bool)get_project_setting(p_preset, "internationalization/locale/include_text_server_data"); + if (!include_data) { + Vector translations = get_project_setting(p_preset, "internationalization/locale/translations"); + translations.push_back(get_project_setting(p_preset, "internationalization/locale/fallback")); + for (const String &t : translations) { + if (TS->is_locale_using_support_data(t)) { + include_data = true; + break; + } + } + } + if (include_data) { + String ts_name = TS->get_support_data_filename(); + String ts_target = "res://" + ts_name; + if (!ts_name.is_empty()) { + bool export_ok = false; + if (FileAccess::exists(ts_target)) { // Include user supplied data file. + const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(ts_target); + if (!ts_data.is_empty()) { + p_editor_export_platform->add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using user provided text server data, text display in the exported project might be broken if export template was built with different ICU version!")); + files[ts_target] = ts_data; + export_ok = true; + } + } else { + String current_version = GODOT_VERSION_FULL_CONFIG; + String template_path = EditorPaths::get_singleton()->get_export_templates_dir().path_join(current_version); + if (p_debug && p_preset->has("custom_template/debug") && p_preset->get("custom_template/debug") != "") { + template_path = p_preset->get("custom_template/debug").operator String().get_base_dir(); + } else if (!p_debug && p_preset->has("custom_template/release") && p_preset->get("custom_template/release") != "") { + template_path = p_preset->get("custom_template/release").operator String().get_base_dir(); + } + String data_file_name = template_path.path_join(ts_name); + if (FileAccess::exists(data_file_name)) { + const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(data_file_name); + if (!ts_data.is_empty()) { + print_line("Using text server data from export templates."); + files[ts_target] = ts_data; + export_ok = true; + } + } else { + const PackedByteArray &ts_data = TS->get_support_data(); + if (!ts_data.is_empty()) { + p_editor_export_platform->add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using editor embedded text server data, text display in the exported project might be broken if export template was built with different ICU version!")); + files[ts_target] = ts_data; + export_ok = true; + } + } + } + if (!export_ok) { + p_editor_export_platform->add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Export"), TTR("Missing text server data, text display in the exported project might be broken!")); + } + } + } + } + + return files; +} + +// Used by the main export function to filter excluded global classes, extensions +// and UIDs based on excluded resources configured in the export preset. +EditorExportPlatformData::FilteredCache EditorExportPlatformUtils::get_filtered_cache(const HashSet &p_paths) { + EditorExportPlatformData::FilteredCache result; + + HashSet extension_list_lines; + Ref ext_file = FileAccess::open(GDExtension::get_extension_list_config_file(), FileAccess::READ); + if (ext_file.is_valid()) { + while (!ext_file->eof_reached()) { + String line = ext_file->get_line().strip_edges(); + extension_list_lines.insert(line); + } + } + + HashMap class_by_path; + Ref global_class_cf; + global_class_cf.instantiate(); + if (global_class_cf->load(ProjectSettings::get_singleton()->get_global_class_list_path()) == OK) { + Array original_list = global_class_cf->get_value("", "list", Array()); + class_by_path.reserve(original_list.size()); + for (const Variant &item : original_list) { + const Dictionary &class_dict = item; + ERR_CONTINUE(!class_dict.has("path")); + class_by_path[class_dict["path"]] = class_dict; + } + } + + Vector extension_lines; + Array global_class_list; + Vector> uid_entries; + extension_lines.reserve(extension_list_lines.size()); + global_class_list.reserve(class_by_path.size()); + uid_entries.reserve(p_paths.size()); + + for (const String &path : p_paths) { + if (extension_list_lines.has(path)) { + extension_lines.push_back(path); + } + if (class_by_path.has(path)) { + global_class_list.push_back(class_by_path[path]); + } + ResourceUID::ID uid = EditorFileSystem::get_singleton()->get_file_uid(path); + if (uid != ResourceUID::INVALID_ID) { + uid_entries.push_back(Pair(uid, path)); + } + } + + // Encode extensions. + for (const String &line : extension_lines) { + result.extension_list.append_array(line.to_utf8_buffer()); + result.extension_list.append('\n'); + } + + // Encode global classes. + if (!global_class_list.is_empty()) { + global_class_cf->set_value("", "list", global_class_list); + result.global_class_list = global_class_cf->encode_to_text().to_utf8_buffer(); + } + + // Encode UIDs. + result.uids = ResourceUID::encode_binary_cache(uid_entries); + + return result; +} diff --git a/editor/export/editor_export_platform_utils.h b/editor/export/editor_export_platform_utils.h new file mode 100644 index 000000000000..cfa27384175d --- /dev/null +++ b/editor/export/editor_export_platform_utils.h @@ -0,0 +1,84 @@ +/**************************************************************************/ +/* editor_export_platform_utils.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#pragma once + +#include "core/io/dir_access.h" +#include "editor/export/editor_export_platform_data.h" +#include "editor/export/editor_export_preset.h" + +class EditorExportPlatformUtils { +public: + struct AsyncPckFileDependenciesState { + private: + void _get_file_dependencies_of(const String &p_file, HashMap *> &p_dependencies); + + public: + List> file_dependencies_lists; + HashMap *> file_dependencies; + + void add_to_file_dependencies(const String &p_file); + void add_to_file_dependencies(const HashSet &p_file_set); + HashMap *> get_file_dependencies_of(const HashSet &p_file_set); + HashMap *> get_file_dependencies_of(const String &p_file); + + void clear() { file_dependencies.clear(); } + }; + + static String get_path_from_dependency(const String &p_dependency); + static int get_pad(int p_alignment, int p_n); + static Variant get_project_setting(const Ref &p_preset, const StringName &p_name); + static bool encrypt_and_store_directory(Ref p_fd, EditorExportPlatformData::PackData &p_pack_data, const Vector &p_key, uint64_t p_seed, uint64_t p_file_base); + static Error encrypt_and_store_data(Ref p_fd, const String &p_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool &r_encrypt); + static Error store_temp_file(const String &p_simplified_path, const PackedByteArray &p_data, const PackedStringArray &p_encoded_included_filters, const PackedStringArray &p_encoded_excluded_filters, const Vector &p_key, uint64_t p_seed, bool p_delta, PackedByteArray &r_encoded_data, EditorExportPlatformData::SavedData &r_saved_data); + // Utility method used to create a directory. + static Error create_directory(const String &p_dir); + // Writes p_data into a file at p_path, creating directories if necessary. + // Note: this will overwrite the file at p_path if it already exists. + static Error store_file_at_path(const String &p_path, const PackedByteArray &p_data); + // Writes string p_data into a file at p_path, creating directories if necessary. + // Note: this will overwrite the file at p_path if it already exists. + static Error store_string_at_path(const String &p_path, const String &p_data); + // Returns the script encryption key. + static String get_script_encryption_key(const Ref &p_preset); + // Converts script encryption key to bytes. + static PackedByteArray convert_script_encryption_key_to_bytes(const String &p_encryption_key); + // Finds resource files from a file system directory. + static void export_find_resources(EditorFileSystemDirectory *p_dir, HashSet &p_paths); + static void export_find_customized_resources(const Ref &p_preset, EditorFileSystemDirectory *p_dir, EditorExportPreset::FileExportMode p_mode, HashSet &p_paths); + static void export_find_dependencies(const String &p_path, HashSet &p_paths); + static void export_find_preset_resources(const Ref &p_preset, HashSet &p_paths); + static void edit_files_with_filter(Ref &p_da, const Vector &p_filters, HashSet &p_list, bool p_exclude); + static void edit_filter_list(HashSet &p_list, const String &p_filter, bool p_exclude); + static Vector filter_extension_list_config_file(const String &p_config_path, const HashSet &p_paths); + static Vector get_forced_export_files(const Ref &p_preset); + static HashMap get_internal_export_files(const Ref &p_editor_export_platform, const Ref &p_preset, bool p_debug); + static EditorExportPlatformData::FilteredCache get_filtered_cache(const HashSet &p_paths); +}; diff --git a/editor/export/gdextension_export_plugin.h b/editor/export/gdextension_export_plugin.h index c80b7c5b3d9b..babaa1d44e48 100644 --- a/editor/export/gdextension_export_plugin.h +++ b/editor/export/gdextension_export_plugin.h @@ -161,11 +161,11 @@ void GDExtensionExportPlugin::_export_file(const String &p_path, const String &p for (const KeyValue &E : libs_found) { if (E.value.count == 0) { if (get_export_platform().is_valid()) { - get_export_platform()->add_message(EditorExportPlatform::EXPORT_MESSAGE_WARNING, TTR("GDExtension"), vformat(TTR("No \"%s\" library found for GDExtension: \"%s\". Possible feature flags for your platform: %s"), E.key, p_path, String(", ").join(features_vector))); + get_export_platform()->add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("GDExtension"), vformat(TTR("No \"%s\" library found for GDExtension: \"%s\". Possible feature flags for your platform: %s"), E.key, p_path, String(", ").join(features_vector))); } } else if (E.value.count > 1) { if (get_export_platform().is_valid()) { - get_export_platform()->add_message(EditorExportPlatform::EXPORT_MESSAGE_WARNING, TTR("GDExtension"), vformat(TTR("Multiple \"%s\" libraries found for GDExtension: \"%s\": \"%s\"."), E.key, p_path, String(", ").join(E.value.libs))); + get_export_platform()->add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("GDExtension"), vformat(TTR("Multiple \"%s\" libraries found for GDExtension: \"%s\": \"%s\"."), E.key, p_path, String(", ").join(E.value.libs))); } } } diff --git a/editor/export/project_export.cpp b/editor/export/project_export.cpp index 567e76e0a939..72b3bcc8077d 100644 --- a/editor/export/project_export.cpp +++ b/editor/export/project_export.cpp @@ -504,6 +504,11 @@ void ProjectExportDialog::_tab_changed(int) { _update_feature_list(); } +void ProjectExportDialog::_on_result_dialog_copy_to_clipboard() { + const String log_text = result_dialog_log->get_parsed_text(); + DisplayServer::get_singleton()->clipboard_set(log_text); +} + void ProjectExportDialog::_update_parameters(const String &p_edited_property) { _update_current_preset(); } @@ -2020,8 +2025,12 @@ ProjectExportDialog::ProjectExportDialog() { result_dialog->set_title(TTR("Project Export")); result_dialog_log = memnew(RichTextLabel); result_dialog_log->set_custom_minimum_size(Size2(300, 80) * EDSCALE); + result_dialog_log->set_selection_enabled(true); result_dialog->add_child(result_dialog_log); + copy_to_clipboard_button = result_dialog->add_button(TTRC("Copy to clipboard"), false, "copy_to_clipboard"); + copy_to_clipboard_button->connect(SceneStringName(pressed), callable_mp(this, &ProjectExportDialog::_on_result_dialog_copy_to_clipboard)); + main_vb->add_child(result_dialog); result_dialog->hide(); diff --git a/editor/export/project_export.h b/editor/export/project_export.h index a4f375c99f1c..b2c9ba3496f6 100644 --- a/editor/export/project_export.h +++ b/editor/export/project_export.h @@ -96,6 +96,7 @@ class ProjectExportDialog : public ConfirmationDialog { RichTextLabel *result_dialog_log = nullptr; AcceptDialog *result_dialog = nullptr; ConfirmationDialog *delete_confirm = nullptr; + Button *copy_to_clipboard_button = nullptr; OptionButton *export_filter = nullptr; LineEdit *include_filters = nullptr; @@ -227,6 +228,8 @@ class ProjectExportDialog : public ConfirmationDialog { void _tab_changed(int); + void _on_result_dialog_copy_to_clipboard(); + protected: void _notification(int p_what); static void _bind_methods(); diff --git a/editor/run/editor_run_native.cpp b/editor/run/editor_run_native.cpp index a1c42d222372..2eaa81044e55 100644 --- a/editor/run/editor_run_native.cpp +++ b/editor/run/editor_run_native.cpp @@ -140,23 +140,23 @@ Error EditorRunNative::start_run_native(int p_id) { bool debug_navigation = EditorSettings::get_singleton()->get_project_metadata("debug_options", "run_debug_navigation", false); if (deploy_debug_remote) { - flags.set_flag(EditorExportPlatform::DEBUG_FLAG_REMOTE_DEBUG); + flags.set_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG); } if (deploy_dumb) { - flags.set_flag(EditorExportPlatform::DEBUG_FLAG_DUMB_CLIENT); + flags.set_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); } if (debug_collisions) { - flags.set_flag(EditorExportPlatform::DEBUG_FLAG_VIEW_COLLISIONS); + flags.set_flag(EditorExportPlatformData::DEBUG_FLAG_VIEW_COLLISIONS); } if (debug_navigation) { - flags.set_flag(EditorExportPlatform::DEBUG_FLAG_VIEW_NAVIGATION); + flags.set_flag(EditorExportPlatformData::DEBUG_FLAG_VIEW_NAVIGATION); } eep->clear_messages(); Error err = eep->run(preset, idx, flags); result_dialog_log->clear(); if (eep->fill_log_messages(result_dialog_log, err)) { - if (eep->get_worst_message_type() >= EditorExportPlatform::EXPORT_MESSAGE_ERROR) { + if (eep->get_worst_message_type() >= EditorExportPlatformData::EXPORT_MESSAGE_ERROR) { result_dialog->popup_centered_ratio(0.5); } } diff --git a/main/main.cpp b/main/main.cpp index 68035f50756e..bee49a40211d 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -4935,6 +4935,8 @@ bool Main::iteration() { bool exit = false; + ResourceLoader::poll_async_pck_install(); + // process all our active interfaces #ifndef XR_DISABLED GodotProfileZoneGrouped(_profile_zone, "xr_server->_process"); diff --git a/misc/scripts/__init__.py b/misc/scripts/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/misc/scripts/copyright_headers.py b/misc/scripts/copyright_headers.py index 3d53c708ac5b..39182985bcf5 100755 --- a/misc/scripts/copyright_headers.py +++ b/misc/scripts/copyright_headers.py @@ -1,9 +1,15 @@ #!/usr/bin/env python3 +import io import os import sys +import typing -header = """\ +if typing.TYPE_CHECKING: + pass + + +HEADER = """\ /**************************************************************************/ /* $filename */ /**************************************************************************/ @@ -35,31 +41,44 @@ /**************************************************************************/ """ -if len(sys.argv) < 2: - print("Invalid usage of copyright_headers.py, it should be called with a path to one or multiple files.") - sys.exit(1) - -for f in sys.argv[1:]: - fname = f - - # Handle replacing $filename with actual filename and keep alignment - fsingle = os.path.basename(fname.strip()) - rep_fl = "$filename" - rep_fi = fsingle - len_fl = len(rep_fl) - len_fi = len(rep_fi) - # Pad with spaces to keep alignment - if len_fi < len_fl: - for x in range(len_fl - len_fi): - rep_fi += " " - elif len_fl < len_fi: - for x in range(len_fi - len_fl): - rep_fl += " " - if header.find(rep_fl) != -1: - text = header.replace(rep_fl, rep_fi) + +def process_file(filename): # type: (str) -> None + filename = filename.strip() + basename = os.path.basename(filename) + + new_buffer = None # type: typing.Union[io.StringIO, None] + with open(filename, "r", encoding="utf-8") as file: + new_buffer = process_file_buffer(basename, file) + with open(filename, "w", encoding="utf-8") as file: + CHUNK_SIZE = 1024 + while True: + chunk = new_buffer.read(CHUNK_SIZE) + if not chunk: + break + file.write(chunk) + + +def process_file_buffer(filename, text_buffer): # type: (str, io.TextIOWrapper) -> io.StringIO + output = io.StringIO() + + # Handle replacing $filename with actual filename and keep alignment. + token = "$filename" + padded_filename = filename + padded_token = token + filename_length = len(filename) + token_length = len(token) + + # Pad with spaces to keep alignment. + if filename_length < token_length: + padded_filename += " " * (token_length - filename_length) + elif token_length < filename_length: + padded_token += " " * (filename_length - token_length) + + if HEADER.find(token) != -1: + output.write(HEADER.replace(padded_token, padded_filename)) else: - text = header.replace("$filename", fsingle) - text += "\n" + output.write(HEADER.replace(token, filename)) + output.write("\n") # We now have the proper header, so we want to ignore the one in the original file # and potentially empty lines and badly formatted lines, while keeping comments that @@ -68,28 +87,35 @@ # In a second pass, we skip all consecutive comment lines starting with "/*", # then we can append the rest (step 2). - with open(fname.strip(), "r", encoding="utf-8") as fileread: - line = fileread.readline() - header_done = False + line = text_buffer.readline() + header_done = False - while line.strip() == "" and line != "": # Skip empty lines at the top - line = fileread.readline() + while line.strip() == "" and line != "": # Skip empty lines at the top + line = text_buffer.readline() - if line.find("/**********") == -1: # Godot header starts this way - # Maybe starting with a non-Godot comment, abort header magic + if line.find("/**********") == -1: # Godot header starts this way + # Maybe starting with a non-Godot comment, abort header magic + header_done = True + + while not header_done: # Handle header now + if line.find("/*") != 0: # No more starting with a comment header_done = True + if line.strip() != "": + output.write(line) + line = text_buffer.readline() + + while line != "": # Dump everything until EOF + output.write(line) + line = text_buffer.readline() + + output.seek(0) + return output - while not header_done: # Handle header now - if line.find("/*") != 0: # No more starting with a comment - header_done = True - if line.strip() != "": - text += line - line = fileread.readline() - while line != "": # Dump everything until EOF - text += line - line = fileread.readline() +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Invalid usage of copyright_headers.py, it should be called with a path to one or multiple files.") + sys.exit(1) - # Write - with open(fname.strip(), "w", encoding="utf-8", newline="\n") as filewrite: - filewrite.write(text) + for f in sys.argv[1:]: + process_file(f) diff --git a/modules/gdscript/gdscript_analyzer.cpp b/modules/gdscript/gdscript_analyzer.cpp index 07dbb2ac26ed..61924c8e35ba 100644 --- a/modules/gdscript/gdscript_analyzer.cpp +++ b/modules/gdscript/gdscript_analyzer.cpp @@ -620,6 +620,11 @@ Error GDScriptAnalyzer::resolve_class_inheritance(GDScriptParser::ClassNode *p_c class_type.native_type = result.native_type; p_class->set_datatype(class_type); + // Add base class to the list of dependencies. + if (result.kind == GDScriptParser::DataType::CLASS) { + parser->add_dependency(result.script_path); + } + // Apply annotations. for (GDScriptParser::AnnotationNode *&E : p_class->annotations) { resolve_annotation(E); @@ -956,6 +961,11 @@ GDScriptParser::DataType GDScriptAnalyzer::resolve_datatype(GDScriptParser::Type } p_type->set_datatype(result); + + if (result.kind == GDScriptParser::DataType::CLASS || result.kind == GDScriptParser::DataType::SCRIPT) { + parser->add_dependency(result.script_path); + } + return result; } @@ -4551,6 +4561,7 @@ void GDScriptAnalyzer::reduce_identifier(GDScriptParser::IdentifierNode *p_ident if (ScriptServer::is_global_class(name)) { p_identifier->set_datatype(make_global_class_meta_type(name, p_identifier)); + parser->add_dependency(p_identifier->get_datatype().script_path); return; } @@ -4593,6 +4604,7 @@ void GDScriptAnalyzer::reduce_identifier(GDScriptParser::IdentifierNode *p_ident } result.is_constant = true; p_identifier->set_datatype(result); + parser->add_dependency(autoload.path); return; } } @@ -4712,7 +4724,6 @@ void GDScriptAnalyzer::reduce_preload(GDScriptParser::PreloadNode *p_preload) { push_error("Preloaded path must be a constant string.", p_preload->path); } else { p_preload->resolved_path = p_preload->path->reduced_value; - // TODO: Save this as script dependency. if (p_preload->resolved_path.is_relative_path()) { p_preload->resolved_path = parser->script_path.get_base_dir().path_join(p_preload->resolved_path); } @@ -4748,6 +4759,8 @@ void GDScriptAnalyzer::reduce_preload(GDScriptParser::PreloadNode *p_preload) { push_error(vformat(R"(Could not preload resource file "%s".)", p_preload->resolved_path), p_preload->path); } } + + parser->add_dependency(p_preload->resolved_path); } } diff --git a/modules/gdscript/gdscript_parser.h b/modules/gdscript/gdscript_parser.h index 35eda1ab1a03..55712c8d4a55 100644 --- a/modules/gdscript/gdscript_parser.h +++ b/modules/gdscript/gdscript_parser.h @@ -1465,6 +1465,8 @@ class GDScriptParser { void reset_extents(Node *p_node, GDScriptTokenizer::Token p_token); void reset_extents(Node *p_node, Node *p_from); + HashSet dependencies; + template T *alloc_node() { T *node = memnew(T); @@ -1641,9 +1643,11 @@ class GDScriptParser { bool annotation_exists(const String &p_annotation_name) const; const List &get_errors() const { return errors; } - const List get_dependencies() const { - // TODO: Keep track of deps. - return List(); + const HashSet &get_dependencies() const { + return dependencies; + } + void add_dependency(const String &p_dependency) { + dependencies.insert(p_dependency); } #ifdef DEBUG_ENABLED diff --git a/platform/android/export/export_plugin.cpp b/platform/android/export/export_plugin.cpp index 29cf1f08a23f..7aa7699ed691 100644 --- a/platform/android/export/export_plugin.cpp +++ b/platform/android/export/export_plugin.cpp @@ -45,6 +45,7 @@ #include "core/version.h" #include "editor/editor_node.h" #include "editor/export/editor_export.h" +#include "editor/export/editor_export_platform_utils.h" #include "editor/export/editor_export_plugin.h" #include "editor/export/export_template_manager.h" #include "editor/file_system/editor_paths.h" @@ -828,8 +829,8 @@ Error EditorExportPlatformAndroid::save_apk_file(const Ref & const String simplified_path = simplify_path(p_path); Vector enc_data; - EditorExportPlatform::SavedData sd; - Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd); + EditorExportPlatformData::SavedData sd; + Error err = EditorExportPlatformUtils::store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd); if (err != OK) { return err; } @@ -874,7 +875,7 @@ Error EditorExportPlatformAndroid::copy_gradle_so(const Ref String dst_path = export_data->libs_directory.path_join(type).path_join(abi).path_join(filename); Vector data = FileAccess::get_file_as_bytes(p_so.path); print_verbose("Copying .so file from " + p_so.path + " to " + dst_path); - Error err = store_file_at_path(dst_path, data); + Error err = EditorExportPlatformUtils::store_file_at_path(dst_path, data); ERR_FAIL_COND_V_MSG(err, err, "Failed to copy .so file from " + p_so.path + " to " + dst_path); export_data->libs.push_back(dst_path); } @@ -1076,7 +1077,7 @@ void EditorExportPlatformAndroid::_write_tmp_manifest(const Ref &p_preset) const { @@ -1189,7 +1190,7 @@ void EditorExportPlatformAndroid::_fix_themes_xml(const Ref // Reconstruct the XML content from the modified lines. String xml_content = String("\n").join(new_lines); - store_string_at_path(themes_xml_path, xml_content); + EditorExportPlatformUtils::store_string_at_path(themes_xml_path, xml_content); print_verbose("Successfully modified " + themes_xml_path + ": " + "\n" + xml_content); } @@ -1960,7 +1961,7 @@ void EditorExportPlatformAndroid::_copy_icons_to_gradle_project(const Ref data; _process_launcher_icons(LAUNCHER_ICONS[i].export_path, p_main_image, LAUNCHER_ICONS[i].dimensions, data); - store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ICONS[i].export_path), data); + EditorExportPlatformUtils::store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ICONS[i].export_path), data); } if (p_foreground.is_valid() && !p_foreground->is_empty()) { @@ -1968,7 +1969,7 @@ void EditorExportPlatformAndroid::_copy_icons_to_gradle_project(const Ref data; _process_launcher_icons(LAUNCHER_ADAPTIVE_ICON_FOREGROUNDS[i].export_path, p_foreground, LAUNCHER_ADAPTIVE_ICON_FOREGROUNDS[i].dimensions, data); - store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_FOREGROUNDS[i].export_path), data); + EditorExportPlatformUtils::store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_FOREGROUNDS[i].export_path), data); } if (p_background.is_valid() && !p_background->is_empty()) { @@ -1976,7 +1977,7 @@ void EditorExportPlatformAndroid::_copy_icons_to_gradle_project(const Ref data; _process_launcher_icons(LAUNCHER_ADAPTIVE_ICON_BACKGROUNDS[i].export_path, p_background, LAUNCHER_ADAPTIVE_ICON_BACKGROUNDS[i].dimensions, data); - store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_BACKGROUNDS[i].export_path), data); + EditorExportPlatformUtils::store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_BACKGROUNDS[i].export_path), data); } if (p_monochrome.is_valid() && !p_monochrome->is_empty()) { @@ -1984,13 +1985,13 @@ void EditorExportPlatformAndroid::_copy_icons_to_gradle_project(const Ref data; _process_launcher_icons(LAUNCHER_ADAPTIVE_ICON_MONOCHROMES[i].export_path, p_monochrome, LAUNCHER_ADAPTIVE_ICON_MONOCHROMES[i].dimensions, data); - store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_MONOCHROMES[i].export_path), data); + EditorExportPlatformUtils::store_file_at_path(gradle_build_dir.path_join(LAUNCHER_ADAPTIVE_ICON_MONOCHROMES[i].export_path), data); monochrome_tag = " \n"; } } // Finalize the icon.xml by formatting the template with the optional monochrome tag. - store_string_at_path(gradle_build_dir.path_join(ICON_XML_PATH), vformat(ICON_XML_TEMPLATE, monochrome_tag)); + EditorExportPlatformUtils::store_string_at_path(gradle_build_dir.path_join(ICON_XML_PATH), vformat(ICON_XML_TEMPLATE, monochrome_tag)); } Vector EditorExportPlatformAndroid::get_enabled_abis(const Ref &p_preset) { @@ -2367,7 +2368,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, String can_export_error; bool can_export_missing_templates; if (!can_export(p_preset, can_export_error, can_export_missing_templates)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), can_export_error); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), can_export_error); return ERR_UNCONFIGURED; } @@ -2383,11 +2384,11 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, } const bool use_wifi_for_remote_debug = EDITOR_GET("export/android/use_wifi_for_remote_debug"); - const bool use_remote = p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT); + const bool use_remote = p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); const bool use_reverse = !use_wifi_for_remote_debug; if (use_reverse) { - p_debug_flags.set_flag(DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST); + p_debug_flags.set_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST); } String tmp_export_path = EditorPaths::get_singleton()->get_temp_dir().path_join("tmpexport." + uitos(OS::get_singleton()->get_unix_time()) + ".apk"); @@ -2458,7 +2459,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, err = OS::get_singleton()->execute(adb, args, &output, &rv, true); print_verbose(output); if (err || rv != 0) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Could not install to device: %s"), output)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Could not install to device: %s"), output)); CLEANUP_AND_RETURN(ERR_CANT_CREATE); } @@ -2477,7 +2478,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, OS::get_singleton()->execute(adb, args, &output, &rv, true); print_verbose(output); - if (p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG)) { int dbg_port = EDITOR_GET("network/debug/remote_port"); args.clear(); args.push_back("-s"); @@ -2492,7 +2493,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, print_line("Reverse result: " + itos(rv)); } - if (p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT)) { + if (p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT)) { int fs_port = EDITOR_GET("filesystem/file_server/port"); args.clear(); @@ -2543,7 +2544,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, Dictionary data = OS::get_singleton()->execute_with_pipe(scrcpy, args, false); if (!data.has("pid") || data["pid"].operator int() <= 0) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not start scrcpy executable. Configure scrcpy path in the Editor Settings (Export > Android > scrcpy > Path).")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not start scrcpy executable. Configure scrcpy path in the Editor Settings (Export > Android > scrcpy > Path).")); CLEANUP_AND_RETURN(ERR_CANT_CREATE); } bool connected = false; @@ -2573,7 +2574,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, print_verbose(err_output); if (!connected) { OS::get_singleton()->kill(data["pid"].operator int()); - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not execute on device, scrcpy failed with the following error:\n" + err_output)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not execute on device, scrcpy failed with the following error:\n" + err_output)); CLEANUP_AND_RETURN(ERR_CANT_CREATE); } } else { @@ -2610,7 +2611,7 @@ Error EditorExportPlatformAndroid::run(const Ref &p_preset, print_verbose(output); if (err || rv != 0 || output.begins_with("Error: Activity not started")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not execute on device.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), TTR("Could not execute on device.")); CLEANUP_AND_RETURN(ERR_CANT_CREATE); } } @@ -3270,7 +3271,7 @@ void EditorExportPlatformAndroid::get_command_line_flags(const Ref &p_preset, bool p_debug, const String &export_path, EditorProgress &ep) { int export_format = int(p_preset->get("gradle_build/export_format")); if (export_format == EXPORT_FORMAT_AAB) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("AAB signing is not supported")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("AAB signing is not supported")); return FAILED; } @@ -3303,9 +3304,9 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre if (!FileAccess::exists(keystore)) { if (p_debug) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not find debug keystore, unable to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not find debug keystore, unable to export.")); } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not find release keystore, unable to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not find release keystore, unable to export.")); } return ERR_FILE_CANT_OPEN; } @@ -3320,7 +3321,7 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre #ifdef ANDROID_ENABLED err = OS_Android::get_singleton()->sign_apk(apk_path, apk_path, keystore, user, password); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Unable to sign apk.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Unable to sign apk.")); return err; } #else @@ -3332,11 +3333,11 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre String apksigner = get_apksigner_path(target_sdk_version.to_int(), true); print_verbose("Starting signing of the APK binary using " + apksigner); if (apksigner == "") { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("All 'apksigner' tools located in Android SDK 'build-tools' directory failed to execute. Please check that you have the correct version installed for your target sdk version. The resulting APK is unsigned.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("All 'apksigner' tools located in Android SDK 'build-tools' directory failed to execute. Please check that you have the correct version installed for your target sdk version. The resulting APK is unsigned.")); return OK; } if (!FileAccess::exists(apksigner)) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("'apksigner' could not be found. Please check that the command is available in the Android SDK build-tools directory. The resulting APK is unsigned.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("'apksigner' could not be found. Please check that the command is available in the Android SDK build-tools directory. The resulting APK is unsigned.")); return OK; } @@ -3364,14 +3365,14 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre int retval; err = OS::get_singleton()->execute(apksigner, args, &output, &retval, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start apksigner executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start apksigner executable.")); return err; } // By design, apksigner does not output credentials in its output unless --verbose is used print_line(output); if (retval) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("'apksigner' returned with error #%d"), retval)); - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("output: \n%s"), output)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("'apksigner' returned with error #%d"), retval)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("output: \n%s"), output)); return ERR_CANT_CREATE; } #endif @@ -3383,7 +3384,7 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre #ifdef ANDROID_ENABLED err = OS_Android::get_singleton()->verify_apk(apk_path); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Unable to verify signed apk.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Unable to verify signed apk.")); return err; } #else @@ -3398,13 +3399,13 @@ Error EditorExportPlatformAndroid::sign_apk(const Ref &p_pre output.clear(); err = OS::get_singleton()->execute(apksigner, args, &output, &retval, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start apksigner executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start apksigner executable.")); return err; } print_verbose(output); if (retval) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("'apksigner' verification of APK failed.")); - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("output: \n%s"), output)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("'apksigner' verification of APK failed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("output: \n%s"), output)); return ERR_CANT_CREATE; } #endif @@ -3570,68 +3571,6 @@ Error EditorExportPlatformAndroid::export_project(const Ref return export_project_helper(p_preset, p_debug, p_path, export_format, should_sign, p_flags); } -Error EditorExportPlatformAndroid::_generate_sparse_pck_metadata(const Ref &p_preset, PackData &p_pack_data, Vector &r_data) { - Error err; - Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export_index", "tmp", false, &err); - if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not create temporary file!")); - return err; - } - int64_t pck_start_pos = ftmp->get_position(); - uint64_t file_base_ofs = 0; - uint64_t dir_base_ofs = 0; - EditorExportPlatform::_store_header(ftmp, p_preset->get_enc_pck() && p_preset->get_enc_directory(), true, file_base_ofs, dir_base_ofs, p_pack_data.salt); - - // Write directory. - uint64_t dir_offset = ftmp->get_position(); - ftmp->seek(dir_base_ofs); - ftmp->store_64(dir_offset - pck_start_pos); - ftmp->seek(dir_offset); - - Vector key; - if (p_preset->get_enc_pck() && p_preset->get_enc_directory()) { - String script_key = _get_script_encryption_key(p_preset); - key.resize(32); - if (script_key.length() == 64) { - for (int i = 0; i < 32; i++) { - int v = 0; - if (i * 2 < script_key.length()) { - char32_t ct = script_key[i * 2]; - if (is_digit(ct)) { - ct = ct - '0'; - } else if (ct >= 'a' && ct <= 'f') { - ct = 10 + ct - 'a'; - } - v |= ct << 4; - } - - if (i * 2 + 1 < script_key.length()) { - char32_t ct = script_key[i * 2 + 1]; - if (is_digit(ct)) { - ct = ct - '0'; - } else if (ct >= 'a' && ct <= 'f') { - ct = 10 + ct - 'a'; - } - v |= ct; - } - key.write[i] = v; - } - } - } - - if (!EditorExportPlatform::_encrypt_and_store_directory(ftmp, p_pack_data, key, p_preset->get_seed(), 0)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Can't create encrypted file.")); - return ERR_CANT_CREATE; - } - - r_data.resize(ftmp->get_length()); - ftmp->seek(0); - ftmp->get_buffer(r_data.ptrw(), r_data.size()); - ftmp.unref(); - - return OK; -} - #ifdef ANDROID_ENABLED // Copies the given keystore to temp file. // Returns the new path on success, or an empty String on failure. @@ -3661,7 +3600,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refget("gradle_build/use_gradle_build")); String gradle_build_directory = use_gradle_build ? ExportTemplateManager::get_android_build_directory(p_preset) : ""; - bool p_give_internet = p_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT) || p_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG); + bool p_give_internet = p_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT) || p_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG); bool apk_expansion = p_preset->get("apk_expansion/enable"); Vector enabled_abis = get_enabled_abis(p_preset); @@ -3701,25 +3640,25 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref EXPORT_FORMAT_AAB || export_format < EXPORT_FORMAT_APK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unsupported export format!")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unsupported export format!")); return ERR_UNCONFIGURED; } String err_string; if (!has_valid_username_and_password(p_preset, err_string)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR(err_string)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR(err_string)); return ERR_UNCONFIGURED; } @@ -3731,14 +3670,14 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref f = FileAccess::open(gradle_base_directory.path_join(".build_version"), FileAccess::READ); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Trying to build from a gradle built template, but no version info for it exists. Please reinstall from the 'Project' menu.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Trying to build from a gradle built template, but no version info for it exists. Please reinstall from the 'Project' menu.")); return ERR_UNCONFIGURED; } String current_version = ExportTemplateManager::get_android_template_identifier(p_preset); String installed_version = f->get_line().strip_edges(); print_verbose("- build version: " + installed_version); if (installed_version != current_version) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR(MISMATCHED_VERSIONS_MESSAGE), installed_version, current_version)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR(MISMATCHED_VERSIONS_MESSAGE), installed_version, current_version)); return ERR_UNCONFIGURED; } } @@ -3746,14 +3685,14 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refget("package/name")); err = _create_project_name_strings_files(p_preset, project_name, gradle_build_directory, get_project_setting(p_preset, "application/config/name_localized")); //project name localization. if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unable to overwrite res/*.xml files with project name.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unable to overwrite res/*.xml files with project name.")); } // Copies the project icon files into the appropriate Gradle project directory. _copy_icons_to_gradle_project(p_preset, main_image, foreground, background, monochrome); @@ -3782,7 +3721,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref enc_data; err = _generate_sparse_pck_metadata(p_preset, user_data.pd, enc_data); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); return err; } - err = store_file_at_path(user_data.assets_directory + "/assets.sparsepck", enc_data); + err = EditorExportPlatformUtils::store_file_at_path(user_data.assets_directory + "/assets.sparsepck", enc_data); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not write PCK directory!")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not write PCK directory!")); return err; } } if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not export project files to gradle project.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not export project files to gradle project.")); return err; } if (user_data.libs.size() > 0) { @@ -3820,13 +3759,13 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refget_resource_dir().path_join(debug_keystore).simplify_path(); } if (!FileAccess::exists(debug_keystore)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not find debug keystore, unable to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not find debug keystore, unable to export.")); return ERR_FILE_CANT_OPEN; } #ifdef ANDROID_ENABLED @@ -3968,7 +3907,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refget_resource_dir().path_join(release_keystore).simplify_path(); } if (!FileAccess::exists(release_keystore)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not find release keystore, unable to export.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not find release keystore, unable to export.")); return ERR_FILE_CANT_OPEN; } #ifdef ANDROID_ENABLED @@ -3994,7 +3933,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refexecute_and_show_output(TTR("Building Android Project (gradle)"), build_command, cmdline, true, false, &build_project_output); if (result != 0) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Building of Android project failed, check output for the error:") + "\n\n" + build_project_output); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Building of Android project failed, check output for the error:") + "\n\n" + build_project_output); return ERR_CANT_CREATE; } else { print_verbose(build_project_output); @@ -4055,7 +3994,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Refexecute_and_show_output(TTR("Moving output"), build_command, copy_args, true, false, ©_binary_output); if (copy_result != 0) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unable to copy and rename export file:") + "\n\n" + copy_binary_output); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Unable to copy and rename export file:") + "\n\n" + copy_binary_output); return ERR_CANT_CREATE; } else { print_verbose(copy_binary_output); @@ -4080,7 +4019,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref enc_data; err = _generate_sparse_pck_metadata(p_preset, ed.pd, enc_data); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Save PCK"), TTR("Could not generate sparse pck metadata!")); return err; } @@ -4321,7 +4260,7 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref &p_preset) const; - Error _generate_sparse_pck_metadata(const Ref &p_preset, PackData &p_pack_data, Vector &r_data); - protected: void _notification(int p_what); diff --git a/platform/android/export/gradle_export_util.cpp b/platform/android/export/gradle_export_util.cpp index a953c1a6a127..6840eb1a4872 100644 --- a/platform/android/export/gradle_export_util.cpp +++ b/platform/android/export/gradle_export_util.cpp @@ -129,48 +129,6 @@ String _get_app_category_label(int category_index) { } } -// Utility method used to create a directory. -Error create_directory(const String &p_dir) { - if (!DirAccess::exists(p_dir)) { - Ref filesystem_da = DirAccess::create(DirAccess::ACCESS_RESOURCES); - ERR_FAIL_COND_V_MSG(filesystem_da.is_null(), ERR_CANT_CREATE, "Cannot create directory '" + p_dir + "'."); - Error err = filesystem_da->make_dir_recursive(p_dir); - ERR_FAIL_COND_V_MSG(err, ERR_CANT_CREATE, "Cannot create directory '" + p_dir + "'."); - } - return OK; -} - -// Writes p_data into a file at p_path, creating directories if necessary. -// Note: this will overwrite the file at p_path if it already exists. -Error store_file_at_path(const String &p_path, const Vector &p_data) { - String dir = p_path.get_base_dir(); - Error err = create_directory(dir); - if (err != OK) { - return err; - } - Ref fa = FileAccess::open(p_path, FileAccess::WRITE); - ERR_FAIL_COND_V_MSG(fa.is_null(), ERR_CANT_CREATE, "Cannot create file '" + p_path + "'."); - fa->store_buffer(p_data.ptr(), p_data.size()); - return OK; -} - -// Writes string p_data into a file at p_path, creating directories if necessary. -// Note: this will overwrite the file at p_path if it already exists. -Error store_string_at_path(const String &p_path, const String &p_data) { - String dir = p_path.get_base_dir(); - Error err = create_directory(dir); - if (err != OK) { - if (OS::get_singleton()->is_stdout_verbose()) { - print_error("Unable to write data into " + p_path); - } - return err; - } - Ref fa = FileAccess::open(p_path, FileAccess::WRITE); - ERR_FAIL_COND_V_MSG(fa.is_null(), ERR_CANT_CREATE, "Cannot create file '" + p_path + "'."); - fa->store_string(p_data); - return OK; -} - // Implementation of EditorExportSaveFunction. // This method will only be called as an input to export_project_files. // It is used by the export_project_files method to save all the asset files into the gradle project. @@ -182,8 +140,8 @@ Error rename_and_store_file_in_gradle_project(const Ref &p_p const String simplified_path = EditorExportPlatform::simplify_path(p_path); Vector enc_data; - EditorExportPlatform::SavedData sd; - Error err = _store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd); + EditorExportPlatformData::SavedData sd; + Error err = EditorExportPlatformUtils::store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, enc_data, sd); if (err != OK) { return err; } @@ -195,7 +153,7 @@ Error rename_and_store_file_in_gradle_project(const Ref &p_p dst_path = export_data->assets_directory + String("/") + simplified_path.trim_prefix("res://"); } print_verbose("Saving project files from " + simplified_path + " into " + dst_path); - err = store_file_at_path(dst_path, enc_data); + err = EditorExportPlatformUtils::store_file_at_path(dst_path, enc_data); export_data->pd.file_ofs.push_back(sd); return err; @@ -221,7 +179,7 @@ Error _create_project_name_strings_files(const Ref &p_preset print_verbose("Creating strings resources for supported locales for project " + p_project_name); // Stores the string into the default values directory. String processed_default_xml_string = vformat(GODOT_PROJECT_NAME_XML_STRING, _android_xml_escape(p_project_name)); - store_string_at_path(p_gradle_build_dir.path_join("res/values/godot_project_name_string.xml"), processed_default_xml_string); + EditorExportPlatformUtils::store_string_at_path(p_gradle_build_dir.path_join("res/values/godot_project_name_string.xml"), processed_default_xml_string); // Searches the Gradle project res/ directory to find all supported locales Ref da = DirAccess::open(p_gradle_build_dir.path_join("res")); @@ -260,10 +218,10 @@ Error _create_project_name_strings_files(const Ref &p_preset if (locale_project_name != p_project_name) { String processed_xml_string = vformat(GODOT_PROJECT_NAME_XML_STRING, _android_xml_escape(locale_project_name)); print_verbose("Storing project name for locale " + locale + " under " + locale_directory); - store_string_at_path(locale_directory, processed_xml_string); + EditorExportPlatformUtils::store_string_at_path(locale_directory, processed_xml_string); } else { // TODO: Once the legacy build system is deprecated we don't need to have xml files for this else branch - store_string_at_path(locale_directory, processed_default_xml_string); + EditorExportPlatformUtils::store_string_at_path(locale_directory, processed_default_xml_string); } } da->list_dir_end(); @@ -414,35 +372,3 @@ String _get_application_tag(const Ref &p_export_platform, manifest_application_text += " \n"; return manifest_application_text; } - -Error _store_temp_file(const String &p_simplified_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta, Vector &r_enc_data, EditorExportPlatform::SavedData &r_sd) { - Error err = OK; - Ref ftmp = FileAccess::create_temp(FileAccess::WRITE_READ, "export", "tmp", false, &err); - if (err != OK) { - return err; - } - r_sd.path_utf8 = p_simplified_path.trim_prefix("res://").utf8(); - r_sd.ofs = 0; - r_sd.size = p_data.size(); - r_sd.delta = p_delta; - err = EditorExportPlatform::_encrypt_and_store_data(ftmp, p_simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, r_sd.encrypted); - if (err != OK) { - return err; - } - - r_enc_data.resize(ftmp->get_length()); - ftmp->seek(0); - ftmp->get_buffer(r_enc_data.ptrw(), r_enc_data.size()); - ftmp.unref(); - - // Store MD5 of original file. - { - unsigned char hash[16]; - CryptoCore::md5(p_data.ptr(), p_data.size(), hash); - r_sd.md5.resize(16); - for (int i = 0; i < 16; i++) { - r_sd.md5.write[i] = hash[i]; - } - } - return OK; -} diff --git a/platform/android/export/gradle_export_util.h b/platform/android/export/gradle_export_util.h index 048c912f9fa5..28a32d8b8e55 100644 --- a/platform/android/export/gradle_export_util.h +++ b/platform/android/export/gradle_export_util.h @@ -60,7 +60,7 @@ static const int XR_MODE_REGULAR = 0; static const int XR_MODE_OPENXR = 1; struct CustomExportData { - EditorExportPlatform::PackData pd; + EditorExportPlatformData::PackData pd; String assets_directory; String libs_directory; bool debug; @@ -80,19 +80,6 @@ int _get_app_category_value(int category_index); String _get_app_category_label(int category_index); -Error _store_temp_file(const String &p_simplified_path, const Vector &p_data, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta, Vector &r_enc_data, EditorExportPlatform::SavedData &r_sd); - -// Utility method used to create a directory. -Error create_directory(const String &p_dir); - -// Writes p_data into a file at p_path, creating directories if necessary. -// Note: this will overwrite the file at p_path if it already exists. -Error store_file_at_path(const String &p_path, const Vector &p_data); - -// Writes string p_data into a file at p_path, creating directories if necessary. -// Note: this will overwrite the file at p_path if it already exists. -Error store_string_at_path(const String &p_path, const String &p_data); - // Implementation of EditorExportSaveFunction. // This method will only be called as an input to export_project_files. // It is used by the export_project_files method to save all the asset files into the gradle project. diff --git a/platform/ios/export/export_plugin.cpp b/platform/ios/export/export_plugin.cpp index fa5211f9ddbe..f20bd52a8e3d 100644 --- a/platform/ios/export/export_plugin.cpp +++ b/platform/ios/export/export_plugin.cpp @@ -237,7 +237,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr Ref da = DirAccess::open(p_iconset_dir); if (da.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not open a directory at path \"%s\"."), p_iconset_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not open a directory at path \"%s\"."), p_iconset_dir)); return ERR_CANT_OPEN; } @@ -288,7 +288,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr Error err = OK; Ref img = _load_icon_or_splash_image(icon_path, &err); if (err != OK || img.is_null() || img->is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Invalid icon (%s): '%s'.", info.preset_key, icon_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Invalid icon (%s): '%s'.", info.preset_key, icon_path)); return ERR_UNCONFIGURED; } else if (info.force_opaque && img->detect_alpha() != Image::ALPHA_NONE) { img->resize(side_size, side_size, (Image::Interpolation)(p_preset->get("application/icon_interpolation").operator int())); @@ -301,7 +301,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr err = img->save_png(p_iconset_dir + exp_name); } if (err) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Failed to export icon (%s): '%s'.", info.preset_key, icon_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Failed to export icon (%s): '%s'.", info.preset_key, icon_path)); return err; } } else { @@ -309,11 +309,11 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr Error err = OK; Ref img = _load_icon_or_splash_image(icon_path, &err); if (err != OK || img.is_null() || img->is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Invalid icon (%s): '%s'.", info.preset_key, icon_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Invalid icon (%s): '%s'.", info.preset_key, icon_path)); return ERR_UNCONFIGURED; } else if (info.force_opaque && img->detect_alpha() != Image::ALPHA_NONE) { if (resize_waning) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Export Icons"), vformat("Icon (%s) must be opaque.", info.preset_key)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Export Icons"), vformat("Icon (%s) must be opaque.", info.preset_key)); } img->resize(side_size, side_size, (Image::Interpolation)(p_preset->get("application/icon_interpolation").operator int())); Ref new_img = Image::create_empty(side_size, side_size, false, Image::FORMAT_RGBA8); @@ -322,7 +322,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr err = new_img->save_png(p_iconset_dir + exp_name); } else if (img->get_width() != side_size || img->get_height() != side_size) { if (resize_waning) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Export Icons"), vformat("Icon (%s): '%s' has incorrect size %s and was automatically resized to %s.", info.preset_key, icon_path, img->get_size(), Vector2i(side_size, side_size))); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Export Icons"), vformat("Icon (%s): '%s' has incorrect size %s and was automatically resized to %s.", info.preset_key, icon_path, img->get_size(), Vector2i(side_size, side_size))); } img->resize(side_size, side_size, (Image::Interpolation)(p_preset->get("application/icon_interpolation").operator int())); err = img->save_png(p_iconset_dir + exp_name); @@ -333,7 +333,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr } if (err) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Failed to export icon (%s): '%s'.", info.preset_key, icon_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat("Failed to export icon (%s): '%s'.", info.preset_key, icon_path)); return err; } } @@ -368,7 +368,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr Ref json_file = FileAccess::open(p_iconset_dir + "Contents.json", FileAccess::WRITE); if (json_file.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not write to a file at path \"%s\"."), p_iconset_dir + "Contents.json")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not write to a file at path \"%s\"."), p_iconset_dir + "Contents.json")); return ERR_CANT_CREATE; } @@ -377,7 +377,7 @@ Error EditorExportPlatformIOS::_export_icons(const Ref &p_pr Ref sizes_file = FileAccess::open(p_iconset_dir + "sizes", FileAccess::WRITE); if (sizes_file.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not write to a file at path \"%s\"."), p_iconset_dir + "sizes")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export Icons"), vformat(TTR("Could not write to a file at path \"%s\"."), p_iconset_dir + "sizes")); return ERR_CANT_CREATE; } diff --git a/platform/linuxbsd/export/export_plugin.cpp b/platform/linuxbsd/export/export_plugin.cpp index 9d00b55691f6..8ea2ddab4a6c 100644 --- a/platform/linuxbsd/export/export_plugin.cpp +++ b/platform/linuxbsd/export/export_plugin.cpp @@ -49,7 +49,7 @@ Error EditorExportPlatformLinuxBSD::_export_debug_script(const Ref &p_preset, const String &p_app_name, const String &p_pkg_name, const String &p_path) { Ref f = FileAccess::open(p_path, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Debug Script Export"), vformat(TTR("Could not open file \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Debug Script Export"), vformat(TTR("Could not open file \"%s\"."), p_path)); return ERR_CANT_CREATE; } @@ -71,7 +71,7 @@ Error EditorExportPlatformLinuxBSD::export_project(const Ref if (!template_path.is_empty()) { String exe_arch = _get_exe_arch(template_path); if (arch != exe_arch) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Mismatching custom export template executable architecture: found \"%s\", expected \"%s\"."), exe_arch, arch)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Mismatching custom export template executable architecture: found \"%s\", expected \"%s\"."), exe_arch, arch)); return ERR_CANT_CREATE; } } @@ -99,7 +99,7 @@ Error EditorExportPlatformLinuxBSD::export_project(const Ref Ref tmp_app_dir = DirAccess::create_for_path(tmp_dir_path); if (export_as_zip) { if (tmp_app_dir.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), tmp_dir_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), tmp_dir_path)); return ERR_CANT_CREATE; } if (DirAccess::exists(tmp_dir_path)) { @@ -125,7 +125,7 @@ Error EditorExportPlatformLinuxBSD::export_project(const Ref err = _export_debug_script(p_preset, pkg_name, path.get_file(), scr_path); FileAccess::set_unix_permissions(scr_path, 0755); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Debug Console Export"), TTR("Could not create console wrapper.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Debug Console Export"), TTR("Could not create console wrapper.")); } } @@ -300,7 +300,7 @@ Error EditorExportPlatformLinuxBSD::fixup_embedded_pck(const String &p_path, int Ref f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to open executable file \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to open executable file \"%s\"."), p_path)); return ERR_CANT_OPEN; } @@ -308,7 +308,7 @@ Error EditorExportPlatformLinuxBSD::fixup_embedded_pck(const String &p_path, int { uint32_t magic = f->get_32(); if (magic != 0x464c457f) { // 0x7F + "ELF" - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable file header corrupted.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable file header corrupted.")); return ERR_FILE_CORRUPT; } } @@ -318,7 +318,7 @@ Error EditorExportPlatformLinuxBSD::fixup_embedded_pck(const String &p_path, int int bits = f->get_8() * 32; if (bits == 32 && p_embedded_size >= 0x100000000) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("32-bit executables cannot have embedded data >= 4 GiB.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("32-bit executables cannot have embedded data >= 4 GiB.")); } // Get info about the section header table. @@ -396,7 +396,7 @@ Error EditorExportPlatformLinuxBSD::fixup_embedded_pck(const String &p_path, int memfree(strings); if (!found) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable \"pck\" section not found.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable \"pck\" section not found.")); return ERR_FILE_CORRUPT; } return OK; @@ -525,7 +525,7 @@ Error EditorExportPlatformLinuxBSD::run(const Ref &p_preset, } } - const bool use_remote = p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT); + const bool use_remote = p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); int dbg_port = EDITOR_GET("network/debug/remote_port"); print_line("Creating temporary directory..."); diff --git a/platform/macos/export/export_plugin.cpp b/platform/macos/export/export_plugin.cpp index 77cf6d5d9a2d..db5c2c9beaef 100644 --- a/platform/macos/export/export_plugin.cpp +++ b/platform/macos/export/export_plugin.cpp @@ -967,13 +967,13 @@ Error EditorExportPlatformMacOS::_export_liquid_glass_icon(const Refexecute(actool, args, &str, &exitcode, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not start 'actool' executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not start 'actool' executable.")); return err; } PList info_plist; if (!info_plist.load_string(str, err_str)) { print_verbose(str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not read 'actool' version.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not read 'actool' version.")); return err; } if (info_plist.get_root()->data_type == PList::PLNodeType::PL_NODE_TYPE_DICT && info_plist.get_root()->data_dict.has("com.apple.actool.version")) { @@ -981,7 +981,7 @@ Error EditorExportPlatformMacOS::_export_liquid_glass_icon(const Refdata_type == PList::PLNodeType::PL_NODE_TYPE_DICT && dict->data_dict.has("short-bundle-version")) { float version = String::utf8(dict->data_dict["short-bundle-version"]->data_string.get_data()).to_float(); if (version < 26.0) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), vformat(TTR("At least version 26.0 of 'actool' is required (version %f found)."), version)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), vformat(TTR("At least version 26.0 of 'actool' is required (version %f found)."), version)); return ERR_UNAVAILABLE; } } @@ -1016,7 +1016,7 @@ Error EditorExportPlatformMacOS::_export_liquid_glass_icon(const Refexecute(actool, args, &str, &exitcode, true); if (err != OK || str.contains("error:") || !FileAccess::exists(p_app_path + "/Contents/Resources/Assets.car") || !FileAccess::exists(plist)) { print_verbose(str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not export liquid glass icon:") + "\n" + str); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Liquid Glass Icons"), TTR("Could not export liquid glass icon:") + "\n" + str); return err; } @@ -1038,7 +1038,7 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres String rcodesign = EDITOR_GET("export/macos/rcodesign").operator String(); if (rcodesign.is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("rcodesign path is not set. Configure rcodesign path in the Editor Settings (Export > macOS > rcodesign).")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("rcodesign path is not set. Configure rcodesign path in the Editor Settings (Export > macOS > rcodesign).")); return Error::FAILED; } @@ -1047,11 +1047,11 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres args.push_back("notary-submit"); if (p_preset->get_or_env("notarization/api_uuid", ENV_MAC_NOTARIZATION_UUID) == "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect issuer ID name not specified.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect issuer ID name not specified.")); return Error::FAILED; } if (p_preset->get_or_env("notarization/api_key", ENV_MAC_NOTARIZATION_KEY) == "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect API key ID not specified.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect API key ID not specified.")); return Error::FAILED; } @@ -1073,25 +1073,25 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres Error err = OS::get_singleton()->execute(rcodesign, args, &str, &exitcode, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Could not start rcodesign executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Could not start rcodesign executable.")); return err; } int rq_offset = str.find("created submission ID:"); if (exitcode != 0 || rq_offset == -1) { print_line("rcodesign (" + p_path + "):\n" + str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Notarization failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Notarization failed, see editor log for details.")); return Error::FAILED; } else { print_verbose("rcodesign (" + p_path + "):\n" + str); int next_nl = str.find_char('\n', rq_offset); String request_uuid = (next_nl == -1) ? str.substr(rq_offset + 23) : str.substr(rq_offset + 23, next_nl - rq_offset - 23); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), vformat(TTR("Notarization request UUID: \"%s\""), request_uuid)); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("The notarization process generally takes less than an hour.")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("You can check the progress manually by opening a Terminal and running the following command:")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"rcodesign notary-log --api-issuer --api-key \""); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("Run the following command to staple the notarization ticket to the exported application (optional):")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"rcodesign staple \""); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), vformat(TTR("Notarization request UUID: \"%s\""), request_uuid)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("The notarization process generally takes less than an hour.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("You can check the progress manually by opening a Terminal and running the following command:")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"rcodesign notary-log --api-issuer --api-key \""); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("Run the following command to staple the notarization ticket to the exported application (optional):")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"rcodesign staple \""); } } break; #ifdef MACOS_ENABLED @@ -1099,7 +1099,7 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres print_verbose("using notarytool notarization..."); if (!FileAccess::exists("/usr/bin/xcrun") && !FileAccess::exists("/bin/xcrun")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Xcode command line tools are not installed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Xcode command line tools are not installed.")); return Error::FAILED; } @@ -1111,17 +1111,17 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres args.push_back(p_path); if (p_preset->get_or_env("notarization/apple_id_name", ENV_MAC_NOTARIZATION_APPLE_ID) == "" && p_preset->get_or_env("notarization/api_uuid", ENV_MAC_NOTARIZATION_UUID) == "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Neither Apple ID name nor App Store Connect issuer ID name not specified.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Neither Apple ID name nor App Store Connect issuer ID name not specified.")); return Error::FAILED; } if (p_preset->get_or_env("notarization/apple_id_name", ENV_MAC_NOTARIZATION_APPLE_ID) != "" && p_preset->get_or_env("notarization/api_uuid", ENV_MAC_NOTARIZATION_UUID) != "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Both Apple ID name and App Store Connect issuer ID name are specified, only one should be set at the same time.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Both Apple ID name and App Store Connect issuer ID name are specified, only one should be set at the same time.")); return Error::FAILED; } if (p_preset->get_or_env("notarization/apple_id_name", ENV_MAC_NOTARIZATION_APPLE_ID) != "") { if (p_preset->get_or_env("notarization/apple_id_password", ENV_MAC_NOTARIZATION_APPLE_PASS) == "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Apple ID password not specified.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("Apple ID password not specified.")); return Error::FAILED; } args.push_back("--apple-id"); @@ -1131,7 +1131,7 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres args.push_back(p_preset->get_or_env("notarization/apple_id_password", ENV_MAC_NOTARIZATION_APPLE_PASS)); } else { if (p_preset->get_or_env("notarization/api_key_id", ENV_MAC_NOTARIZATION_KEY_ID) == "") { - add_message(EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect API key ID not specified.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Notarization"), TTR("App Store Connect API key ID not specified.")); return Error::FAILED; } args.push_back("--issuer"); @@ -1157,27 +1157,27 @@ Error EditorExportPlatformMacOS::_notarize(const Ref &p_pres int exitcode = 0; Error err = OS::get_singleton()->execute("xcrun", args, &str, &exitcode, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Could not start xcrun executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Could not start xcrun executable.")); return err; } int rq_offset = str.find("id:"); if (exitcode != 0 || rq_offset == -1) { print_line("notarytool (" + p_path + "):\n" + str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Notarization failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Notarization"), TTR("Notarization failed, see editor log for details.")); return Error::FAILED; } else { print_verbose("notarytool (" + p_path + "):\n" + str); int next_nl = str.find_char('\n', rq_offset); String request_uuid = (next_nl == -1) ? str.substr(rq_offset + 4) : str.substr(rq_offset + 4, next_nl - rq_offset - 4); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), vformat(TTR("Notarization request UUID: \"%s\""), request_uuid)); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("The notarization process generally takes less than an hour.")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("See instructions on finding your team ID: https://developer.apple.com/help/glossary/team-id")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("You can check the progress manually by opening a Terminal and running the following command:")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun notarytool log --issuer --key-id --key \" or"); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun notarytool log --team-id --apple-id --password \""); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("Run the following command to staple the notarization ticket to the exported application (optional):")); - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun stapler staple \""); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), vformat(TTR("Notarization request UUID: \"%s\""), request_uuid)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("The notarization process generally takes less than an hour.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("See instructions on finding your team ID: https://developer.apple.com/help/glossary/team-id")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("You can check the progress manually by opening a Terminal and running the following command:")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun notarytool log --issuer --key-id --key \" or"); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun notarytool log --team-id --apple-id --password \""); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t" + TTR("Run the following command to staple the notarization ticket to the exported application (optional):")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), "\t\t\"xcrun stapler staple \""); } } break; #endif @@ -1195,7 +1195,7 @@ void EditorExportPlatformMacOS::_code_sign(const Ref &p_pres String error_msg; Error err = CodeSign::codesign(false, true, p_path, p_ent_path, error_msg); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Built-in CodeSign failed with error \"%s\"."), error_msg)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Built-in CodeSign failed with error \"%s\"."), error_msg)); return; } } break; @@ -1204,7 +1204,7 @@ void EditorExportPlatformMacOS::_code_sign(const Ref &p_pres String rcodesign = EDITOR_GET("export/macos/rcodesign").operator String(); if (rcodesign.is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Xrcodesign path is not set. Configure rcodesign path in the Editor Settings (Export > macOS > rcodesign).")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Xrcodesign path is not set. Configure rcodesign path in the Editor Settings (Export > macOS > rcodesign).")); return; } @@ -1242,13 +1242,13 @@ void EditorExportPlatformMacOS::_code_sign(const Ref &p_pres Error err = OS::get_singleton()->execute(rcodesign, args, &str, &exitcode, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start rcodesign executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start rcodesign executable.")); return; } if (exitcode != 0) { print_line("rcodesign (" + p_path + "):\n" + str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Code signing failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Code signing failed, see editor log for details.")); return; } else { print_verbose("rcodesign (" + p_path + "):\n" + str); @@ -1259,7 +1259,7 @@ void EditorExportPlatformMacOS::_code_sign(const Ref &p_pres print_verbose("using xcode codesign..."); if (!FileAccess::exists("/usr/bin/codesign") && !FileAccess::exists("/bin/codesign")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Xcode command line tools are not installed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Xcode command line tools are not installed.")); return; } @@ -1308,13 +1308,13 @@ void EditorExportPlatformMacOS::_code_sign(const Ref &p_pres Error err = OS::get_singleton()->execute("codesign", args, &str, &exitcode, true); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start codesign executable, make sure Xcode command line tools are installed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start codesign executable, make sure Xcode command line tools are installed.")); return; } if (exitcode != 0) { print_line("codesign (" + p_path + "):\n" + str); - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Code signing failed, see editor log for details.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Code signing failed, see editor log for details.")); return; } else { print_verbose("codesign (" + p_path + "):\n" + str); @@ -1341,7 +1341,7 @@ void EditorExportPlatformMacOS::_code_sign_directory(const Ref dir_access{ DirAccess::open(p_path, &dir_access_error) }; if (dir_access_error != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Cannot sign directory %s."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Cannot sign directory %s."), p_path)); return; } @@ -1373,7 +1373,7 @@ void EditorExportPlatformMacOS::_code_sign_directory(const Refcurrent_is_dir()) { _code_sign_directory(p_preset, current_file_path, p_ent_path, p_helper_ent_path, p_should_error_on_non_code); } else if (p_should_error_on_non_code) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Cannot sign file %s."), current_file)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Cannot sign file %s."), current_file)); } current_file = dir_access->get_next(); @@ -1396,7 +1396,7 @@ Error EditorExportPlatformMacOS::_copy_and_sign_files(Ref &dir_access Error err{ OK }; if (dir_access->dir_exists(p_src_path)) { #ifndef UNIX_ENABLED - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Relative symlinks are not supported, exported \"%s\" might be broken!"), p_src_path.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Relative symlinks are not supported, exported \"%s\" might be broken!"), p_src_path.get_file())); #endif print_verbose("export framework: " + p_src_path + " -> " + p_in_app_path); @@ -1420,7 +1420,7 @@ Error EditorExportPlatformMacOS::_copy_and_sign_files(Ref &dir_access err = dir_access->copy_dir(p_src_path, p_in_app_path, -1, true); } if (err == OK && plist_missing) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Export"), vformat(TTR("\"%s\": Info.plist missing or invalid, new Info.plist generated."), p_src_path.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Export"), vformat(TTR("\"%s\": Info.plist missing or invalid, new Info.plist generated."), p_src_path.get_file())); // Generate Info.plist String lib_name = p_src_path.get_basename().get_file(); String lib_id = p_preset->get("application/bundle_identifier"); @@ -1536,13 +1536,13 @@ Error EditorExportPlatformMacOS::_create_pkg(const Ref &p_pr String str; Error err = OS::get_singleton()->execute("xcrun", args, &str, nullptr, true); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PKG Creation"), TTR("Could not start productbuild executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PKG Creation"), TTR("Could not start productbuild executable.")); return err; } print_verbose("productbuild returned: " + str); if (str.contains("productbuild: error:")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PKG Creation"), TTR("`productbuild` failed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PKG Creation"), TTR("`productbuild` failed.")); return FAILED; } @@ -1568,16 +1568,16 @@ Error EditorExportPlatformMacOS::_create_dmg(const String &p_dmg_path, const Str String str; Error err = OS::get_singleton()->execute("hdiutil", args, &str, nullptr, true); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("Could not start hdiutil executable.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("Could not start hdiutil executable.")); return err; } print_verbose("hdiutil returned: " + str); if (str.contains("create failed")) { if (str.contains("File exists")) { - add_message(EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("`hdiutil create` failed - file exists.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("`hdiutil create` failed - file exists.")); } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("`hdiutil create` failed.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("DMG Creation"), TTR("`hdiutil create` failed.")); } return FAILED; } @@ -1599,7 +1599,7 @@ bool EditorExportPlatformMacOS::is_executable(const String &p_path) const { Error EditorExportPlatformMacOS::_export_debug_script(const Ref &p_preset, const String &p_app_name, const String &p_pkg_name, const String &p_path) { Ref f = FileAccess::open(p_path, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Debug Script Export"), vformat(TTR("Could not open file \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Debug Script Export"), vformat(TTR("Could not open file \"%s\"."), p_path)); return ERR_CANT_CREATE; } @@ -1629,7 +1629,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p const String base_dir = p_path.get_base_dir(); if (!DirAccess::exists(base_dir)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Target folder does not exist or is inaccessible: \"%s\""), base_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Target folder does not exist or is inaccessible: \"%s\""), base_dir)); return ERR_FILE_BAD_PATH; } @@ -1646,7 +1646,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p String err; src_pkg_name = find_export_template("macos.zip", &err); if (src_pkg_name.is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), TTR("Export template not found.") + "\n" + err); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), TTR("Export template not found.") + "\n" + err); return ERR_FILE_NOT_FOUND; } } @@ -1660,7 +1660,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p unzFile src_pkg_zip = unzOpen2(src_pkg_name.utf8().get_data(), &io); if (!src_pkg_zip) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not find template app to export: \"%s\"."), src_pkg_name)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not find template app to export: \"%s\"."), src_pkg_name)); return ERR_FILE_NOT_FOUND; } @@ -1689,7 +1689,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p export_format = "pkg"; #endif } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Invalid export format.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Invalid export format.")); return ERR_CANT_CREATE; } @@ -1714,7 +1714,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p Ref tmp_app_dir = DirAccess::create_for_path(tmp_base_path_name); if (tmp_app_dir.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory: \"%s\"."), tmp_base_path_name)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory: \"%s\"."), tmp_base_path_name)); err = ERR_CANT_CREATE; } @@ -1736,7 +1736,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p print_verbose("Creating " + tmp_app_path_name + "/Contents/MacOS"); err = tmp_app_dir->make_dir_recursive(tmp_app_path_name + "/Contents/MacOS"); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/MacOS")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/MacOS")); } } @@ -1744,7 +1744,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p print_verbose("Creating " + tmp_app_path_name + "/Contents/Frameworks"); err = tmp_app_dir->make_dir_recursive(tmp_app_path_name + "/Contents/Frameworks"); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Frameworks")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Frameworks")); } } @@ -1752,7 +1752,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p print_line("Creating " + tmp_app_path_name + "/Contents/Helpers"); err = tmp_app_dir->make_dir_recursive(tmp_app_path_name + "/Contents/Helpers"); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Helpers")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Helpers")); } } @@ -1760,7 +1760,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p print_verbose("Creating " + tmp_app_path_name + "/Contents/Resources"); err = tmp_app_dir->make_dir_recursive(tmp_app_path_name + "/Contents/Resources"); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Resources")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), tmp_app_path_name + "/Contents/Resources")); } } @@ -1925,21 +1925,21 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p if (((info.external_fa >> 16L) & 0120000) == 0120000) { #ifndef UNIX_ENABLED - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Relative symlinks are not supported on this OS, the exported project might be broken!")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Relative symlinks are not supported on this OS, the exported project might be broken!")); #endif // Handle symlinks in the archive. file = tmp_app_path_name.path_join(file); if (err == OK) { err = tmp_app_dir->make_dir_recursive(file.get_base_dir()); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), file.get_base_dir())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), file.get_base_dir())); } } if (err == OK) { String lnk_data = String::utf8((const char *)data.ptr(), data.size()); err = tmp_app_dir->create_link(lnk_data, file); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not created symlink \"%s\" -> \"%s\"."), lnk_data, file)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not created symlink \"%s\" -> \"%s\"."), lnk_data, file)); } print_verbose(vformat("ADDING SYMLINK %s => %s\n", file, lnk_data)); } @@ -2023,7 +2023,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p if (err == OK) { err = tmp_app_dir->make_dir_recursive(file.get_base_dir()); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), file.get_base_dir())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not create directory \"%s\"."), file.get_base_dir())); } } if (err == OK) { @@ -2036,12 +2036,12 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p FileAccess::set_unix_permissions(file, 0755); #ifndef UNIX_ENABLED if (export_format == "app") { - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), "Contents/MacOS/" + file.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), "Contents/MacOS/" + file.get_file())); } #endif } } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not open \"%s\"."), file)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not open \"%s\"."), file)); err = ERR_CANT_CREATE; } } @@ -2054,7 +2054,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p unzClose(src_pkg_zip); if (!found_binary) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Requested template binary \"%s\" not found. It might be missing from your template archive."), binary_to_use)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Requested template binary \"%s\" not found. It might be missing from your template archive."), binary_to_use)); err = ERR_FILE_NOT_FOUND; } @@ -2066,11 +2066,11 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p FileAccess::set_unix_permissions(scr_path, 0755); #ifndef UNIX_ENABLED if (export_format == "app") { - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), scr_path.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), scr_path.get_file())); } #endif if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not create console wrapper.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), TTR("Could not create console wrapper.")); } } } @@ -2106,12 +2106,12 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p bool lib_validation = p_preset->get("codesign/entitlements/disable_library_validation"); if (!shared_objects.is_empty() && sign_enabled && ad_hoc && !lib_validation) { - add_message(EXPORT_MESSAGE_INFO, TTR("Entitlements Modified"), TTR("Ad-hoc signed applications require the 'Disable Library Validation' entitlement to load dynamic libraries.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Entitlements Modified"), TTR("Ad-hoc signed applications require the 'Disable Library Validation' entitlement to load dynamic libraries.")); lib_validation = true; } if (!shared_objects.is_empty() && sign_enabled && codesign_tool == 2) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("'rcodesign' doesn't support signing applications with embedded dynamic libraries.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("'rcodesign' doesn't support signing applications with embedded dynamic libraries.")); } bool sandbox = p_preset->get("codesign/entitlements/app_sandbox/enabled"); @@ -2269,7 +2269,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p ent_f->store_line(""); ent_f->store_line(""); } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not create entitlements file.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not create entitlements file.")); err = ERR_CANT_CREATE; } @@ -2287,7 +2287,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p ent_f->store_line(""); ent_f->store_line(""); } else { - add_message(EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not create helper entitlements file.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Code Signing"), TTR("Could not create helper entitlements file.")); err = ERR_CANT_CREATE; } } @@ -2304,7 +2304,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p FileAccess::set_unix_permissions(tmp_app_path_name + "/Contents/Helpers/" + hlp_path.get_file(), 0755); #ifndef UNIX_ENABLED if (export_format == "app") { - add_message(EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), "Contents/Helpers/" + hlp_path.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Export"), vformat(TTR("Unable to set Unix permissions for executable \"%s\". Use \"chmod +x\" to set it after transferring the exported .app to macOS or Linux."), "Contents/Helpers/" + hlp_path.get_file())); } #endif } @@ -2419,7 +2419,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p if (err == OK && noto_enabled) { if (export_format == "pkg") { - add_message(EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("Notarization requires the app to be archived first, select the DMG or ZIP export format instead.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Notarization"), TTR("Notarization requires the app to be archived first, select the DMG or ZIP export format instead.")); } else { if (ep.step(TTR("Sending archive for notarization"), 4)) { return ERR_SKIP; @@ -2742,7 +2742,7 @@ Error EditorExportPlatformMacOS::run(const Ref &p_preset, in } } - const bool use_remote = p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT); + const bool use_remote = p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); int dbg_port = EDITOR_GET("network/debug/remote_port"); print_line("Creating temporary directory..."); diff --git a/platform/web/detect.py b/platform/web/detect.py index 68608e17ef64..67ac5c22171f 100644 --- a/platform/web/detect.py +++ b/platform/web/detect.py @@ -11,7 +11,7 @@ create_engine_file, create_template_zip, get_template_zip_path, - run_closure_compiler, + package_js_module_generator, ) from SCons.Util import WhereIs @@ -53,7 +53,6 @@ def get_opts(): BoolVariable( "dlink_enabled", "Enable WebAssembly dynamic linking (GDExtension support). Produces bigger binaries", False ), - BoolVariable("use_closure_compiler", "Use closure compiler to minimize JavaScript code", False), BoolVariable( "proxy_to_pthread", "Use Emscripten PROXY_TO_PTHREAD option to run the main application code to a separate thread", @@ -197,21 +196,6 @@ def configure(env: "SConsEnvironment"): if env["use_safe_heap"]: env.Append(LINKFLAGS=["-sSAFE_HEAP=1"]) - # Closure compiler - if env["use_closure_compiler"] and cc_semver < (4, 0, 11): - print_warning( - '"use_closure_compiler=yes" support requires Emscripten 4.0.11 (detected %s.%s.%s), using "use_closure_compiler=no" instead.' - % cc_semver - ) - env["use_closure_compiler"] = False - - if env["use_closure_compiler"]: - # For emscripten support code. - env.Append(LINKFLAGS=["--closure", "1"]) - # Register builder for our Engine files - jscc = env.Builder(generator=run_closure_compiler, suffix=".cc.js", src_suffix=".js") - env.Append(BUILDERS={"BuildJS": jscc}) - # Add helper method for adding libraries, externs, pre-js, post-js. env["JS_LIBS"] = [] env["JS_PRE"] = [] @@ -231,6 +215,13 @@ def configure(env: "SConsEnvironment"): # Add method for creating the final zip file env.AddMethod(create_template_zip, "CreateTemplateZip") + # Add method for bundling JS modules. + env.Append( + BUILDERS={ + "PackageJSModule": env.Builder(generator=package_js_module_generator), + } + ) + # Use TempFileMunge since some AR invocations are too long for cmd.exe. # Use POSIX-style paths, required with TempFileMunge. env["ARCOM_POSIX"] = env["ARCOM"].replace("$TARGET", "$TARGET.posix").replace("$SOURCES", "$SOURCES.posix") diff --git a/platform/web/doc_classes/EditorExportPlatformWeb.xml b/platform/web/doc_classes/EditorExportPlatformWeb.xml index d5a933a81e14..114bab42b23d 100644 --- a/platform/web/doc_classes/EditorExportPlatformWeb.xml +++ b/platform/web/doc_classes/EditorExportPlatformWeb.xml @@ -12,6 +12,27 @@ $DOCS_URL/tutorials/platform/web/index.html + + Determines the initial install mode. + - [b]Install Everything:[/b] Default behavior. Godot will download and install every exported resource before launching the game. + - [b]Only Install Required Resources:[/b] Async behavior. Use this setting to speed up considerably the loading time of your game. There are some important quirks to know about this mode, please read the note below. + [b]Note:[/b] About "Only Install Required Resources": + Godot will only download and install resources: + - Detected as essential to start the game by the export process. + - Those specified by filters or manually. + Developers need to call [method ResourceLoader.load_threaded_request] in order for Godot to download and install non-initial resources. Otherwise, Godot will fail to load the resource, and your game may not run as expected. + + + Filters to remove resources included by [member async/initial_install_only_required_resources/filters_to_include]. + (comma-separated, e.g. [code]*.json, *.txt, docs/*[/code]) + + + Filters to add resources to the initial install, even if they aren't detected as essential to start the game by the export process. + (comma-separated, e.g. [code]*.json, *.txt, docs/*[/code]) + + + Specify the resources to add to the initial install, even if they aren't detected as essential to start the game by the export process. + File path to the custom export template used for debug builds. If left empty, the default template is used. diff --git a/platform/web/emscripten_helpers.py b/platform/web/emscripten_helpers.py index 37029054f419..ba8cd382c826 100644 --- a/platform/web/emscripten_helpers.py +++ b/platform/web/emscripten_helpers.py @@ -1,49 +1,83 @@ import json import os +import typing -from SCons.Util import WhereIs +if typing.TYPE_CHECKING: + T = typing.TypeVar("T") + +from misc.scripts.copyright_headers import process_file_buffer as process_file_buffer_copyright_buffer from platform_methods import get_build_version -def run_closure_compiler(target, source, env, for_signature): - closure_bin = os.path.join( - os.path.dirname(WhereIs("emcc")), - "node_modules", - ".bin", - "google-closure-compiler", - ) - cmd = [WhereIs("node"), closure_bin] - cmd.extend(["--compilation_level", "ADVANCED_OPTIMIZATIONS"]) - for f in env["JSEXTERNS"]: - cmd.extend(["--externs", f.get_abspath()]) - for f in source: - cmd.extend(["--js", f.get_abspath()]) - cmd.extend(["--js_output_file", target[0].get_abspath()]) - return " ".join(cmd) +def ensure_list(value): # type: (typing.Union[T, typing.List[T]]) -> typing.List[T] + if not isinstance(value, list): + return [value] + return value def create_engine_file(env, target, source, externs, threads_enabled): - if env["use_closure_compiler"]: - return env.BuildJS(target, source, JSEXTERNS=externs) subst_dict = {"___GODOT_THREADS_ENABLED": "true" if threads_enabled else "false"} return env.Substfile(target=target, source=[env.File(s) for s in source], SUBST_DICT=subst_dict) +def package_js_module_generator(target, source, env, for_signature): + if for_signature: + return source + + target = target[0] + source = source[0] + + def get_wrapper(filename): + def wrapper(target, source, env): + return package_js_module_action_ensure_copyright_buffer(target, source, env, filename) + + return wrapper + + source_filename = os.path.basename(source.get_abspath()) + wrapper = get_wrapper(source_filename) + return env.Action(wrapper, "Ensuring copyright buffer.") + + +def package_js_module_action_ensure_copyright_buffer(target, source, env, filename): + target = target[0] + source = source[0] + + with open(source, mode="r", encoding="utf-8") as source_file: + with open(target, mode="w", encoding="utf-8") as target_file: + new_contents = process_file_buffer_copyright_buffer(filename, source_file) + CHUNK_SIZE = 1024 + while True: + chunk = new_contents.read(CHUNK_SIZE) + if not chunk: + break + target_file.write(chunk) + + def create_template_zip(env, js, wasm, side): binary_name = "godot.editor" if env.editor_build else "godot" zip_dir = env.Dir(env.GetTemplateZipPath()) + in_files = [ js, wasm, "#platform/web/js/libs/audio.worklet.js", "#platform/web/js/libs/audio.position.worklet.js", + env.PackageJSModule( + target="#bin/obj/platform/web/js/modules/utils/concurrency.js", + source="#platform/web/js/modules/utils/concurrency.js", + ), + env.PackageJSModule( + target="#bin/obj/platform/web/js/modules/utils/wait.js", source="#platform/web/js/modules/utils/wait.js" + ), ] out_files = [ zip_dir.File(binary_name + ".js"), zip_dir.File(binary_name + ".wasm"), zip_dir.File(binary_name + ".audio.worklet.js"), zip_dir.File(binary_name + ".audio.position.worklet.js"), + zip_dir.File(binary_name + ".utils.concurrency.js"), + zip_dir.File(binary_name + ".utils.wait.js"), ] # Dynamic linking (extensions) specific. if env["dlink_enabled"]: diff --git a/platform/web/eslint.config.cjs b/platform/web/eslint.config.cjs index 7bc1280396f0..ce457fdcd068 100644 --- a/platform/web/eslint.config.cjs +++ b/platform/web/eslint.config.cjs @@ -140,6 +140,16 @@ module.exports = [ }, }, + // module files (browser) + { + files: ['js/modules/**/*.js', 'platform/web/js/modules/**/*.js'], + languageOptions: { + globals: { + ...globals.browser, + }, + }, + }, + // libraries and modules (browser) { files: [ diff --git a/platform/web/export/export_plugin.cpp b/platform/web/export/export_plugin.cpp index f495a821a985..fda348b70f26 100644 --- a/platform/web/export/export_plugin.cpp +++ b/platform/web/export/export_plugin.cpp @@ -34,11 +34,17 @@ #include "run_icon_svg.gen.h" #include "core/config/project_settings.h" -#include "core/io/dir_access.h" +#include "core/crypto/hashing_context.h" +#include "core/extension/gdextension.h" +#include "core/io/config_file.h" +#include "core/io/json.h" +#include "core/io/resource_loader.h" #include "core/io/zip_io.h" -#include "core/os/os.h" +#include "core/string/string_builder.h" #include "editor/editor_string_names.h" #include "editor/export/editor_export.h" +#include "editor/export/editor_export_platform.h" +#include "editor/export/editor_export_platform_utils.h" #include "editor/file_system/editor_paths.h" #include "editor/import/resource_importer_texture_settings.h" #include "editor/settings/editor_settings.h" @@ -48,18 +54,334 @@ #include "modules/modules_enabled.gen.h" // IWYU pragma: keep. For mono. #include "modules/svg/image_loader_svg.h" +#include + +/** + * EditorExportPlatformWeb::ExportData::ResourceData + */ +uint32_t EditorExportPlatformWeb::ExportData::ResourceData::get_size() const { + uint32_t size = 0; + if (native_file.exists) { + size += native_file.size; + } + if (remap_file.exists) { + size += remap_file.size; + } + if (remapped_file.exists) { + size += remapped_file.size; + } + return size; +} + +Dictionary EditorExportPlatformWeb::ExportData::ResourceData::get_as_resource_dictionary() const { + Dictionary data; + Dictionary resources; + + if (native_file.exists) { + resources[native_file.resource_path] = native_file.get_as_dictionary(); + } + if (remap_file.exists) { + resources[remap_file.resource_path] = remap_file.get_as_dictionary(); + } + if (remapped_file.exists) { + resources[remapped_file.resource_path] = remapped_file.get_as_dictionary(); + } + data["files"] = resources; + data["totalSize"] = get_size(); + return data; +} + +String EditorExportPlatformWeb::ExportData::ResourceData::get_resource_path() const { + if (remap_file.exists) { + return remap_file.resource_path; + } + return native_file.resource_path; +} + +void EditorExportPlatformWeb::ExportData::ResourceData::flatten_dependencies(LocalVector *p_deps) const { + ERR_FAIL_NULL(p_deps); + + for (const ResourceData *dependency : dependencies) { + if (p_deps->has(dependency)) { + continue; + } + p_deps->push_back(dependency); + dependency->flatten_dependencies(p_deps); + } +} + +HashSet EditorExportPlatformWeb::ExportData::get_features_set() const { + List features_list; + + preset->get_platform()->get_platform_features(&features_list); + preset->get_platform()->get_preset_features(preset, &features_list); + + String custom = preset->get_custom_features(); + Vector custom_list = custom.split(","); + for (const String &custom_list_element : custom_list) { + String custom_list_element_stripped = custom_list_element.strip_edges(); + if (!custom_list_element_stripped.is_empty()) { + features_list.push_back(custom_list_element_stripped); + } + } + + HashSet features_set; + for (const String &feature : features_list) { + features_set.insert(feature); + } + return features_set; +} + +EditorExportPlatformWeb::ExportData::ResourceData *EditorExportPlatformWeb::ExportData::add_dependency(const String &p_path, const HashSet &p_features_set, Ref p_uid_cache, /* bool p_encrypt, */ Error *r_error) { + ResourceData *data = nullptr; + List::Element *data_iterator = nullptr; + + String remap_path; + bool has_suffix_import = false; + +#define SET_ERR(m_err) \ + if (r_error != nullptr) { \ + *r_error = (m_err); \ + } \ + ((void)0) + +#define _HANDLE_ERR_COND_V_MSG(m_cond, m_err, m_msg) \ + if (unlikely((m_cond))) { \ + SET_ERR(m_err); \ + if (data != nullptr && data_iterator != nullptr) { \ + dependencies.erase(data_iterator); \ + } \ + ERR_FAIL_V_MSG(nullptr, (m_msg)); \ + return nullptr; \ + } \ + ((void)0) + + _HANDLE_ERR_COND_V_MSG(p_path.is_empty(), ERR_INVALID_PARAMETER, "p_path.is_empty()"); + + if (dependencies_map.has(p_path)) { + SET_ERR(OK); + return dependencies_map[p_path]; + } + + data_iterator = dependencies.push_back({}); + data = &data_iterator->get(); + data->path = p_path; + update_file(&data->native_file, p_path); + + if (FileAccess::exists(data->native_file.absolute_path + SUFFIX_IMPORT)) { + has_suffix_import = true; + remap_path = data->native_file.resource_path + SUFFIX_IMPORT; + } else if (FileAccess::exists(data->native_file.absolute_path + SUFFIX_REMAP)) { + remap_path = data->native_file.resource_path + SUFFIX_REMAP; + } + + _HANDLE_ERR_COND_V_MSG(!data->native_file.exists && remap_path.is_empty(), ERR_FILE_NOT_FOUND, vformat(R"*("%s" doesn't exist, and there is no remap/import file.)*", data->native_file.absolute_path)); + + if (!remap_path.is_empty()) { + update_file(&data->remap_file, remap_path); + _HANDLE_ERR_COND_V_MSG(!data->remap_file.exists, ERR_FILE_NOT_FOUND, vformat(R"*("%s" doesn't exist)*", data->remap_file.absolute_path)); + + Error err; + Ref remap_file_access = FileAccess::open(data->remap_file.absolute_path, FileAccess::READ, &err); + _HANDLE_ERR_COND_V_MSG(err != OK, err, vformat(R"*(Error while opening "%s": %s)*", data->remap_file.absolute_path, error_names[err])); + + Ref remap_file; + remap_file.instantiate(); + remap_file->parse(remap_file_access->get_as_text()); + + const String PREFIX_PATH = "path."; + const String PATH_UID = "uid"; + + String remapped_path; + String uid_path; + Vector remap_section_keys = remap_file->get_section_keys("remap"); + for (const String &remap_section_key : remap_section_keys) { + bool found = false; + if (remap_section_key == PATH_UID) { + p_uid_cache->seek(0); + uid_path = ResourceUID::get_path_from_cache(p_uid_cache, remap_file->get_value("remap", remap_section_key)); + continue; + } + if (remap_section_key.begins_with(PREFIX_PATH)) { + String type = remap_section_key.trim_prefix(PREFIX_PATH); + if (p_features_set.has(type)) { + found = true; + } + } + if (remap_section_key == "path") { + found = true; + } + if (!found) { + continue; + } + remapped_path = remap_file->get_value("remap", remap_section_key); + break; + } + if (remapped_path.is_empty() && !uid_path.is_empty()) { + remapped_path = uid_path; + } + _HANDLE_ERR_COND_V_MSG(remapped_path.is_empty(), ERR_PARSE_ERROR, vformat(R"*(Could not find any remap path in %s file "%s")*", has_suffix_import ? "import" : "remap", data->remap_file.absolute_path)); + + update_file(&data->remapped_file, remapped_path); + } + + dependencies_map.insert(p_path, data); + + File *resource_file = nullptr; + if (data->native_file.exists && !data->remap_file.exists) { + resource_file = &data->native_file; + } else if (data->remapped_file.exists) { + resource_file = &data->remapped_file; + } + + if (resource_file != nullptr) { + List remapped_dependencies; + ResourceLoader::get_dependencies(resource_file->absolute_path, &remapped_dependencies); + for (const String &remapped_dependency : remapped_dependencies) { + Error error; + String remapped_dependency_path = EditorExportPlatformUtils::get_path_from_dependency(remapped_dependency); + ResourceData *dependency = add_dependency(remapped_dependency_path, p_features_set, p_uid_cache, &error); + _HANDLE_ERR_COND_V_MSG(error != OK, error, vformat(R"*(Error while processing remapped dependencies of "%s": couldn't add dependency of "%s".)*", resource_file->absolute_path, remapped_dependency_path)); + data->dependencies.push_back(dependency); + } + } + + SET_ERR(OK); + return data; + +#undef _HANDLE_ERR_COND_V_MSG +#undef SET_ERR +} + +void EditorExportPlatformWeb::ExportData::update_file(File *p_file, const String &p_resource_path) { + ERR_FAIL_NULL(p_file); + ERR_FAIL_COND(p_resource_path.is_empty()); + + p_file->resource_path = p_resource_path; + p_file->absolute_path = res_to_global(p_resource_path); + p_file->exists = FileAccess::exists(p_file->absolute_path); + if (!p_file->exists) { + return; + } + + p_file->size = FileAccess::get_size(p_file->absolute_path); + if (p_file->size == 0) { + return; + } + + Ref context_md5; + context_md5.instantiate(); + context_md5->start(HashingContext::HASH_MD5); + + Ref context_sha256; + context_sha256.instantiate(); + context_sha256->start(HashingContext::HASH_SHA256); + + const uint64_t CHUNK_SIZE = 1024; + Error error; + Ref file = FileAccess::open(p_file->absolute_path, FileAccess::READ, &error); + if (error != OK) { + ERR_FAIL_MSG(vformat(R"*(Error while opening "%s": %s)*", p_file->absolute_path, error_names[error])); + } + + while (file->get_position() < file->get_length()) { + uint64_t remaining = file->get_length() - file->get_position(); + PackedByteArray chunk = file->get_buffer(MIN(remaining, CHUNK_SIZE)); + context_md5->update(chunk); + context_sha256->update(chunk); + } + + PackedByteArray hash_md5 = context_md5->finish(); + PackedByteArray hash_sha256 = context_sha256->finish(); + + p_file->md5 = String::hex_encode_buffer(hash_md5.ptr(), hash_md5.size()); + p_file->sha256 = String::hex_encode_buffer(hash_sha256.ptr(), hash_sha256.size()); +} + +Dictionary EditorExportPlatformWeb::ExportData::get_deps_json_dictionary(const ResourceData *p_dependency) { + Dictionary deps; + Dictionary resources; + + // Resources. + deps["resources"] = resources; + resources[p_dependency->path] = p_dependency->get_as_resource_dictionary(); + + // Dependencies. + Dictionary deps_dependencies; + deps["dependencies"] = deps_dependencies; + + // Recursive dependency lambda. + std::function _l_add_deps_dependencies; + _l_add_deps_dependencies = [&](const ExportData::ResourceData *l_dependency) -> void { + resources[l_dependency->path] = l_dependency->get_as_resource_dictionary(); + LocalVector local_dependencies; + l_dependency->flatten_dependencies(&local_dependencies); + + PackedStringArray paths_array; + for (const ExportData::ResourceData *local_dependency : local_dependencies) { + if (local_dependency->path != l_dependency->path) { + paths_array.push_back(local_dependency->path); + } + if (!deps_dependencies.has(local_dependency->path)) { + // Prevent infinite recursion. + deps_dependencies[local_dependency->path] = {}; + _l_add_deps_dependencies(local_dependency); + } + } + paths_array.sort_custom(); + deps_dependencies[l_dependency->path] = paths_array; + }; + + // Loop through each dependencies to find their dependencies. + for (const ExportData::ResourceData *dependency : p_dependency->dependencies) { + _l_add_deps_dependencies(dependency); + } + + // Register the asked dependency itself. + _l_add_deps_dependencies(p_dependency); + + return deps; +} + +Error EditorExportPlatformWeb::ExportData::save_deps_json(const ResourceData *p_dependency) { + Dictionary deps = get_deps_json_dictionary(p_dependency); + String resource_path = p_dependency->get_resource_path(); + if (resource_path == p_dependency->remap_file.resource_path) { + if (resource_path.ends_with(SUFFIX_REMAP)) { + resource_path = resource_path.trim_suffix(SUFFIX_REMAP); + } else { + resource_path = resource_path.trim_suffix(SUFFIX_IMPORT); + } + } + + String deps_json_file_path = res_to_global(resource_path) + ".deps.json"; + Error error; + Ref deps_json_file = FileAccess::open(deps_json_file_path, FileAccess::WRITE, &error); + if (error != OK) { + ERR_PRINT(vformat(R"*(Could not write to "%s".)*", deps_json_file_path)); + return error; + } + deps_json_file->store_string(JSON::stringify(deps, String(" ").repeat(2))); + + return OK; +} + +/** + * EditorExportPlatformWeb + */ + Error EditorExportPlatformWeb::_extract_template(const String &p_template, const String &p_dir, const String &p_name, bool pwa) { Ref io_fa; zlib_filefunc_def io = zipio_create_io(&io_fa); unzFile pkg = unzOpen2(p_template.utf8().get_data(), &io); if (!pkg) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not open template for export: \"%s\"."), p_template)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not open template for export: \"%s\"."), p_template)); return ERR_FILE_NOT_FOUND; } if (unzGoToFirstFile(pkg) != UNZ_OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Invalid export template: \"%s\"."), p_template)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Invalid export template: \"%s\"."), p_template)); unzClose(pkg); return ERR_FILE_CORRUPT; } @@ -93,7 +415,7 @@ Error EditorExportPlatformWeb::_extract_template(const String &p_template, const String dst = p_dir.path_join(file.replace("godot", p_name)); Ref f = FileAccess::open(dst, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not write file: \"%s\"."), dst)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not write file: \"%s\"."), dst)); unzClose(pkg); return ERR_FILE_CANT_WRITE; } @@ -107,7 +429,7 @@ Error EditorExportPlatformWeb::_extract_template(const String &p_template, const Error EditorExportPlatformWeb::_write_or_error(const uint8_t *p_content, int p_size, String p_path) { Ref f = FileAccess::open(p_path, FileAccess::WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), p_path)); return ERR_FILE_CANT_WRITE; } f->store_buffer(p_content, p_size); @@ -132,14 +454,14 @@ void EditorExportPlatformWeb::_replace_strings(const HashMap &p_ } } -void EditorExportPlatformWeb::_fix_html(Vector &p_html, const Ref &p_preset, const String &p_name, bool p_debug, BitField p_flags, const Vector p_shared_objects, const Dictionary &p_file_sizes) { +void EditorExportPlatformWeb::_fix_html(Vector &p_html, const Ref &p_preset, const String &p_name, bool p_debug, BitField p_flags, const Vector p_shared_objects, const Dictionary &p_file_sizes, const Dictionary &p_async_pck_data) { // Engine.js config Dictionary config; Array libs; for (int i = 0; i < p_shared_objects.size(); i++) { libs.push_back(p_shared_objects[i].path.get_file()); } - Vector flags = gen_export_flags(p_flags & (~DEBUG_FLAG_DUMB_CLIENT)); + Vector flags = gen_export_flags(p_flags & (~EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT)); Array args; for (int i = 0; i < flags.size(); i++) { args.push_back(flags[i]); @@ -156,6 +478,17 @@ void EditorExportPlatformWeb::_fix_html(Vector &p_html, const Refget("threads/godot_pool_size"); config["emscriptenPoolSize"] = p_preset->get("threads/emscripten_pool_size"); + AsyncInitialInstallMode async_initial_install_mode = (AsyncInitialInstallMode)(int)p_preset->get("async/initial_install_mode"); + switch (async_initial_install_mode) { + case ASYNC_INITIAL_INSTALL_MODE_EVERYTHING: { + config["mainPack"] = p_name + ".pck"; + } break; + case ASYNC_INITIAL_INSTALL_MODE_ONLY_REQUIRED_RESOURCES: { + config["mainPack"] = p_name + ".asyncpck"; + config["asyncPckData"] = p_async_pck_data; + } break; + } + String head_include; if (p_preset->get("html/export_icon")) { head_include += "\n"; @@ -203,7 +536,7 @@ Error EditorExportPlatformWeb::_add_manifest_icon(const Ref Error err = OK; icon = _load_icon_or_splash_image(p_icon, &err); if (err != OK || icon.is_null() || icon->is_empty()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Icon Creation"), vformat(TTR("Could not read file: \"%s\"."), p_icon)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Icon Creation"), vformat(TTR("Could not read file: \"%s\"."), p_icon)); return err; } if (icon->get_width() != p_size || icon->get_height() != p_size) { @@ -215,7 +548,7 @@ Error EditorExportPlatformWeb::_add_manifest_icon(const Ref } const Error err = icon->save_png(icon_dest); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Icon Creation"), vformat(TTR("Could not write file: \"%s\"."), icon_dest)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Icon Creation"), vformat(TTR("Could not write file: \"%s\"."), icon_dest)); return err; } Dictionary icon_dict; @@ -261,8 +594,19 @@ Error EditorExportPlatformWeb::_build_pwa(const Ref &p_prese // Heavy files that are cached on demand. Array opt_cache_files = { name + ".wasm", - name + ".pck" }; + + AsyncInitialInstallMode async_initial_install_mode = (AsyncInitialInstallMode)(int)p_preset->get("async/initial_install_mode"); + switch (async_initial_install_mode) { + case ASYNC_INITIAL_INSTALL_MODE_EVERYTHING: { + opt_cache_files.push_back(name + ".pck"); + } break; + + case ASYNC_INITIAL_INSTALL_MODE_ONLY_REQUIRED_RESOURCES: { + // TODO: Add AsyncPCK contents to the cache. + } break; + } + if (extensions) { opt_cache_files.push_back(name + ".side.wasm"); for (int i = 0; i < p_shared_objects.size(); i++) { @@ -276,7 +620,7 @@ Error EditorExportPlatformWeb::_build_pwa(const Ref &p_prese { Ref f = FileAccess::open(sw_path, FileAccess::READ); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PWA"), vformat(TTR("Could not read file: \"%s\"."), sw_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PWA"), vformat(TTR("Could not read file: \"%s\"."), sw_path)); return ERR_FILE_CANT_READ; } sw.resize(f->get_length()); @@ -296,7 +640,7 @@ Error EditorExportPlatformWeb::_build_pwa(const Ref &p_prese const String offline_dest = dir.path_join(name + ".offline.html"); err = da->copy(ProjectSettings::get_singleton()->globalize_path(offline_page), offline_dest); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PWA"), vformat(TTR("Could not read file: \"%s\"."), offline_dest)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PWA"), vformat(TTR("Could not read file: \"%s\"."), offline_dest)); return err; } } @@ -371,6 +715,11 @@ void EditorExportPlatformWeb::get_export_options(List *r_options) r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "custom_template/debug", PROPERTY_HINT_GLOBAL_FILE, "*.zip"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "custom_template/release", PROPERTY_HINT_GLOBAL_FILE, "*.zip"), "")); + r_options->push_back(ExportOption(PropertyInfo(Variant::INT, "async/initial_install_mode", PROPERTY_HINT_ENUM, "Install Everything,Only Install Required Resources"), 0, true)); + r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "async/initial_install_only_required_resources/filters_to_include"), "")); + r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "async/initial_install_only_required_resources/filters_to_exclude"), "")); + r_options->push_back(ExportOption(PropertyInfo(Variant::PACKED_STRING_ARRAY, "async/initial_install_only_required_resources/forced_resources", PROPERTY_HINT_ARRAY_TYPE, MAKE_FILE_ARRAY_TYPE_HINT("*")), PackedStringArray())); + r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "variant/extensions_support"), false)); // GDExtension support. r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "variant/thread_support"), false, true)); // Thread support (i.e. run with or without COEP/COOP headers). r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "vram_texture_compression/for_desktop"), true)); // S3TC @@ -397,6 +746,10 @@ void EditorExportPlatformWeb::get_export_options(List *r_options) } bool EditorExportPlatformWeb::get_export_option_visibility(const EditorExportPreset *p_preset, const String &p_option) const { + if (p_option == "async/initial_install_only_required_resources/forced_resources" || p_option == "async/initial_install_only_required_resources/filters_to_include" || p_option == "async/initial_install_only_required_resources/filters_to_exclude") { + return (int)p_preset->get("async/initial_install_mode") != ASYNC_INITIAL_INSTALL_MODE_EVERYTHING; + } + bool advanced_options_enabled = p_preset->are_advanced_options_enabled(); if (p_option == "custom_template/debug" || p_option == "custom_template/release") { return advanced_options_enabled; @@ -430,6 +783,13 @@ bool EditorExportPlatformWeb::has_valid_export_configuration(const Refget("async/initial_install_mode") != AsyncInitialInstallMode::ASYNC_INITIAL_INSTALL_MODE_EVERYTHING) { + if (String(EditorExportPlatformUtils::get_project_setting(p_preset, "application/run/main_scene")).is_empty()) { + err += TTR("No main scene has been set. The main scene must be set for the web platform in order to preload the minimal files.") + "\n"; + } + } + bool valid = false; bool extensions = (bool)p_preset->get("variant/extensions_support"); bool thread_support = (bool)p_preset->get("variant/thread_support"); @@ -496,12 +856,20 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p const bool export_icon = p_preset->get("html/export_icon"); const bool pwa = p_preset->get("progressive_web_app/enabled"); - const String base_dir = p_path.get_base_dir(); - const String base_path = p_path.get_basename(); - const String base_name = p_path.get_file().get_basename(); + String path = p_path; + if (!path.is_absolute_path()) { + if (!path.begins_with("res://")) { + path = "res://" + path; + } + path = ProjectSettings::get_singleton()->globalize_path(path); + } + + const String base_dir = path.get_base_dir() + "/"; + const String base_path = path.get_basename(); + const String base_name = path.get_file().get_basename(); if (!DirAccess::exists(base_dir)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Target folder does not exist or is inaccessible: \"%s\""), base_dir)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Target folder does not exist or is inaccessible: \"%s\""), base_dir)); return ERR_FILE_BAD_PATH; } @@ -515,29 +883,354 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p } if (!template_path.is_empty() && !FileAccess::exists(template_path)) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Template file not found: \"%s\"."), template_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Template file not found: \"%s\"."), template_path)); return ERR_FILE_NOT_FOUND; } - // Export pck and shared objects + Error error; + + // Export pck and shared objects. Vector shared_objects; - String pck_path = base_path + ".pck"; - Error error = save_pack(p_preset, p_debug, pck_path, &shared_objects); - if (error != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), pck_path)); - return error; - } + String pck_path; - { - Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); - for (int i = 0; i < shared_objects.size(); i++) { - String dst = base_dir.path_join(shared_objects[i].path.get_file()); - error = da->copy(shared_objects[i].path, dst); + // Async PCK related. + Dictionary async_pck_data; + // Parse generated file sizes (pck and wasm, to help show a meaningful loading bar). + Dictionary file_sizes; + + AsyncInitialInstallMode async_initial_install_mode = (AsyncInitialInstallMode)(int)p_preset->get("async/initial_install_mode"); + switch (async_initial_install_mode) { + case ASYNC_INITIAL_INSTALL_MODE_EVERYTHING: { + pck_path = base_path + ".pck"; + + error = save_pack(p_preset, p_debug, pck_path, &shared_objects); if (error != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), shared_objects[i].path.get_file())); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), pck_path)); return error; } - } + + { + Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + for (int i = 0; i < shared_objects.size(); i++) { + String dst = base_dir.path_join(shared_objects[i].path.get_file()); + error = da->copy(shared_objects[i].path, dst); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), shared_objects[i].path.get_file())); + return error; + } + } + } + + // Updating file sizes. + Ref f = FileAccess::open(pck_path, FileAccess::READ); + if (f.is_valid()) { + file_sizes[pck_path.get_file()] = (uint64_t)f->get_length(); + } + + } break; + + case ASYNC_INITIAL_INSTALL_MODE_ONLY_REQUIRED_RESOURCES: { + pck_path = base_path + ".asyncpck"; + + if (DirAccess::dir_exists_absolute(pck_path)) { + Ref pck_path_access = DirAccess::create_for_path(pck_path); + pck_path_access->change_dir(pck_path); + pck_path_access->erase_contents_recursive(); + pck_path_access->change_dir(".."); + pck_path_access->remove_absolute(pck_path); + } + + ExportData export_data; + export_data.assets_directory = pck_path.path_join("assets"); + export_data.libraries_directory = pck_path.path_join("libraries"); + export_data.pack_data.path = "assets.sparsepck"; + export_data.pack_data.use_sparse_pck = true; + export_data.preset = p_preset; + + HashSet features_set = export_data.get_features_set(); + + error = export_project_files(p_preset, p_debug, &EditorExportPlatformWeb::_rename_and_store_file_in_async_pck, nullptr, &export_data); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write async pck: \"%s\"."), pck_path)); + return error; + } + + PackedByteArray encoded_data; + error = _generate_sparse_pck_metadata(p_preset, export_data.pack_data, encoded_data, true); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not encode contents of async pck: \"%s\"."), pck_path)); + return error; + } + + error = EditorExportPlatformUtils::store_file_at_path(export_data.assets_directory.path_join("assets.sparsepck"), encoded_data); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not store contents of async pck: \"%s\"."), pck_path)); + return error; + } + + // Ensures that Godot doesn't start to load content from an AsyncPCK as resources. + error = EditorExportPlatformUtils::store_file_at_path(pck_path.path_join(".gdignore"), PackedByteArray()); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not store contents of async pck: \"%s\"."), pck_path)); + return error; + } + + { + Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + for (int i = 0; i < shared_objects.size(); i++) { + String dst = export_data.libraries_directory.path_join(shared_objects[i].path.get_file()); + error = da->copy(shared_objects[i].path, dst); + if (error != OK) { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), shared_objects[i].path.get_file())); + return error; + } + } + } + + { + Dictionary async_pck_data_directories; + Dictionary async_pck_data_initial_install; + + async_pck_data["directories"] = async_pck_data_directories; + async_pck_data["initialLoad"] = async_pck_data_initial_install; + + const String PREFIX_ASSETS_DIRECTORY = export_data.assets_directory + "/"; + + const String SECTION_REMAP = "remap"; + const String SECTION_KEY_PATH = "path"; + const String SECTION_KEY_PATH_PREFIX = "path."; + + const String PATH_GODOT_DIR = ".godot/"; + + HashSet exported_files; + HashSet internal_files; + HashSet standalone_files; + HashSet remap_files; + HashSet import_files; + + uint64_t total_size = 0; + + Ref uid_cache = FileAccess::open(export_data.res_to_global(PATH_GODOT_UID_CACHE), FileAccess::READ, &error); + if (error != OK) { + return error; + } + + for (const String &exported_file : export_data.exported_files) { + String local_exported_file = PREFIX_RES + exported_file.trim_prefix(PREFIX_ASSETS_DIRECTORY).simplify_path(); + exported_files.insert(local_exported_file); + } + + for (const String &exported_file : exported_files) { + if (exported_file.begins_with(PATH_GODOT_DIR) || exported_file == PATH_PROJECT_BINARY || exported_file == PATH_ASSETS_SPARSEPCK) { + internal_files.insert(exported_file); + continue; + } + + if (exported_file.ends_with(SUFFIX_REMAP)) { + remap_files.insert(exported_file); + continue; + } else if (exported_file.ends_with(SUFFIX_IMPORT)) { + import_files.insert(exported_file); + continue; + } + + standalone_files.insert(exported_file); + } + + for (const String &internal_file : internal_files) { + export_data.add_dependency(internal_file, features_set, uid_cache, &error); + if (error != OK) { + return error; + } + } + + for (const String &remap_file : remap_files) { + export_data.add_dependency(remap_file.trim_suffix(SUFFIX_REMAP), features_set, uid_cache, &error); + if (error != OK) { + return error; + } + } + + for (const String &import_file : import_files) { + export_data.add_dependency(import_file.trim_suffix(SUFFIX_IMPORT), features_set, uid_cache, &error); + if (error != OK) { + return error; + } + } + + for (const String &standalone_file : standalone_files) { + export_data.add_dependency(standalone_file, features_set, uid_cache, &error); + if (error != OK) { + return error; + } + } + + for (ExportData::ResourceData &dependency : export_data.dependencies) { + if (dependency.path.begins_with(PREFIX_RES + PATH_GODOT_DIR)) { + continue; + } + error = export_data.save_deps_json(&dependency); + if (error != OK) { + return error; + } + } + + HashSet initial_install_dependencies; + { + Vector initial_install_include_filters; + Vector initial_install_exclude_filters; + + Vector initial_install_include_split = String(p_preset->get("async/initial_install_only_required_resources/filters_to_include")).split(","); + for (const String &initial_install_include_element : initial_install_include_split) { + String initial_install_include_filter = initial_install_include_element.strip_edges(); + if (initial_install_include_filter.is_empty()) { + continue; + } + initial_install_include_filters.push_back(initial_install_include_filter); + } + + Vector initial_install_exclude_split = String(p_preset->get("async/initial_install_only_required_resources/filters_to_exclude")).split(","); + for (const String &initial_install_exclude_element : initial_install_exclude_split) { + String initial_install_exclude_filter = initial_install_exclude_element.strip_edges(); + if (initial_install_exclude_filter.is_empty()) { + continue; + } + initial_install_exclude_filters.push_back(initial_install_exclude_filter); + } + + if (initial_install_include_filters.size() > 0) { + for (const ExportData::ResourceData &dependency : export_data.dependencies) { + const String &dependency_path = dependency.path; + bool add_as_initial_install = false; + for (const String &in_filter : initial_install_include_filters) { + if (dependency_path.matchn(in_filter) || dependency_path.trim_prefix(PREFIX_RES).matchn(in_filter)) { + add_as_initial_install = true; + break; + } + } + + for (const String &ex_filter : initial_install_exclude_filters) { + if (dependency_path.matchn(ex_filter) || dependency_path.trim_prefix(PREFIX_RES).matchn(ex_filter)) { + add_as_initial_install = false; + break; + } + } + + if (add_as_initial_install) { + initial_install_dependencies.insert(&dependency); + } + } + } + } + + HashSet mandatory_initial_install_files = _get_mandatory_initial_install_files(p_preset); + for (const String &mandatory_initial_install_file : mandatory_initial_install_files) { + export_data.add_dependency(mandatory_initial_install_file, features_set, uid_cache); + } + for (const String &mandatory_initial_install_file : mandatory_initial_install_files) { + ExportData::ResourceData *mandatory_resource_data = export_data.dependencies_map[mandatory_initial_install_file]; + initial_install_dependencies.insert(mandatory_resource_data); + LocalVector mandatory_resource_data_dependencies; + mandatory_resource_data->flatten_dependencies(&mandatory_resource_data_dependencies); + for (const ExportData::ResourceData *mandatory_resource_data_dependency : mandatory_resource_data_dependencies) { + initial_install_dependencies.insert(mandatory_resource_data_dependency); + } + } + + { + PackedStringArray initial_install_paths; + HashSet initial_install_assets; + for (const ExportData::ResourceData *dependency : initial_install_dependencies) { + if (dependency->remap_file.exists || dependency->native_file.exists) { + initial_install_assets.insert(dependency); + } + } + + LocalVector initial_install_assets_data; + for (const ExportData::ResourceData *initial_install_asset : initial_install_assets) { + initial_install_assets_data.push_back(initial_install_asset); + } + + _add_resource_data_tree_message(initial_install_assets_data, "Files that will be initially loaded (sorted in alphabetical order):", true, false); + _add_resource_data_tree_message(initial_install_assets_data, "Files that will be initially loaded (sorted by size):", false, true); + + uint64_t initial_install_assets_size = initial_install_assets_data.size(); + for (uint64_t i = 0; i < initial_install_assets_size; i++) { + const ExportData::ResourceData *initial_install_asset = initial_install_assets_data[i]; + + uint64_t asset_size = 0; + if (initial_install_asset->remap_file.exists) { + asset_size += initial_install_asset->remap_file.size; + asset_size += initial_install_asset->remapped_file.size; + } else if (initial_install_asset->native_file.exists) { + asset_size += initial_install_asset->native_file.size; + } else { + ERR_FAIL_V(ERR_BUG); + } + + total_size += asset_size; + } + + StringBuilder log_entry_builder; + log_entry_builder.append("If some files seem to be missing from this list, be sure to edit \"async/initial_install_only_required_resources/*\" in the preset settings.\n"); + log_entry_builder.append("For files not in this list, you will need to call `ResourceLoader.load_threaded_request()` beforehand.\n"); + log_entry_builder.append("\n"); + log_entry_builder.append(vformat("Total initial load size: %s", String::humanize_size(total_size))); + + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Initial load"), log_entry_builder.as_string()); + } + + { + async_pck_data_directories["assets"] = export_data.assets_directory.trim_prefix(base_dir); + async_pck_data_directories["libraries"] = export_data.libraries_directory.trim_prefix(base_dir); + } + + for (const ExportData::ResourceData *dependency : initial_install_dependencies) { + Dictionary dependency_dict; + Array dependency_dependencies; + Array dependency_files; + + dependency_dict["files"] = dependency_files; + + for (const ExportData::ResourceData *dependency_dependency : dependency->dependencies) { + dependency_dependencies.push_back(dependency_dependency->path); + } + + if (dependency_dependencies.size() > 0) { + dependency_dict["dependencies"] = dependency_dependencies; + } + + if (dependency->native_file.exists) { + dependency_files.push_back(dependency->native_file.resource_path); + } + if (dependency->remap_file.exists) { + dependency_files.push_back(dependency->remap_file.resource_path); + } + if (dependency->remapped_file.exists) { + dependency_files.push_back(dependency->remapped_file.resource_path); + } + + async_pck_data_initial_install[dependency->path] = dependency_dict; + } + + for (ExportData::ResourceData &dependency : export_data.dependencies) { + if (dependency.native_file.exists) { + file_sizes[dependency.native_file.absolute_path.trim_prefix(base_dir)] = dependency.native_file.size; + } + if (dependency.remap_file.exists) { + file_sizes[dependency.remap_file.absolute_path.trim_prefix(base_dir)] = dependency.remap_file.size; + } + if (dependency.remapped_file.exists) { + file_sizes[dependency.remapped_file.absolute_path.trim_prefix(base_dir)] = dependency.remapped_file.size; + } + } + } + } break; + + default: { + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR(R"*(Invalid `async/initial_install_mode` value: %s)*"), async_initial_install_mode)); + return ERR_INVALID_PARAMETER; + } break; } // Extract templates. @@ -547,13 +1240,7 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p return error; } - // Parse generated file sizes (pck and wasm, to help show a meaningful loading bar). - Dictionary file_sizes; - Ref f = FileAccess::open(pck_path, FileAccess::READ); - if (f.is_valid()) { - file_sizes[pck_path.get_file()] = (uint64_t)f->get_length(); - } - f = FileAccess::open(base_path + ".wasm", FileAccess::READ); + Ref f = FileAccess::open(base_path + ".wasm", FileAccess::READ); if (f.is_valid()) { file_sizes[base_name + ".wasm"] = (uint64_t)f->get_length(); } @@ -563,7 +1250,7 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p Vector html; f = FileAccess::open(html_path, FileAccess::READ); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not read HTML shell: \"%s\"."), html_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not read HTML shell: \"%s\"."), html_path)); return ERR_FILE_CANT_READ; } html.resize(f->get_length()); @@ -571,8 +1258,8 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p f.unref(); // close file. // Generate HTML file with replaced strings. - _fix_html(html, p_preset, base_name, p_debug, p_flags, shared_objects, file_sizes); - Error err = _write_or_error(html.ptr(), html.size(), p_path); + _fix_html(html, p_preset, base_name, p_debug, p_flags, shared_objects, file_sizes, async_pck_data); + Error err = _write_or_error(html.ptr(), html.size(), path); if (err != OK) { // Message is supplied by the subroutine method. return err; @@ -583,7 +1270,7 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p Ref splash = _get_project_splash(p_preset); const String splash_png_path = base_path + ".png"; if (splash->save_png(splash_png_path) != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), splash_png_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), splash_png_path)); return ERR_FILE_CANT_WRITE; } @@ -593,20 +1280,20 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p Ref favicon = _get_project_icon(p_preset); const String favicon_png_path = base_path + ".icon.png"; if (favicon->save_png(favicon_png_path) != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), favicon_png_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), favicon_png_path)); return ERR_FILE_CANT_WRITE; } favicon->resize(180, 180); const String apple_icon_png_path = base_path + ".apple-touch-icon.png"; if (favicon->save_png(apple_icon_png_path) != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), apple_icon_png_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), apple_icon_png_path)); return ERR_FILE_CANT_WRITE; } } // Generate the PWA worker and manifest if (pwa) { - err = _build_pwa(p_preset, p_path, shared_objects); + err = _build_pwa(p_preset, path, shared_objects); if (err != OK) { // Message is supplied by the subroutine method. return err; @@ -616,6 +1303,54 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p return OK; } +void EditorExportPlatformWeb::_add_resource_data_tree_message(LocalVector &p_resource_data_entries, const String &p_context, bool p_sort_with_file_no_case_comparator, bool p_sort_with_size_comparator) { + if (p_sort_with_file_no_case_comparator) { + p_resource_data_entries.sort_custom(); + } + if (p_sort_with_size_comparator) { + p_resource_data_entries.sort_custom(); + } + + StringBuilder log_entry_builder; + const String new_line_char = "\n"; + log_entry_builder.append(vformat("%s\n", p_context)); + + uint64_t initial_install_assets_size = p_resource_data_entries.size(); + for (uint64_t i = 0; i < initial_install_assets_size; i++) { + const ExportData::ResourceData *initial_install_asset = p_resource_data_entries[i]; + + uint64_t asset_size = 0; + if (initial_install_asset->remap_file.exists) { + asset_size += initial_install_asset->remap_file.size; + asset_size += initial_install_asset->remapped_file.size; + } else if (initial_install_asset->native_file.exists) { + asset_size += initial_install_asset->native_file.size; + } else { + ERR_FAIL(); + } + + String fork_char = i < initial_install_assets_size - 1 + ? U"├" + : U"â””"; + String parent_tree_line = i < initial_install_assets_size - 1 + ? U"|" + : U" "; + + log_entry_builder.append(vformat(UR"*(%s── 📦 "%s" [%s]%s)*", fork_char, initial_install_asset->path, String::humanize_size(asset_size), new_line_char)); + + if (initial_install_asset->remap_file.exists) { + log_entry_builder.append(vformat(UR"*(%s ├ 📤 "%s" [%s]%s)*", parent_tree_line, initial_install_asset->remap_file.resource_path, String::humanize_size(initial_install_asset->remap_file.size), new_line_char)); + log_entry_builder.append(vformat(UR"*(%s â”” 📤 "%s" [%s]%s)*", parent_tree_line, initial_install_asset->remapped_file.resource_path, String::humanize_size(initial_install_asset->remapped_file.size), new_line_char)); + } else if (initial_install_asset->native_file.exists) { + log_entry_builder.append(vformat(UR"*(%s â”” 📤 "%s" [%s]%s)*", parent_tree_line, initial_install_asset->native_file.resource_path, String::humanize_size(initial_install_asset->native_file.size), new_line_char)); + } + } + + log_entry_builder.append("\n====================\n"); + + add_message(EditorExportPlatformData::EXPORT_MESSAGE_INFO, TTR("Initial load"), log_entry_builder.as_string()); +} + bool EditorExportPlatformWeb::poll_export() { Ref preset = EditorExport::get_singleton()->get_runnable_preset_for_platform(this); @@ -788,7 +1523,7 @@ Error EditorExportPlatformWeb::run(const Ref &p_preset, int switch (p_option) { // Run in Browser. case 0: { - Error err = _export_project(p_preset, p_debug_flags); + Error err = _run_export_project(p_preset, p_debug_flags); if (err != OK) { return err; } @@ -801,7 +1536,7 @@ Error EditorExportPlatformWeb::run(const Ref &p_preset, int // Start HTTP Server. case 1: { - Error err = _export_project(p_preset, p_debug_flags); + Error err = _run_export_project(p_preset, p_debug_flags); if (err != OK) { return err; } @@ -818,7 +1553,7 @@ Error EditorExportPlatformWeb::run(const Ref &p_preset, int switch (p_option) { // Run in Browser. case 0: { - Error err = _export_project(p_preset, p_debug_flags); + Error err = _run_export_project(p_preset, p_debug_flags); if (err != OK) { return err; } @@ -827,7 +1562,7 @@ Error EditorExportPlatformWeb::run(const Ref &p_preset, int // Re-export Project. case 1: { - return _export_project(p_preset, p_debug_flags); + return _run_export_project(p_preset, p_debug_flags); } break; // Stop HTTP Server. @@ -845,13 +1580,13 @@ Error EditorExportPlatformWeb::run(const Ref &p_preset, int return FAILED; } -Error EditorExportPlatformWeb::_export_project(const Ref &p_preset, int p_debug_flags) { +Error EditorExportPlatformWeb::_run_export_project(const Ref &p_preset, int p_debug_flags) { const String dest = EditorPaths::get_singleton()->get_temp_dir().path_join("web"); Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); if (!da->dir_exists(dest)) { Error err = da->make_dir_recursive(dest); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Could not create HTTP server directory: %s."), dest)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Could not create HTTP server directory: %s."), dest)); return err; } } @@ -866,7 +1601,7 @@ Error EditorExportPlatformWeb::_export_project(const Ref &p_ DirAccess::remove_file_or_error(basepath + ".audio.worklet.js"); DirAccess::remove_file_or_error(basepath + ".audio.position.worklet.js"); DirAccess::remove_file_or_error(basepath + ".service.worker.js"); - DirAccess::remove_file_or_error(basepath + ".pck"); + DirAccess::remove_file_or_error(basepath + ".asyncpck"); DirAccess::remove_file_or_error(basepath + ".png"); DirAccess::remove_file_or_error(basepath + ".side.wasm"); DirAccess::remove_file_or_error(basepath + ".wasm"); @@ -899,7 +1634,7 @@ Error EditorExportPlatformWeb::_start_server(const String &p_bind_host, const ui server->stop(); Error err = server->listen(p_bind_port, bind_ip, p_use_tls, tls_key, tls_cert); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Error starting HTTP server: %d."), err)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Run"), vformat(TTR("Error starting HTTP server: %d."), err)); } return err; } @@ -909,33 +1644,126 @@ Error EditorExportPlatformWeb::_stop_server() { return OK; } -Ref EditorExportPlatformWeb::get_run_icon() const { - return run_icon; +Error EditorExportPlatformWeb::_rename_and_store_file_in_async_pck(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const Vector &p_key, uint64_t p_seed, bool p_delta) { + ExportData *export_data = static_cast(p_userdata); + const String simplified_path = EditorExportPlatform::simplify_path(p_path); + + Vector encoded_data; + EditorExportPlatformData::SavedData saved_data; + Error err = EditorExportPlatformUtils::store_temp_file(simplified_path, p_data, p_enc_in_filters, p_enc_ex_filters, p_key, p_seed, p_delta, encoded_data, saved_data); + if (err != OK) { + return err; + } + + const String target_path = export_data->assets_directory.path_join(simplified_path.trim_prefix("res://")); + export_data->exported_files.insert(target_path); + err = EditorExportPlatformUtils::store_file_at_path(target_path, encoded_data); + + export_data->pack_data.file_ofs.push_back(saved_data); + + return OK; } -void EditorExportPlatformWeb::initialize() { - if (EditorNode::get_singleton()) { - server.instantiate(); +HashSet EditorExportPlatformWeb::_get_mandatory_initial_install_files(const Ref &p_preset) { + HashSet mandatory_initial_install_files; - Ref img = memnew(Image); - const bool upsample = !Math::is_equal_approx(Math::round(EDSCALE), EDSCALE); + { + // Main scene. + mandatory_initial_install_files.insert( + EditorExportPlatformUtils::get_path_from_dependency( + EditorExportPlatformUtils::get_project_setting(p_preset, "application/run/main_scene"))); + } - ImageLoaderSVG::create_image_from_string(img, _web_logo_svg, EDSCALE, upsample, false); - logo = ImageTexture::create_from_image(img); + { + // Translation files. + PackedStringArray translations = EditorExportPlatformUtils::get_project_setting(p_preset, "internationalization/locale/translations"); + for (const String &translation : translations) { + mandatory_initial_install_files.insert(EditorExportPlatformUtils::get_path_from_dependency(translation)); + } + } - ImageLoaderSVG::create_image_from_string(img, _web_run_icon_svg, EDSCALE, upsample, false); - run_icon = ImageTexture::create_from_image(img); + { + // Autoload files. + const HashMap &autoload_list = ProjectSettings::get_singleton()->get_autoload_list(); + for (const KeyValue &key_value : autoload_list) { + mandatory_initial_install_files.insert( + EditorExportPlatformUtils::get_path_from_dependency(key_value.value.path)); + } + } - Ref theme = EditorNode::get_singleton()->get_editor_theme(); - if (theme.is_valid()) { - stop_icon = theme->get_icon(SNAME("Stop"), EditorStringName(EditorIcons)); - restart_icon = theme->get_icon(SNAME("Reload"), EditorStringName(EditorIcons)); - } else { - stop_icon.instantiate(); - restart_icon.instantiate(); + { + // Global class files. + LocalVector global_classes; + ScriptServer::get_global_class_list(global_classes); + + for (const StringName &global_class : global_classes) { + String global_class_path = ScriptServer::get_global_class_path(global_class); + mandatory_initial_install_files.insert( + EditorExportPlatformUtils::get_path_from_dependency(global_class_path)); } } + + { + // Single files. + auto _l_add_project_setting_if_file_exists = [&](const String &l_project_setting) -> void { + const String project_setting_file = ResourceUID::ensure_path(EditorExportPlatformUtils::get_project_setting(p_preset, l_project_setting)); + String path = EditorExportPlatformUtils::get_path_from_dependency(project_setting_file); + if (FileAccess::exists(path)) { + mandatory_initial_install_files.insert(path); + } + }; + + // Icon path. + _l_add_project_setting_if_file_exists("application/config/icon"); + // Default bus layout path. + _l_add_project_setting_if_file_exists("audio/buses/default_bus_layout"); + // Certificate bundle override. + _l_add_project_setting_if_file_exists("network/tls/certificate_bundle_override"); + // Default environment. + _l_add_project_setting_if_file_exists("rendering/environment/defaults/default_environment"); + // Default XR action map. + _l_add_project_setting_if_file_exists("xr/openxr/default_action_map"); + } + + { + // Export-related files. + mandatory_initial_install_files.insert(PATH_PROJECT_BINARY); + mandatory_initial_install_files.insert(PATH_ASSETS_SPARSEPCK); + mandatory_initial_install_files.insert(PATH_GODOT_UID_CACHE); + mandatory_initial_install_files.insert(PATH_GODOT_GLOBAL_SCRIPT_CLASS_CACHE); + } + + return mandatory_initial_install_files; } -EditorExportPlatformWeb::~EditorExportPlatformWeb() { +Ref EditorExportPlatformWeb::get_run_icon() const { + return run_icon; +} + +void EditorExportPlatformWeb::initialize() { + if (!EditorNode::get_singleton()) { + return; + } + + server.instantiate(); + + Ref image; + image.instantiate(); + + const bool upsample = !Math::is_equal_approx(Math::round(EDSCALE), EDSCALE); + + ImageLoaderSVG::create_image_from_string(image, _web_logo_svg, EDSCALE, upsample, false); + logo = ImageTexture::create_from_image(image); + + ImageLoaderSVG::create_image_from_string(image, _web_run_icon_svg, EDSCALE, upsample, false); + run_icon = ImageTexture::create_from_image(image); + + Ref theme = EditorNode::get_singleton()->get_editor_theme(); + if (theme.is_valid()) { + stop_icon = theme->get_icon(SNAME("Stop"), EditorStringName(EditorIcons)); + restart_icon = theme->get_icon(SNAME("Reload"), EditorStringName(EditorIcons)); + } else { + stop_icon.instantiate(); + restart_icon.instantiate(); + } } diff --git a/platform/web/export/export_plugin.h b/platform/web/export/export_plugin.h index 5de8ff06dd91..8b13a65f76ec 100644 --- a/platform/web/export/export_plugin.h +++ b/platform/web/export/export_plugin.h @@ -35,6 +35,7 @@ #include "editor/editor_node.h" #include "editor/editor_string_names.h" #include "editor/export/editor_export_platform.h" +#include "editor/export/editor_export_preset.h" #include "main/splash.gen.h" class ImageTexture; @@ -42,12 +43,96 @@ class ImageTexture; class EditorExportPlatformWeb : public EditorExportPlatform { GDCLASS(EditorExportPlatformWeb, EditorExportPlatform); + static inline const String PREFIX_RES = "res://"; + static inline const String SUFFIX_IMPORT = ".import"; + static inline const String SUFFIX_REMAP = ".remap"; + + static inline const String PATH_PROJECT_BINARY = "res://project.binary"; + static inline const String PATH_ASSETS_SPARSEPCK = "res://assets.sparsepck"; + static inline const String PATH_GODOT_UID_CACHE = "res://.godot/uid_cache.bin"; + static inline const String PATH_GODOT_GLOBAL_SCRIPT_CLASS_CACHE = "res://.godot/global_script_class_cache.cfg"; + enum RemoteDebugState { REMOTE_DEBUG_STATE_UNAVAILABLE, REMOTE_DEBUG_STATE_AVAILABLE, REMOTE_DEBUG_STATE_SERVING, }; + enum AsyncInitialInstallMode { + ASYNC_INITIAL_INSTALL_MODE_EVERYTHING = 0, + ASYNC_INITIAL_INSTALL_MODE_ONLY_REQUIRED_RESOURCES = 1, + }; + + struct ExportData { + struct File { + bool exists = false; + String resource_path; + String absolute_path; + uint32_t size = 0; + String md5; + String sha256; + + Dictionary get_as_dictionary() const { + Dictionary data; + data["size"] = size; + return data; + } + }; + + struct ResourceData { + struct SizeComparator { + _ALWAYS_INLINE_ bool operator()(const ExportData::ResourceData *p_a, const ExportData::ResourceData *p_b) const { + return p_a->get_size() < p_b->get_size(); + } + }; + struct FileNoCaseComparator { + _ALWAYS_INLINE_ bool operator()(const ExportData::ResourceData *p_a, const ExportData::ResourceData *p_b) const { + return ::FileNoCaseComparator()(p_a->path, p_b->path); + } + }; + + String path; + File native_file; + File remap_file; + File remapped_file; + LocalVector dependencies; + + uint32_t get_size() const; + Dictionary get_as_resource_dictionary() const; + String get_resource_path() const; + void flatten_dependencies(LocalVector *p_deps) const; + }; + + List dependencies; + HashMap dependencies_map; + EditorExportPlatformData::PackData pack_data; + String assets_directory; + String libraries_directory; + bool debug; + LocalVector libraries; + Ref preset; + + HashSet exported_files; + + HashSet get_features_set() const; + String res_to_global(const String &p_res_path) const { + ERR_FAIL_COND_V_MSG(!p_res_path.begins_with(PREFIX_RES), String(), vformat(R"*(Cannot convert non-resource path ("%s") to global.)*", p_res_path)); + String res_path = simplify_path(p_res_path); + return assets_directory.path_join(res_path.trim_prefix(PREFIX_RES)); + } + String global_to_res(const String &p_global_path) const { + return "res://" + p_global_path.trim_prefix(assets_directory.trim_suffix("/") + "/"); + } + String global_to_local(const String &p_global_path) const { + return p_global_path.trim_prefix(assets_directory.get_base_dir()); + } + + ResourceData *add_dependency(const String &p_path, const HashSet &p_features_set, Ref p_uid_cache, /* bool p_encrypt = false, */ Error *r_error = nullptr); + void update_file(File *p_file, const String &p_resource_path); + Dictionary get_deps_json_dictionary(const ResourceData *p_dependency); + Error save_deps_json(const ResourceData *p_dependency); + }; + Ref logo; Ref run_icon; Ref stop_icon; @@ -102,16 +187,22 @@ class EditorExportPlatformWeb : public EditorExportPlatform { Error _extract_template(const String &p_template, const String &p_dir, const String &p_name, bool pwa); void _replace_strings(const HashMap &p_replaces, Vector &r_template); - void _fix_html(Vector &p_html, const Ref &p_preset, const String &p_name, bool p_debug, BitField p_flags, const Vector p_shared_objects, const Dictionary &p_file_sizes); + void _fix_html(Vector &p_html, const Ref &p_preset, const String &p_name, bool p_debug, BitField p_flags, const Vector p_shared_objects, const Dictionary &p_file_sizes, const Dictionary &p_async_pck_data_jsondeps_json); + Error _add_manifest_icon(const Ref &p_preset, const String &p_path, const String &p_icon, int p_size, Array &r_arr); Error _build_pwa(const Ref &p_preset, const String p_path, const Vector &p_shared_objects); Error _write_or_error(const uint8_t *p_content, int p_len, String p_path); - Error _export_project(const Ref &p_preset, int p_debug_flags); + Error _run_export_project(const Ref &p_preset, int p_debug_flags); Error _launch_browser(const String &p_bind_host, uint16_t p_bind_port, bool p_use_tls); Error _start_server(const String &p_bind_host, uint16_t p_bind_port, bool p_use_tls); Error _stop_server(); + void _add_resource_data_tree_message(LocalVector &p_resource_data_entries, const String &p_context, bool p_sort_with_file_no_case_comparator = false, bool p_sort_with_size_comparator = false); + + static HashSet _get_mandatory_initial_install_files(const Ref &p_preset); + static Error _rename_and_store_file_in_async_pck(const Ref &p_preset, void *p_userdata, const String &p_path, const Vector &p_data, int p_file, int p_total, const Vector &p_enc_in_filters, const Vector &p_enc_ex_filters, const PackedByteArray &p_key, uint64_t p_seed, bool p_delta); + public: virtual void get_preset_features(const Ref &p_preset, List *r_features) const override; @@ -146,5 +237,4 @@ class EditorExportPlatformWeb : public EditorExportPlatform { String get_debug_protocol() const override { return "ws://"; } virtual void initialize() override; - ~EditorExportPlatformWeb(); }; diff --git a/platform/web/godot_js.h b/platform/web/godot_js.h index 29b291d566a4..d9ce0e1f0d25 100644 --- a/platform/web/godot_js.h +++ b/platform/web/godot_js.h @@ -38,14 +38,23 @@ extern "C" { #endif +typedef struct { + uint32_t length; + char *data; +} godot_js_string; + // Emscripten extern char *godot_js_emscripten_get_version(); -// Config +// Runtime. +typedef char *(*godot_js_runtime_get_config_file_as_json_callback)(const char *p_config_file_data); +extern void godot_js_runtime_set_get_config_file_as_json_cb(godot_js_runtime_get_config_file_as_json_callback p_callback); + +// Config. extern void godot_js_config_locale_get(char *p_ptr, int p_ptr_max); extern void godot_js_config_canvas_id_get(char *p_ptr, int p_ptr_max); -// OS +// OS. extern void godot_js_os_finish_async(void (*p_callback)()); extern void godot_js_os_request_quit_cb(void (*p_callback)()); extern int godot_js_os_fs_is_persistent(); @@ -55,10 +64,14 @@ extern void godot_js_os_shell_open(const char *p_uri); extern int godot_js_os_hw_concurrency_get(); extern int godot_js_os_thread_pool_size_get(); extern int godot_js_os_has_feature(const char *p_ftr); +extern int godot_js_os_async_pck_is_file_installed(const char *p_pck_dir, const char *p_path); +extern int godot_js_os_async_pck_add_install_file_callback(const char *p_pck_dir, const char *p_path, void (*p_callback)(const char *p_status_json_ptr, int p_status_json_len)); +extern int godot_js_os_async_pck_install_file(const char *p_pck_dir, const char *p_path); +extern char *godot_js_os_async_pck_install_file_get_status(const char *p_pck_dir, const char *p_path, const int32_t *p_return_string_length); extern int godot_js_pwa_cb(void (*p_callback)()); extern int godot_js_pwa_update(); -// Input +// Input. extern void godot_js_input_mouse_button_cb(int (*p_callback)(int p_pressed, int p_button, double p_x, double p_y, int p_modifiers)); extern void godot_js_input_mouse_move_cb(void (*p_callback)(double p_x, double p_y, double p_rel_x, double p_rel_y, int p_modifiers, double p_pressure)); extern void godot_js_input_mouse_wheel_cb(int (*p_callback)(int p_delta_mode, double p_delta_x, double p_delta_y)); @@ -71,7 +84,7 @@ extern void godot_js_set_ime_position(int p_x, int p_y); extern void godot_js_set_ime_cb(void (*p_input)(int p_type, const char *p_text), void (*p_callback)(int p_type, int p_repeat, int p_modifiers), char r_code[32], char r_key[32]); extern int godot_js_is_ime_focused(); -// Input gamepad +// Input gamepad. extern void godot_js_input_gamepad_cb(void (*p_on_change)(int p_index, int p_connected, const char *p_id, const char *p_guid)); extern int godot_js_input_gamepad_sample(); extern int godot_js_input_gamepad_sample_count(); @@ -79,7 +92,7 @@ extern int godot_js_input_gamepad_sample_get(int p_idx, float r_btns[16], int32_ extern void godot_js_input_paste_cb(void (*p_callback)(const char *p_text)); extern void godot_js_input_drop_files_cb(void (*p_callback)(const char **p_filev, int p_filec)); -// TTS +// TTS. extern int godot_js_tts_is_speaking(); extern int godot_js_tts_is_paused(); extern int godot_js_tts_get_voices(void (*p_callback)(int p_size, const char **p_voices)); @@ -88,7 +101,7 @@ extern void godot_js_tts_pause(); extern void godot_js_tts_resume(); extern void godot_js_tts_stop(); -// Display +// Display. extern int godot_js_display_screen_dpi_get(); extern double godot_js_display_pixel_ratio_get(); extern void godot_js_display_alert(const char *p_text); @@ -96,11 +109,11 @@ extern int godot_js_display_touchscreen_is_available(); extern int godot_js_display_is_swap_ok_cancel(); extern void godot_js_display_setup_canvas(int p_width, int p_height, int p_fullscreen, int p_hidpi); -// Display canvas +// Display canvas. extern void godot_js_display_canvas_focus(); extern int godot_js_display_canvas_is_focused(); -// Display window +// Display window. extern void godot_js_display_desired_size_set(int p_width, int p_height); extern int godot_js_display_size_update(); extern void godot_js_display_window_size_get(int32_t *p_x, int32_t *p_y); @@ -111,11 +124,11 @@ extern void godot_js_display_window_title_set(const char *p_text); extern void godot_js_display_window_icon_set(const uint8_t *p_ptr, int p_len); extern int godot_js_display_has_webgl(int p_version); -// Display clipboard +// Display clipboard. extern int godot_js_display_clipboard_set(const char *p_text); extern int godot_js_display_clipboard_get(void (*p_callback)(const char *p_text)); -// Display cursor +// Display cursor. extern void godot_js_display_cursor_set_shape(const char *p_cursor); extern int godot_js_display_cursor_is_hidden(); extern void godot_js_display_cursor_set_custom_shape(const char *p_shape, const uint8_t *p_ptr, int p_len, int p_hotspot_x, int p_hotspot_y); @@ -123,12 +136,12 @@ extern void godot_js_display_cursor_set_visible(int p_visible); extern void godot_js_display_cursor_lock_set(int p_lock); extern int godot_js_display_cursor_is_locked(); -// Display listeners +// Display listeners. extern void godot_js_display_fullscreen_cb(void (*p_callback)(int p_fullscreen)); extern void godot_js_display_window_blur_cb(void (*p_callback)()); extern void godot_js_display_notification_cb(void (*p_callback)(int p_notification), int p_enter, int p_exit, int p_in, int p_out); -// Display Virtual Keyboard +// Display Virtual Keyboard. extern int godot_js_display_vk_available(); extern int godot_js_display_tts_available(); extern void godot_js_display_vk_cb(void (*p_input)(const char *p_text, int p_cursor)); diff --git a/platform/web/js/engine/config.js b/platform/web/js/engine/config.js index d867376e4151..96d53f994933 100644 --- a/platform/web/js/engine/config.js +++ b/platform/web/js/engine/config.js @@ -52,6 +52,14 @@ const InternalConfig = function (initConfig) { // eslint-disable-line no-unused- * @type {?string} */ mainPack: null, + /** + * Main scene to preload. + * + * @memberof EngineConfig + * @default + * @type {?Object} + */ + asyncPckData: null, /** * Specify a language code to select the proper localization for the game. * @@ -271,6 +279,7 @@ const InternalConfig = function (initConfig) { // eslint-disable-line no-unused- this.fileSizes = parse('fileSizes', this.fileSizes); this.emscriptenPoolSize = parse('emscriptenPoolSize', this.emscriptenPoolSize); this.godotPoolSize = parse('godotPoolSize', this.godotPoolSize); + this.asyncPckData = parse('asyncPckData', this.asyncPckData); this.args = parse('args', this.args); this.onExecute = parse('onExecute', this.onExecute); this.onExit = parse('onExit', this.onExit); @@ -291,16 +300,22 @@ const InternalConfig = function (initConfig) { // eslint-disable-line no-unused- 'noExitRuntime': false, 'dynamicLibraries': [`${loadPath}.side.wasm`].concat(this.gdextensionLibs), 'emscriptenPoolSize': this.emscriptenPoolSize, - 'instantiateWasm': function (imports, onSuccess) { - function done(result) { - onSuccess(result['instance'], result['module']); - } + 'instantiateWasm': function (pImports, pOnSuccess) { + const onResult = (pResult) => { + pOnSuccess(pResult['instance'], pResult['module']); + }; + const onError = (pError) => { + this.printErr('Could not instantiate streaming of wasm.', pError); + }; if (typeof (WebAssembly.instantiateStreaming) !== 'undefined') { - WebAssembly.instantiateStreaming(Promise.resolve(r), imports).then(done); + WebAssembly.instantiateStreaming(r, pImports) + .then(onResult) + .catch(onError); } else { - r.arrayBuffer().then(function (buffer) { - WebAssembly.instantiate(buffer, imports).then(done); - }); + r.arrayBuffer() + .then((pBuffer) => WebAssembly.instantiate(pBuffer, pImports)) + .then(onResult) + .catch(onError); } r = null; return {}; @@ -360,6 +375,9 @@ const InternalConfig = function (initConfig) { // eslint-disable-line no-unused- return { 'canvas': this.canvas, 'canvasResizePolicy': this.canvasResizePolicy, + 'mainPack': this.mainPack, + 'asyncPckData': this.asyncPckData, + 'fileSizes': this.fileSizes, 'locale': locale, 'persistentDrops': this.persistentDrops, 'virtualKeyboard': this.experimentalVK, diff --git a/platform/web/js/engine/engine.js b/platform/web/js/engine/engine.js index 1aeeb62f18de..03302f8aec50 100644 --- a/platform/web/js/engine/engine.js +++ b/platform/web/js/engine/engine.js @@ -43,7 +43,9 @@ const Engine = (function () { Engine.load = function (basePath, size) { if (loadPromise == null) { loadPath = basePath; - loadPromise = preloader.loadPromise(`${loadPath}.wasm`, size, true); + const wasmPath = `${loadPath}.wasm`; + preloader.preparePreload(wasmPath, wasmPath, size); + loadPromise = preloader.loadPromise(wasmPath, size, true); requestAnimationFrame(preloader.animateProgress); } return loadPromise; @@ -78,36 +80,32 @@ const Engine = (function () { if (initPromise) { return initPromise; } + if (loadPromise == null) { if (!basePath) { - initPromise = Promise.reject(new Error('A base path must be provided when calling `init` and the engine is not loaded.')); + const initPromiseError = new Error('A base path must be provided when calling `init` and the engine is not loaded.'); + initPromise = Promise.reject(initPromiseError); return initPromise; } + Engine.load(basePath, this.config.fileSizes[`${basePath}.wasm`]); } - const me = this; - function doInit(promise) { - // Care! Promise chaining is bogus with old emscripten versions. - // This caused a regression with the Mono build (which uses an older emscripten version). - // Make sure to test that when refactoring. - return new Promise(function (resolve, reject) { - promise.then(function (response) { - const cloned = new Response(response.clone().body, { 'headers': [['content-type', 'application/wasm']] }); - Godot(me.config.getModuleConfig(loadPath, cloned)).then(function (module) { - const paths = me.config.persistentPaths; - module['initFS'](paths).then(function (err) { - me.rtenv = module; - if (me.config.unloadAfterInit) { - Engine.unload(); - } - resolve(); - }); - }); - }); - }); - } - preloader.setProgressFunc(this.config.onProgress); - initPromise = doInit(loadPromise); + + const doInit = async () => { + const loadResponse = await loadPromise; + const module = await Godot(this.config.getModuleConfig(loadPath, loadResponse)); + const paths = this.config.persistentPaths; + const err = await module['initFS'](paths); + if (err != null) { + window['console'].error('Error while initializing Godot:', err); + } + this.rtenv = module; + if (this.config.unloadAfterInit) { + Engine.unload(); + } + }; + + initPromise = doInit(); return initPromise; }, @@ -128,6 +126,7 @@ const Engine = (function () { * @returns {Promise} A Promise that resolves once the file is loaded. */ preloadFile: function (file, path) { + preloader.preparePreload(file, path, this.config.fileSizes[file]); return preloader.preload(file, path, this.config.fileSizes[file]); }, @@ -140,44 +139,56 @@ const Engine = (function () { * * Fails if a canvas cannot be found on the page, or not specified in the configuration. * + * @async * @param {EngineConfig} override An optional configuration override. - * @return {Promise} Promise that resolves once the engine started. + * @return {void} */ - start: function (override) { + start: async function (override) { this.config.update(override); const me = this; - return me.init().then(function () { - if (!me.rtenv) { - return Promise.reject(new Error('The engine must be initialized before it can be started')); - } + await me.init(); - let config = {}; - try { - config = me.config.getGodotConfig(function () { - me.rtenv = null; - }); - } catch (e) { - return Promise.reject(e); - } - // Godot configuration. - me.rtenv['initConfig'](config); + if (!me.rtenv) { + throw new Error('The engine must be initialized before it can be started'); + } - // Preload GDExtension libraries. - if (me.config.gdextensionLibs.length > 0 && !me.rtenv['loadDynamicLibrary']) { - return Promise.reject(new Error('GDExtension libraries are not supported by this engine version. ' - + 'Enable "Extensions Support" for your export preset and/or build your custom template with "dlink_enabled=yes".')); - } - return new Promise(function (resolve, reject) { - for (const file of preloader.preloadedFiles) { - me.rtenv['copyToFS'](file.path, file.buffer); - } - preloader.preloadedFiles.length = 0; // Clear memory - me.rtenv['callMain'](me.config.args); - initPromise = null; - me.installServiceWorker(); - resolve(); + let config = {}; + try { + config = me.config.getGodotConfig(function () { + me.rtenv = null; }); - }); + } catch (err) { + const newErr = new Error('Error getting Godot config.'); + newErr.cause = err; + throw newErr; + } + + // Godot configuration. + me.rtenv['initConfig'](config); + await me.rtenv['initOS'](); + + // Preload GDExtension libraries. + if (me.config.gdextensionLibs.length > 0 && !me.rtenv['loadDynamicLibrary']) { + throw new Error( + 'GDExtension libraries are not supported by this engine version. ' + + 'Enable "Extensions Support" for your export preset and/or build your custom template with "dlink_enabled=yes".' + ); + } + + try { + // eslint-disable-next-line no-unused-vars -- We don't use `_path`. + for (const [_path, file] of preloader.preloadedFiles) { + me.rtenv['copyToFS'](file.path, file.buffer); + } + preloader.preloadedFiles.length = 0; // Clear memory + me.rtenv['callMain'](me.config.args); + initPromise = null; + me.installServiceWorker(); + } catch (err) { + const newErr = new Error('Error while initializing.'); + newErr.cause = err; + throw newErr; + } }, /** @@ -193,20 +204,65 @@ const Engine = (function () { * @param {EngineConfig} override An optional configuration override. * @return {Promise} Promise that resolves once the game started. */ - startGame: function (override) { + startGame: async function (override) { this.config.update(override); + this.insertImportMap(); + // Add main-pack argument. const exe = this.config.executable; - const pack = this.config.mainPack || `${exe}.pck`; + let pack = this.config.mainPack || `${exe}.pck`; + if (pack.endsWith('/')) { + pack = pack.substring(0, pack.length - 1); + } + this.config.args = ['--main-pack', pack].concat(this.config.args); // Start and init with execName as loadPath if not inited. const me = this; - return Promise.all([ - this.init(exe), - this.preloadFile(pack, pack), - ]).then(function () { - return me.start.apply(me); + const asyncFilesToPreload = []; + + if (pack.endsWith('.asyncpck')) { + if (this.config.asyncPckData == null) { + throw new Error('No Main Scene dependencies found.'); + } + + const asyncPckData = this.config['asyncPckData']; + const asyncPckAssetsDir = asyncPckData['directories']['assets']; + + const asyncPckInitialLoadFilesSet = new Set(); + const asyncPckDataInitialLoad = asyncPckData['initialLoad']; + for (const value of Object.values(asyncPckDataInitialLoad)) { + for (const resourcePath of Object.values(value['files'])) { + asyncPckInitialLoadFilesSet.add(resourcePath); + } + } + + const asyncPckInitialLoadFiles = Array.from(asyncPckInitialLoadFilesSet); + + const resToLocal = (pPath) => { + const PREFIX_RES = 'res://'; + let path = pPath; + if (path.startsWith(PREFIX_RES)) { + path = path.substring('res://'.length); + } + return `${asyncPckAssetsDir}/${path}`; + }; + + for (const resourcePath of asyncPckInitialLoadFiles) { + const pathToPreload = resToLocal(resourcePath); + asyncFilesToPreload.push(this.preloadFile(pathToPreload, pathToPreload)); + } + } else { + asyncFilesToPreload.push(this.preloadFile(pack, pack)); + } + + preloader.init({ + fileSizes: this.config.fileSizes, }); + preloader.setProgressFunc(this.config.onProgress); + + await Promise.all([this.init(exe), ...asyncFilesToPreload]); + + return me.start.apply(me); }, /** @@ -249,6 +305,27 @@ const Engine = (function () { } return Promise.resolve(); }, + + /** + * Install the JavaScript module import map. + */ + insertImportMap() { + const IMPORTMAP_ID = 'godotengine-importmap-engine'; + if (document.getElementById(IMPORTMAP_ID)) { + return; + } + const scriptElement = document.createElement('script'); + scriptElement.id = IMPORTMAP_ID; + scriptElement.type = 'importmap'; + const scriptElementContent = { + imports: { + '@godotengine/utils/concurrencyQueueManager': `./${this.config.executable}.utils.concurrency.js`, + '@godotengine/utils/wait': `./${this.config.executable}.utils.wait.js`, + }, + }; + scriptElement.textContent = JSON.stringify(scriptElementContent, null, 2); + document.head.insertAdjacentElement('beforeend', scriptElement); + }, }; Engine.prototype = proto; diff --git a/platform/web/js/engine/preloader.js b/platform/web/js/engine/preloader.js index 564c68d26449..ad9d79b29ba3 100644 --- a/platform/web/js/engine/preloader.js +++ b/platform/web/js/engine/preloader.js @@ -1,133 +1,217 @@ const Preloader = /** @constructor */ function () { // eslint-disable-line no-unused-vars - function getTrackedResponse(response, load_status) { - function onloadprogress(reader, controller) { - return reader.read().then(function (result) { - if (load_status.done) { - return Promise.resolve(); - } - if (result.value) { - controller.enqueue(result.value); - load_status.loaded += result.value.length; - } - if (!result.done) { - return onloadprogress(reader, controller); - } - load_status.done = true; + const DOWNLOAD_ATTEMPTS_MAX = 4; + const loadingFiles = {}; + const lastProgress = { loaded: 0, total: 0 }; + let progressFunc = null; + let concurrencyQueueManager = null; + let filesSizeTotal = 0; + + /** + * @typedef {{ + * path: string; + * buffer: Uint8Array | null; + * fileSize: number; + * }} PreloadedFile + * @type {Map} + */ + this.preloadedFiles = new Map(); + + function getTrackedResponse(pResponse, pLoadStatus) { + async function onLoadProgress(pReader, pController) { + const { done, value } = await pReader.read(); + if (pLoadStatus.done) { return Promise.resolve(); - }); + } + if (done) { + pLoadStatus.done = true; + return Promise.resolve(); + } + pController.enqueue(value); + pLoadStatus.loaded += value.byteLength; + return onLoadProgress(pReader, pController); } - const reader = response.body.getReader(); + + const reader = pResponse.clone().body.getReader(); return new Response(new ReadableStream({ - start: function (controller) { - onloadprogress(reader, controller).then(function () { - controller.close(); - }); + start: async function (pController) { + try { + await onLoadProgress(reader, pController); + } finally { + pController.close(); + } }, - }), { headers: response.headers }); + }), { headers: pResponse.headers }); } - function loadFetch(file, tracker, fileSize, raw) { - tracker[file] = { - total: fileSize || 0, - loaded: 0, - done: false, - }; - return fetch(file).then(function (response) { + async function loadFetch(pFile, pFileSize, pIsRaw) { + if (pFile in loadingFiles) { + loadingFiles[pFile].requested = true; + } else { + loadingFiles[pFile] = { + file: pFile, + total: pFileSize || 0, + loaded: 0, + requested: true, + done: false, + }; + } + + try { + const response = await fetch(pFile); + if (!response.ok) { - return Promise.reject(new Error(`Failed loading file '${file}'`)); + throw new Error(`Got response ${response.status}: ${response.statusText}`); } - const tr = getTrackedResponse(response, tracker[file]); - if (raw) { + const tr = getTrackedResponse(response, loadingFiles[pFile]); + if (pIsRaw) { return Promise.resolve(tr); } + return tr.arrayBuffer(); - }); + } catch (error) { + const newError = new Error(`loadFetch for "${pFile}" failed:`); + newError.cause = error; + throw newError; + } } - function retry(func, attempts = 1) { + function retry(pCallback, pAttempts = 1) { function onerror(err) { - if (attempts <= 1) { + if (pAttempts <= 1) { return Promise.reject(err); } return new Promise(function (resolve, reject) { setTimeout(function () { - retry(func, attempts - 1).then(resolve).catch(reject); + retry(pCallback, pAttempts - 1).then(resolve).catch(reject); }, 1000); }); } - return func().catch(onerror); + return pCallback().catch(onerror); } - const DOWNLOAD_ATTEMPTS_MAX = 4; - const loadingFiles = {}; - const lastProgress = { loaded: 0, total: 0 }; - let progressFunc = null; - - const animateProgress = function () { + this.animateProgress = () => { let loaded = 0; - let total = 0; - let totalIsValid = true; - let progressIsFinal = true; + const requestedFiles = Object.values(loadingFiles) + .filter((pLoadingFile) => pLoadingFile.requested); + let progressIsFinal = false; + if (requestedFiles.length > 0 && requestedFiles.every((pRequestedFile) => pRequestedFile.done)) { + progressIsFinal = true; + } - Object.keys(loadingFiles).forEach(function (file) { - const stat = loadingFiles[file]; - if (!stat.done) { - progressIsFinal = false; + // eslint-disable-next-line no-unused-vars + for (const [_file, status] of Object.entries(loadingFiles)) { + if (!status.requested) { + continue; } - if (!totalIsValid || stat.total === 0) { - totalIsValid = false; - total = 0; - } else { - total += stat.total; + + if (!status.done) { + progressIsFinal = false; } - loaded += stat.loaded; - }); - if (loaded !== lastProgress.loaded || total !== lastProgress.total) { + + loaded += status.loaded; + } + + if (loaded !== lastProgress.loaded || filesSizeTotal !== lastProgress.total) { lastProgress.loaded = loaded; - lastProgress.total = total; + lastProgress.total = filesSizeTotal; + if (typeof progressFunc === 'function') { - progressFunc(loaded, total); + progressFunc(loaded, filesSizeTotal); } } if (!progressIsFinal) { - requestAnimationFrame(animateProgress); + window.requestAnimationFrame(() => this.animateProgress()); } }; - this.animateProgress = animateProgress; - - this.setProgressFunc = function (callback) { - progressFunc = callback; + this.setProgressFunc = function (pCallback) { + progressFunc = pCallback; }; - this.loadPromise = function (file, fileSize, raw = false) { - return retry(loadFetch.bind(null, file, loadingFiles, fileSize, raw), DOWNLOAD_ATTEMPTS_MAX); + this.loadPromise = async function (pFile, pFileSize, pIsRaw = false) { + if (concurrencyQueueManager == null) { + const { ConcurrencyQueueManager } = await import('@godotengine/utils/concurrencyQueueManager'); + // Another `loadPromise()` could have ended while awaiting. + if (concurrencyQueueManager == null) { + concurrencyQueueManager = new ConcurrencyQueueManager(); + } + } + + try { + return await concurrencyQueueManager.queue(() => retry( + async () => await loadFetch(pFile, pFileSize, pIsRaw), + DOWNLOAD_ATTEMPTS_MAX + )); + } catch (error) { + const newError = new Error(`An error occurred while running \`Preloader.loadPromise("${pFile}", ${pFileSize}, pIsRaw = ${pIsRaw})\``); + newError.cause = error; + throw error; + } }; - this.preloadedFiles = []; - this.preload = function (pathOrBuffer, destPath, fileSize) { + this.preload = async (pPathOrBuffer, pDestPath, pFileSize) => { let buffer = null; - if (typeof pathOrBuffer === 'string') { + if (typeof pPathOrBuffer === 'string') { + const path = pPathOrBuffer; + const preloadedFileEntry = this.preloadedFiles.get(path); const me = this; - return this.loadPromise(pathOrBuffer, fileSize).then(function (buf) { - me.preloadedFiles.push({ - path: destPath || pathOrBuffer, - buffer: buf, + + if (preloadedFileEntry == null) { + filesSizeTotal += pFileSize; + } + + buffer = await this.loadPromise(path, pFileSize); + + if (preloadedFileEntry == null) { + me.preloadedFiles.set(path, { + path: pDestPath ?? pPathOrBuffer, + buffer, + fileSize: pFileSize, }); - return Promise.resolve(); - }); - } else if (pathOrBuffer instanceof ArrayBuffer) { - buffer = new Uint8Array(pathOrBuffer); - } else if (ArrayBuffer.isView(pathOrBuffer)) { - buffer = new Uint8Array(pathOrBuffer.buffer); + } else { + preloadedFileEntry.buffer = buffer; + } + + return; + } else if (pPathOrBuffer instanceof ArrayBuffer) { + buffer = new Uint8Array(pPathOrBuffer); + } else if (ArrayBuffer.isView(pPathOrBuffer)) { + buffer = new Uint8Array(pPathOrBuffer.buffer); } - if (buffer) { - this.preloadedFiles.push({ - path: destPath, - buffer: pathOrBuffer, - }); - return Promise.resolve(); + if (buffer == null) { + throw new Error('Invalid object for preloading'); + } + + filesSizeTotal += pFileSize; + this.preloadedFiles.set(buffer, { + path: pDestPath, + buffer, + fileSize: pFileSize, + }); + }; + + this.preparePreload = (pPath, pDestPath, pFileSize) => { + this.preloadedFiles.set(pPath, { + path: pDestPath, + buffer: null, + fileSize: pFileSize, + }); + filesSizeTotal += pFileSize; + }; + + this.init = (pOptions = {}) => { + const { + fileSizes: loadingFileSizes = {}, + } = pOptions; + + for (const [file, fileSize] of Object.entries(loadingFileSizes)) { + loadingFiles[file] = { + file, + total: fileSize || 0, + loaded: 0, + requested: false, + done: false, + }; } - return Promise.reject(new Error('Invalid object for preloading')); }; }; diff --git a/platform/web/js/libs/library_godot_os.js b/platform/web/js/libs/library_godot_os.js index 0e931b794e91..dbd85d2a263b 100644 --- a/platform/web/js/libs/library_godot_os.js +++ b/platform/web/js/libs/library_godot_os.js @@ -64,6 +64,9 @@ const GodotConfig = { godot_pool_size: 4, on_execute: null, on_exit: null, + mainPack: '', + asyncPckData: null, + fileSizes: null, init_config: function (p_opts) { GodotConfig.canvas_resize_policy = p_opts['canvasResizePolicy']; @@ -74,6 +77,9 @@ const GodotConfig = { GodotConfig.godot_pool_size = p_opts['godotPoolSize']; GodotConfig.on_execute = p_opts['onExecute']; GodotConfig.on_exit = p_opts['onExit']; + GodotConfig.mainPack = p_opts['mainPack']; + GodotConfig.asyncPckData = p_opts['asyncPckData'] ?? null; + GodotConfig.fileSizes = p_opts['fileSizes'] ?? null; if (p_opts['focusCanvas']) { GodotConfig.canvas.focus(); } @@ -90,6 +96,9 @@ const GodotConfig = { GodotConfig.persistent_drops = false; GodotConfig.on_execute = null; GodotConfig.on_exit = null; + GodotConfig.mainPack = ''; + GodotConfig.asyncPckData = null; + GodotConfig.fileSizes = null; }, }, @@ -227,17 +236,556 @@ const GodotFS = { }; mergeInto(LibraryManager.library, GodotFS); -const GodotOS = { +class AsyncPCKFile { + static get Status() { + return Object.freeze({ + STATUS_ERROR: 'STATUS_ERROR', + STATUS_IDLE: 'STATUS_IDLE', + STATUS_LOADING: 'STATUS_LOADING', + STATUS_INSTALLED: 'STATUS_INSTALLED', + }); + } + + constructor(pAsyncPCKPath, pPath, pSize) { + this.asyncPCKPath = pAsyncPCKPath; + this.path = pPath; + { + const assetsDir = GodotOS.asyncPCKGetAsyncPCKAssetsDir(this.asyncPCKPath); + this.localPath = `${assetsDir}/${GodotOS._removeResPrefix(this.path)}`; + } + + this._status = GodotOS.AsyncPCKFile.Status.STATUS_IDLE; + this._installed = false; + this._size = pSize; + this._progress = 0; + this._progressRatio = 0; + this._loadPromise = null; + this._error = null; + } + + get status() { + return this._status; + } + + get size() { + return this._size; + } + + get progress() { + return this._progress; + } + + get progressRatio() { + return this._progressRatio; + } + + get error() { + return this._error; + } + + _addToProgress(pAddedBytes) { + this._setProgress(this._progress + pAddedBytes); + } + + _setProgress(pProgress) { + let progress = pProgress; + if (progress <= 0) { + progress = 0; + } + this._progress = progress; + if (this._progress > this._size) { + this._size = progress; + } + if (this._size > 0) { + this._progressRatio = progress / this._size; + } + } + + async load() { + if (this._loadPromise != null) { + return this._loadPromise; + } + + if (this._installed) { + GodotRuntime.print( + `AsyncPCKFile "${this.path}" (of AsyncPck "${this.asyncPCKPath}") is already installed, skipping loading.` + ); + return Promise.resolve(); + } + if (this._status == GodotOS.AsyncPCKFile.Status.STATUS_LOADING) { + GodotRuntime.print( + `AsyncPCKFile "${this.path}" (of AsyncPck "${this.asyncPCKPath}") is currently loading, skipping loading.` + ); + return Promise.resolve(); + } + this._status = GodotOS.AsyncPCKFile.Status.STATUS_LOADING; + + this._loadPromise = this._load(); + return await this._loadPromise; + } + + async _load() { + try { + const fileBuffer = await this._loadAttempt(); + + GodotFS.copy_to_fs(this.localPath, fileBuffer); + this._status = GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED; + } catch (err) { + this._status = GodotOS.AsyncPCKFile.Status.STATUS_ERROR; + + const newError = new Error( + `AsyncPCKFile "${this.path}" (of AsyncPck "${this.asyncPCKPath}"): error while loading "${this.localPath}"` + ); + newError.cause = err; + this._error = newError; + throw newError; + } finally { + this._loadPromise = null; + } + } + + async _loadAttempt(pRetryCount = 0) { + try { + const fileResponse = await GodotOS.asyncPCKFetch(this.localPath); + if (!fileResponse.ok) { + this._status = GodotOS.AsyncPCKFile.Status.STATUS_ERROR; + throw new Error(`Couldn't load file "${this.localPath}".`); + } + + const chunks = []; + const reader = fileResponse.body.getReader(); + + while (true) { + const { done, value: chunk } = await reader.read(); // eslint-disable-line no-await-in-loop + if (done) { + break; + } + this._addToProgress(chunk.byteLength); + chunks.push(chunk); + } + + const fileBuffer = new Uint8Array(this._progress); + let filePosition = 0; + for (const chunk of chunks) { + fileBuffer.set(chunk, filePosition); + filePosition += chunk.byteLength; + } + + return fileBuffer; + } catch (err) { + const newError = new Error( + `AsyncPCKFile "${this.path}" (of AsyncPck "${this.asyncPCKPath}"): error while attempting to load "${this.localPath}". (attempt ${pRetryCount + 1}/${GodotOS._asyncPCKFetchMaxRetry})` + ); + newError.cause = err; + + if (pRetryCount == GodotOS._asyncPCKFetchMaxRetry) { + const maxRetryError = new Error(`Maximum retry count (${GodotOS._asyncPCKFetchMaxRetry}) reached.`); + maxRetryError.cause = newError; + throw maxRetryError; + } + + GodotRuntime.error(newError); + this._error = newError; + + // Exponent wait. + await GodotOS._wait(GodotOS._asyncPCKWaitTimeBaseMs * 2 ** pRetryCount, 'ms'); + return this._loadAttempt(pRetryCount + 1); + } + } + + flagAsInstalled() { + this._status = GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED; + this._progress = this._size; + this._progressRatio = 1.0; + } + + getAsJsonObject() { + let error = ''; + if (this._error != null) { + error = this._error.message; + } + + return { + local_path: this.localPath, + status: this._status, + size: this._size, + progress: this._progress, + progress_ratio: this._progressRatio, + error, + }; + } +} + +class AsyncPCKResource { + static createAndInitialize(pAsyncPCK, pPath, pFiles, pDependencies = []) { + const asyncPCKResource = new GodotOS.AsyncPCKResource(pAsyncPCK, pPath); + asyncPCKResource.initialize(pFiles, pDependencies); + asyncPCKResource.insertInInstallMap(); + return asyncPCKResource; + } + + static create(pAsyncPCK, pPath) { + const asyncPCKResource = new GodotOS.AsyncPCKResource(pAsyncPCK, pPath); + asyncPCKResource.insertInInstallMap(); + return asyncPCKResource; + } + + constructor(pAsyncPCKPath, pPath) { + this.asyncPCKPath = pAsyncPCKPath; + this.path = pPath; + this.files = []; + this.dependencies = []; + + this._initialized = false; + this._loadPromise = null; + } + + initialize(pFiles, pDependencies = []) { + if (this._initialized) { + throw new Error(`Cannot initialize AsyncPCKResource more than once. ("${this.path}" of "${this.asyncPCKPath}")`); + } + this._initialized = true; + + this.dependencies = pDependencies; + + for (const [filePath, fileDefinition] of Object.entries(pFiles)) { + const asyncPCKFile = new GodotOS.AsyncPCKFile(this.asyncPCKPath, filePath, fileDefinition['size']); + this.files.push(asyncPCKFile); + } + } + + get initialized() { + return this._initialized; + } + + get size() { + return this.files.reduce((pAccumulatorValue, pFile) => pAccumulatorValue + pFile.size, 0); + } + + get progress() { + return this.files.reduce((pAccumulatorValue, pFile) => pAccumulatorValue + pFile.progress, 0); + } + + get progressRatio() { + const currentSize = this.size; + if (currentSize <= 0) { + return 0; + } + return this.progress / currentSize; + } + + get status() { + if (this.files.find(GodotOS.AsyncPCKResource.isStatusError) != null) { + return GodotOS.AsyncPCKFile.Status.STATUS_ERROR; + } + if (this.files.length > 0 && this.files.every(GodotOS.AsyncPCKResource.isStatusInstalled)) { + return GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED; + } + if (this.files.find(GodotOS.AsyncPCKResource.isStatusLoading) != null) { + return GodotOS.AsyncPCKFile.Status.STATUS_LOADING; + } + return GodotOS.AsyncPCKFile.Status.STATUS_IDLE; + } + + get errors() { + return this.files + .filter(GodotOS.AsyncPCKResource.isStatusError) + .map((pFile) => pFile.error) + .filter((pFileError) => pFileError !== ''); + } + + get allDependencies() { + const dependenciesMap = new Map(); + for (const dependency of this.dependencies) { + const asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(this.asyncPCKPath, dependency); + if (asyncPCKResource == null) { + throw new Error( + `Cannot get dependencies of a non-resource ("${dependency}" of "${this.asyncPCKPath}")` + ); + } + asyncPCKResource._getAllDependencies(dependenciesMap); + } + return Object.fromEntries(dependenciesMap); + } + + _getAllDependencies(pDependenciesMap) { + if (pDependenciesMap.has(this.path)) { + return; + } + pDependenciesMap.set(this.path, this.getAsJsonObject({ withDependencies: false })); + for (const dependency of this.dependencies) { + const asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(this.asyncPCKPath, dependency); + if (asyncPCKResource == null) { + throw new Error( + `Cannot get dependencies of a non-resource ("${dependency}" of "${this.asyncPCKPath}")` + ); + } + asyncPCKResource._getAllDependencies(pDependenciesMap); + } + } + + get allDependenciesResources() { + const dependenciesResources = []; + const allDependencies = this.allDependencies; + for (const dependency of Object.keys(allDependencies)) { + const asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(this.asyncPCKPath, dependency); + if (asyncPCKResource == null) { + throw new Error( + `Cannot get dependencies of a non-resource ("${dependency}" of "${this.asyncPCKPath}")` + ); + } + dependenciesResources.push(asyncPCKResource); + } + return dependenciesResources; + } + + async load() { + if (this._loadPromise != null) { + return this._loadPromise; + } + if (this.status == GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED) { + return Promise.resolve(); + } + this._loadPromise = this._load(); + return await this._loadPromise; + } + + async _load() { + try { + await Promise.allSettled( + this.files.map((pFile) => { + if (pFile.status == GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED) { + return Promise.resolve(); + } + return pFile.load(pFile); + }) + ); + } catch (err) { + const newError = new Error( + `AsyncPCKResource "${this.path}" (of AsyncPCK "${this.asyncPCKPath}"): error while loading"` + ); + newError.cause = err; + throw newError; + } finally { + this._loadPromise = null; + } + } + + flagAsInstalled() { + for (const file of this.files) { + file.flagAsInstalled(); + } + } + + insertInInstallMap() { + if (!GodotOS._asyncPCKInstallMap.has(this.asyncPCKPath)) { + GodotOS._asyncPCKInstallMap.set(this.asyncPCKPath, new Map()); + } + if (GodotOS._asyncPCKInstallMap.get(this.asyncPCKPath).has(this.path)) { + const asyncPCKResource = GodotOS._asyncPCKInstallMap.get(this.asyncPCKPath).get(this.path); + if (!asyncPCKResource.isTemporary) { + throw new Error( + `AsyncPCKResource "${this.path}" (of AsyncPCK "${this.asyncPCKPath}"): cannot install over a non-temporary AsyncPCKResource"` + ); + } + } + GodotOS._asyncPCKInstallMap.get(this.asyncPCKPath).set(this.path, this); + } + + removeFromInstallMap() { + if (!GodotOS._asyncPCKInstallMap.has(this.asyncPCKPath)) { + return; + } + GodotOS._asyncPCKInstallMap.get(this.asyncPCKPath).delete(this.path); + } + + getAsJsonObject(pOptions = {}) { + const { withDependencies = true } = pOptions; + const jsonData = {}; + if (withDependencies) { + const dependenciesResources = this.allDependenciesResources; + jsonData['files'] = {}; + jsonData['files'][this.path] = this.getAsJsonObject({ withDependencies: false }); + for (const dependencyResource of dependenciesResources) { + jsonData['files'][dependencyResource.path] = dependencyResource.getAsJsonObject({ + withDependencies: false, + }); + } + + const jsonDataSize + = this.size + + dependenciesResources.reduce( + (pAccumulator, pDependencyResource) => pAccumulator + pDependencyResource.size, + 0 + ); + const jsonDataProgress + = this.progress + + dependenciesResources.reduce( + (pAccumulator, pDependencyResource) => pAccumulator + pDependencyResource.progress, + 0 + ); + let jsonDataProgressRatio = 0; + if (jsonDataSize > 0) { + jsonDataProgressRatio = jsonDataProgress / jsonDataSize; + } + + const jsonDataErrors = Object.assign( + {}, + (() => { + const thisInstanceErrors = this.errors; + if (thisInstanceErrors.length === 0) { + return {}; + } + return { [this.path]: thisInstanceErrors }; + })(), + ...dependenciesResources.map((pDependencyResource) => { + const errors = pDependencyResource.errors; + if (errors.length === 0) { + return {}; + } + return { + [pDependencyResource.path]: pDependencyResource.errors, + }; + }) + ); + + jsonData['size'] = jsonDataSize; + jsonData['progress'] = jsonDataProgress; + jsonData['progress_ratio'] = jsonDataProgressRatio; + jsonData['errors'] = jsonDataErrors; + + let status = this.status; + if ( + status == GodotOS.AsyncPCKFile.Status.STATUS_IDLE + || status == GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED + ) { + if (dependenciesResources.find(GodotOS.AsyncPCKResource.isStatusError) != null) { + status = GodotOS.AsyncPCKFile.Status.STATUS_ERROR; + } + if (dependenciesResources.length > 0 && dependenciesResources.every(GodotOS.AsyncPCKResource.isStatusInstalled)) { + status = GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED; + } + if (dependenciesResources.find(GodotOS.AsyncPCKResource.isStatusLoading) != null) { + status = GodotOS.AsyncPCKFile.Status.STATUS_LOADING; + } + } + jsonData['status'] = status; + } else { + jsonData['size'] = this.size; + jsonData['progress'] = this.progress; + jsonData['progress_ratio'] = this.progressRatio; + jsonData['status'] = this.status; + jsonData['errors'] = this.errors; + } + + return jsonData; + } + + static isStatus(pStatus, pFile) { + return pFile.status == pStatus; + } + + static isStatusError(pFile) { + return GodotOS.AsyncPCKResource.isStatus(GodotOS.AsyncPCKFile.Status.STATUS_ERROR, pFile); + } + + static isStatusLoading(pFile) { + return GodotOS.AsyncPCKResource.isStatus(GodotOS.AsyncPCKFile.Status.STATUS_LOADING, pFile); + } + + static isStatusInstalled(pFile) { + return GodotOS.AsyncPCKResource.isStatus(GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED, pFile); + } +} + +const _GodotOS = { $GodotOS__deps: ['$GodotRuntime', '$GodotConfig', '$GodotFS'], $GodotOS__postset: [ + 'Module["initOS"] = async () => { await GodotOS.init(); };', 'Module["request_quit"] = function() { GodotOS.request_quit() };', 'Module["onExit"] = GodotOS.cleanup;', 'GodotOS._fs_sync_promise = Promise.resolve();', - ].join(''), + 'GodotOS._asyncPCKInstallMap = new Map();', + ].join(' '), $GodotOS: { - request_quit: function () {}, + request_quit: function () { }, _async_cbs: [], _fs_sync_promise: null, + AsyncPCKFile: AsyncPCKFile, + AsyncPCKResource: AsyncPCKResource, + _asyncPCKInstallMap: null, + _asyncPCKConcurrencyQueueManager: null, + _asyncPCKFetchMaxRetry: 5, + _asyncPCKWaitTimeBaseMs: 100, + _asyncPCKCallbacks: [], + _mainPack: '', + _prefixRes: 'res://', + + _trimLastSlash: function (pPath) { + if (pPath.endsWith('/')) { + return pPath.substring(0, pPath.length - 1); + } + return pPath; + }, + _addResPrefix: function (pPath) { + let path = pPath; + if (!path.startsWith(GodotOS._prefixRes)) { + path = GodotOS._prefixRes + path; + } + return path; + }, + _removeResPrefix: function (pPath) { + let path = pPath; + if (path.startsWith(GodotOS._prefixRes)) { + path = path.substring(GodotOS._prefixRes.length); + } + return path; + }, + + init: async function () { + const { wait } = await import('@godotengine/utils/wait'); + GodotOS._wait = wait; + + GodotOS._mainPack = GodotConfig.mainPack ?? ''; + if (GodotOS._mainPack.endsWith('.asyncpck')) { + const { ConcurrencyQueueManager } = await import('@godotengine/utils/concurrencyQueueManager'); + // eslint-disable-next-line require-atomic-updates -- We set `GodotOS._concurrencyQueueManager` only once: at init time. + GodotOS._asyncPCKConcurrencyQueueManager = new ConcurrencyQueueManager(); + GodotOS.initAsyncPck(); + } + }, + + initAsyncPck: function () { + const data = GodotConfig.asyncPckData; + const fileSizes = GodotConfig.fileSizes; + + const initialLoad = data['initialLoad']; + for (const [resourceKey, resourceValue] of Object.entries(initialLoad)) { + const dependencies = {}; + for (const resourceDependency of resourceValue['files']) { + let assetsPath = data.directories.assets; + if (!assetsPath.endsWith('/')) { + assetsPath += '/'; + } + const fileSizePath = assetsPath + resourceDependency.substring('res://'.length); + dependencies[resourceDependency] = { + size: fileSizes[fileSizePath], + }; + } + let asyncPckResource = GodotOS.asyncPCKGetAsyncPCKResource(GodotOS._mainPack, resourceKey); + if (asyncPckResource != null) { + continue; + } + + asyncPckResource = GodotOS.AsyncPCKResource.createAndInitialize( + GodotOS._mainPack, + resourceKey, + dependencies, + resourceValue?.dependencies ?? [] + ); + asyncPckResource.flagAsInstalled(); + } + }, atexit: function (p_promise_cb) { GodotOS._async_cbs.push(p_promise_cb); @@ -253,20 +801,124 @@ const GodotOS = { }, finish_async: function (callback) { - GodotOS._fs_sync_promise.then(function (err) { - const promises = []; - GodotOS._async_cbs.forEach(function (cb) { - promises.push(new Promise(cb)); + GodotOS._fs_sync_promise + .then(function (err) { + const promises = []; + GodotOS._async_cbs.forEach(function (cb) { + promises.push(new Promise(cb)); + }); + return Promise.all(promises); + }) + .then(function () { + return GodotFS.sync(); // Final FS sync. + }) + .then(function (err) { + // Always deferred. + setTimeout(function () { + callback(); + }, 0); }); - return Promise.all(promises); - }).then(function () { - return GodotFS.sync(); // Final FS sync. - }).then(function (err) { - // Always deferred. - setTimeout(function () { - callback(); - }, 0); + }, + + asyncPCKFetch: async function (...pArgs) { + if (GodotOS._asyncPCKConcurrencyQueueManager == null) { + throw new ReferenceError('`GodotOS._asyncPCKConcurrencyQueueManager` is null.'); + } + return await GodotOS._asyncPCKConcurrencyQueueManager.queue(() => fetch(...pArgs)); + }, + + asyncPCKGetAsyncPCKAssetsDir: function (pPckDir) { + let pckDir = GodotOS._trimLastSlash(pPckDir); + if (pckDir.endsWith('.asyncpck')) { + pckDir = `${pckDir}/assets`; + } + return pckDir; + }, + + asyncPCKGetAsyncPCKResource: function (pPckDir, pPath) { + if (!GodotOS._asyncPCKInstallMap.has(pPckDir)) { + return null; + } + const path = GodotOS._addResPrefix(pPath); + if (!GodotOS._asyncPCKInstallMap.get(pPckDir).has(path)) { + return null; + } + return GodotOS._asyncPCKInstallMap.get(pPckDir).get(path); + }, + + asyncPCKIsFileInstalled: function (pPckDir, pPath) { + const asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(pPckDir, pPath); + if (asyncPCKResource == null) { + return false; + } + return asyncPCKResource.status === GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED; + }, + + asyncPCKInstallFile: async function (pPckDir, pPath) { + let asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(pPckDir, pPath); + if (asyncPCKResource != null) { + if ( + asyncPCKResource.status != GodotOS.AsyncPCKFile.Status.STATUS_LOADING + && asyncPCKResource.status != GodotOS.AsyncPCKFile.Status.STATUS_INSTALLED + ) { + // `GodotOS.AsyncPCKResource.load()` returns it's loading promise if it exists. + await asyncPCKResource.load(); + } + return; + } + asyncPCKResource = GodotOS.AsyncPCKResource.create(pPckDir, pPath); + + const assetsDir = GodotOS.asyncPCKGetAsyncPCKAssetsDir(pPckDir); + const path = GodotOS._removeResPrefix(pPath); + const depsJsonPath = `${assetsDir}/${path}.deps.json`; + const depsJsonResponse = await GodotOS.asyncPCKFetch(depsJsonPath); + if (!depsJsonResponse.ok) { + GodotRuntime.error(`Couldn't load dependencies file "${depsJsonPath}".`); + asyncPCKResource.removeFromInstallMap(); + return; + } + + const remapResponseJson = await depsJsonResponse.json(); + const dependencies = remapResponseJson['dependencies']; + const resources = remapResponseJson['resources']; + + // Initialize the desired resource ASAP. + asyncPCKResource.initialize(resources[pPath].files, Object.keys(dependencies)); + + const localAsyncPCKResources = Object.entries(resources).map(([pResourcePath, pResourceDefinition]) => { + let localAsyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(pPckDir, pResourcePath); + const resourceFiles = pResourceDefinition['files']; + const resourceDependencies = dependencies?.[pResourcePath] ?? []; + + if (localAsyncPCKResource == null) { + localAsyncPCKResource = GodotOS.AsyncPCKResource.createAndInitialize( + pPckDir, + pResourcePath, + resourceFiles, + resourceDependencies + ); + } else if (!localAsyncPCKResource.initialized) { + localAsyncPCKResource.initialize(resourceFiles, resourceDependencies); + } + + return localAsyncPCKResource; }); + + await Promise.allSettled( + localAsyncPCKResources.map(async (pAsyncPCKResource) => { + await pAsyncPCKResource.load(); + }) + ); + }, + + asyncPCKInstallFileGetStatus: function (pPckDir, pPath) { + const path = GodotOS._addResPrefix(pPath); + const asyncPCKResource = GodotOS.asyncPCKGetAsyncPCKResource(pPckDir, path); + if (asyncPCKResource == null) { + return null; + } + const jsonObject = asyncPCKResource.getAsJsonObject(); + return jsonObject; }, }, @@ -322,6 +974,45 @@ const GodotOS = { return 0; }, + godot_js_os_async_pck_is_file_installed__proxy: 'sync', + godot_js_os_async_pck_is_file_installed__sig: 'ipp', + godot_js_os_async_pck_is_file_installed: function (pPckDirPtr, pPathPtr) { + const pckDir = GodotOS._trimLastSlash(GodotRuntime.parseString(pPckDirPtr)); + const path = GodotOS._addResPrefix(GodotOS._trimLastSlash(GodotRuntime.parseString(pPathPtr))); + return GodotOS.asyncPCKIsFileInstalled(pckDir, path); + }, + + godot_js_os_async_pck_install_file__proxy: 'sync', + godot_js_os_async_pck_install_file__sig: 'ipp', + godot_js_os_async_pck_install_file: function (pPckDirPtr, pPathPtr) { + const pckDir = GodotOS._trimLastSlash(GodotRuntime.parseString(pPckDirPtr)); + const path = GodotOS._addResPrefix(GodotOS._trimLastSlash(GodotRuntime.parseString(pPathPtr))); + + GodotOS.asyncPCKInstallFile(pckDir, path).catch((err) => { + GodotRuntime.error(`GodotOS.installAsyncFile("${pckDir}", "${path}")`, err); + }); + return 0; + }, + + godot_js_os_async_pck_install_file_get_status__proxy: 'sync', + godot_js_os_async_pck_install_file_get_status__sig: 'ppp', + godot_js_os_async_pck_install_file_get_status: function (pPckDirPtr, pPathPtr, pReturnStringLengthPtr) { + const pckDir = GodotOS._trimLastSlash(GodotRuntime.parseString(pPckDirPtr)); + const path = GodotOS._addResPrefix(GodotOS._trimLastSlash(GodotRuntime.parseString(pPathPtr))); + + const status = GodotOS.asyncPCKInstallFileGetStatus(pckDir, path); + if (status == null) { + return 0; + } + + const statusJson = JSON.stringify(status, null, 2); + const statusJsonPtr = GodotRuntime.allocString(statusJson); + if (pReturnStringLengthPtr !== 0) { + GodotRuntime.setHeapValue(pReturnStringLengthPtr, GodotRuntime.strlen(statusJson), 'i32'); + } + return statusJsonPtr; + }, + godot_js_os_execute__proxy: 'sync', godot_js_os_execute__sig: 'ii', godot_js_os_execute: function (p_json) { @@ -378,8 +1069,8 @@ const GodotOS = { }, }; -autoAddDeps(GodotOS, '$GodotOS'); -mergeInto(LibraryManager.library, GodotOS); +autoAddDeps(_GodotOS, '$GodotOS'); +mergeInto(LibraryManager.library, _GodotOS); /* * Godot event listeners. diff --git a/platform/web/js/libs/library_godot_runtime.js b/platform/web/js/libs/library_godot_runtime.js index 4114aa566ed4..96293d562025 100644 --- a/platform/web/js/libs/library_godot_runtime.js +++ b/platform/web/js/libs/library_godot_runtime.js @@ -30,6 +30,44 @@ const GodotRuntime = { $GodotRuntime: { + _getConfigFileAsJsonCallback: null, + + getConfigFileFromString: function (pConfigFileDataAsString) { + if (pConfigFileDataAsString.length == 0) { + return null; + } + if (GodotRuntime._getConfigFileAsJsonCallback == null) { + GodotRuntime.error('Could not get config file as JSON, callback not yet set.'); + return null; + } + globalThis['GodotRuntime'] = GodotRuntime; + globalThis['HEAPU32'] = HEAPU32; + globalThis['HEAPU8'] = HEAPU8; + globalThis['HEAP8'] = HEAP8; + + let configFileDataPtr = GodotRuntime.allocString(pConfigFileDataAsString); + const configFilePtr = GodotRuntime._getConfigFileAsJsonCallback(configFileDataPtr); + GodotRuntime.free(configFileDataPtr); + configFileDataPtr = 0; + + if (configFilePtr === 0) { + GodotRuntime.error('configFilePtr is nullptr'); + return null; + } + const configFile = GodotRuntime.parseString(configFilePtr); + + if (configFile.length === 0) { + GodotRuntime.error('configFile is empty', configFile); + return null; + } + const configFileJson = JSON.parse(configFile); + + GodotRuntime.free(configFileDataPtr); + configFileDataPtr = 0; + + return configFileJson; + }, + /* * Functions */ @@ -129,6 +167,12 @@ const GodotRuntime = { return stringToUTF8Array(p_str, HEAP8, p_ptr, p_len); }, }, + + godot_js_runtime_set_get_config_file_as_json_cb__proxy: 'async', + godot_js_runtime_set_get_config_file_as_json_cb__sig: 'pp', + godot_js_runtime_set_get_config_file_as_json_cb: function (pCallbackPtr) { + GodotRuntime._getConfigFileAsJsonCallback = GodotRuntime.get_func(pCallbackPtr); + }, }; autoAddDeps(GodotRuntime, '$GodotRuntime'); mergeInto(LibraryManager.library, GodotRuntime); diff --git a/platform/web/js/modules/utils/concurrency.js b/platform/web/js/modules/utils/concurrency.js new file mode 100644 index 000000000000..b67a452e0652 --- /dev/null +++ b/platform/web/js/modules/utils/concurrency.js @@ -0,0 +1,130 @@ +/** + * @file Simple concurrency queue manager. + * @example + * // This will limit the number of simultaneous fetch calls to 2. + * const manager = ConcurrencyQueueManager({ limit: 2 }); + * for (let i; i < 100; i++) { + * manager.queue(async () => { + * const targetUrl = new URL('https://example.org/'); + * targetUrl.searchParams.set("i", i); + * return await fetch(targetUrl); + * }); + * } + */ + +const defaultValues = Object.freeze({ + concurrencyLimit: 5, +}); + +/** + * @template T + * @typedef {() => Promise} PromiseWrapper + * @typedef {{ + * symbol: Symbol, + * promiseWrapper: PromiseWrapper, + * }} ConcurrencyQueueManagerItem + */ + +/** + * @typedef {{ + * limit: number, + * }} ConcurrencyQueueManagerOptions + */ + +export class ConcurrencyQueueManager { + /** @type {number} */ + #limit = defaultValues.concurrencyLimit; + /** @type {ConcurrencyQueueManagerItem[]} */ + #queue = []; + /** @type {ConcurrencyQueueManagerItem[]} */ + #active = []; + + #eventTarget = new EventTarget(); + + /** + * @constructor + * @param {ConcurrencyQueueManagerOptions} [pOptions={}] + */ + constructor(pOptions = {}) { + const { limit = defaultValues.concurrencyLimit } = pOptions; + this.#limit = limit; + } + + get limit() { + return this.#limit; + } + + /** + * Adds a function that returns a promise to the queue. Will return after the promise is executed. + * @async + * @template T + * @param {PromiseWrapper} pPromiseWrapper + * @returns {Promise} + * @throws + */ + async queue(pPromiseWrapper) { + if (pPromiseWrapper == null) { + throw new ReferenceError('pPromiseWrapper is null.'); + } + + const queueItem = await this.#waitForActiveQueueItem(pPromiseWrapper); + try { + return await queueItem.promiseWrapper(); + } catch (error) { + const newError = new Error('ConcurrencyQueueManager detected an error in a managed promise.'); + newError.cause = error; + throw error; + } finally { + const queueIndex = this.#active.indexOf(queueItem); + this.#active.splice(queueIndex, 1); + while (this.#queue.length > 0 && this.#active.length < this.#limit) { + const concurrencyQueueItem = this.#queue[0]; + this.#queue.splice(0, 1); + this.#active.push(queueItem); + this.#eventTarget.dispatchEvent(new CustomEvent('queuenext', { detail: concurrencyQueueItem })); + } + } + } + + /** + * Waits for the queue to be available. + * @template T + * @param {PromiseWrapper} pPromiseWrapper + * @returns {Promise>} + * @throws + */ + #waitForActiveQueueItem(pPromiseWrapper) { + if (pPromiseWrapper == null) { + throw new ReferenceError('pPromiseWrapper is null.'); + } + + const symbol = Symbol('ConcurrencyQueueItemId'); + /** @type {ConcurrencyQueueManagerItem} */ + const queueItem = { + symbol, + promiseWrapper: pPromiseWrapper, + }; + + if (this.#active.length < this.#limit) { + this.#active.push(queueItem); + return queueItem; + } + + this.#queue.push(queueItem); + return new Promise((pResolve, _pReject) => { + /** @type {(event: CustomEvent) => void} */ + const onQueueNext = (pEvent) => { + if (pEvent?.detail?.symbol !== symbol) { + return; + } + this.#eventTarget.removeEventListener('queuenext', onQueueNext); + pResolve(pEvent.detail); + }; + this.#eventTarget.addEventListener('queuenext', onQueueNext); + }); + } +} + +export default { + ConcurrencyQueueManager, +}; diff --git a/platform/web/js/modules/utils/wait.js b/platform/web/js/modules/utils/wait.js new file mode 100644 index 000000000000..a60852215f76 --- /dev/null +++ b/platform/web/js/modules/utils/wait.js @@ -0,0 +1,47 @@ +/** + * @file Wait utilities. + */ + +/** + * Waits the specified amount of time then returns. + * @async + * @param {number} pNumber + * @param {string} [pUnit="s"] + * @returns {Promise} + * @throws + */ +export function wait(pNumber, pUnit = 's') { + if (typeof pNumber != 'number') { + throw new TypeError('pNumber is not a number.'); + } + if (typeof pUnit != 'string') { + throw new TypeError('pUnit is not a string.'); + } + + let waitTime = 0; + + switch (pUnit.toLowerCase()) { + case 's': + case 'second': + case 'seconds': + waitTime = pNumber * 1000; + break; + + case 'ms': + case 'millisecond': + case 'milliseconds': + waitTime = pNumber; + break; + + default: + throw new Error(`Unknown pUnit (${pUnit})`); + } + + return new Promise((resolve) => { + setTimeout(() => resolve(), waitTime); + }); +} + +export default { + wait, +}; diff --git a/platform/web/os_web.cpp b/platform/web/os_web.cpp index b3e8784a40cb..85ed3440e2ff 100644 --- a/platform/web/os_web.cpp +++ b/platform/web/os_web.cpp @@ -36,7 +36,9 @@ #include "ip_web.h" #include "net_socket_web.h" +#include "core/io/config_file.h" #include "core/io/file_access.h" +#include "core/io/json.h" #include "core/os/main_loop.h" #include "core/os/os.h" #include "core/profiling/profiling.h" @@ -157,6 +159,85 @@ int OS_Web::get_default_thread_pool_size() const { #endif } +bool OS_Web::async_pck_is_file_installed(const String &p_path) const { + String path = ResourceUID::ensure_path(p_path); + ERR_FAIL_COND_V_MSG(!path.begins_with("res://"), false, vformat(R"*(Not able to install "%s" from a ".asyncpck".)*", path)); + + if (!OS::get_singleton()->async_pck_is_file_installable(path)) { + return false; + } + + Error err; + String pck_path = async_pck_get_async_pck_path(p_path, &err); + if (err != OK) { + return false; + } + + String pck_base_dir = pck_path.get_base_dir().get_base_dir(); + return godot_js_os_async_pck_is_file_installed(pck_base_dir.utf8().get_data(), path.utf8().get_data()); +} + +Error OS_Web::async_pck_install_file(const String &p_path) const { + String path = ResourceUID::ensure_path(p_path); + ERR_FAIL_COND_V_MSG(!path.begins_with("res://"), ERR_FILE_BAD_PATH, vformat(R"*(Not able to install "%s" from a ".asyncpck".)*", path)); + + if (async_pck_is_file_installed(path)) { + return OK; + } + + Error err; + String pck_path = async_pck_get_async_pck_path(path, &err); + if (err != OK) { + return err; + } + + String pck_base_dir = pck_path.get_base_dir().get_base_dir(); + return static_cast(godot_js_os_async_pck_install_file(pck_base_dir.utf8().get_data(), path.utf8().get_data())); +} + +Dictionary OS_Web::async_pck_install_file_get_status(const String &p_path) const { + String path = ResourceUID::ensure_path(p_path); + + if (FileAccess::exists(path)) { + Dictionary status; + status["files"] = Dictionary(); + status["size"] = 0; + status["progress"] = 0; + status["progress_ratio"] = 1; + status["status"] = "STATUS_INSTALLED"; + return status; + } + + Error err; + String pck_path = async_pck_get_async_pck_path(path, &err); + if (err != OK) { + Dictionary status; + status["files"] = Dictionary(); + status["size"] = 0; + status["progress"] = 0; + status["progress_ratio"] = 1; + status["status"] = "STATUS_ERROR"; + status["errors"] = Dictionary(); + return status; + } + + String pck_base_dir = pck_path.get_base_dir().get_base_dir(); + + int32_t status_text_length = 0; + char *status_text_ptr = godot_js_os_async_pck_install_file_get_status(pck_base_dir.utf8().get_data(), path.utf8().get_data(), &status_text_length); + if (status_text_ptr == nullptr || status_text_length <= 0) { + Dictionary status; + status["files"] = Dictionary(); + status["size"] = 0; + status["progress"] = 0; + status["progress_ratio"] = 1; + status["status"] = "STATUS_ERROR"; + status["errors"] = Dictionary(); + return status; + } + return JSON::parse_string(String::utf8(status_text_ptr, status_text_length)); +} + bool OS_Web::_check_internal_feature_support(const String &p_feature) { if (p_feature == "web") { return true; @@ -260,6 +341,35 @@ void OS_Web::update_pwa_state_callback() { } } +char *OS_Web::get_config_as_json_callback(const char *p_config_file_data_ptr) { + ERR_FAIL_NULL_V(p_config_file_data_ptr, nullptr); + String config_file_data_as_string = String::utf8(p_config_file_data_ptr); + ERR_FAIL_COND_V(config_file_data_as_string.is_empty(), nullptr); + + Ref config_file; + config_file.instantiate(); + config_file->parse(config_file_data_as_string); + + Dictionary json_config_file_data; + for (const String §ion : config_file->get_sections()) { + Dictionary section_data; + for (const String &key : config_file->get_section_keys(section)) { + section_data[key] = config_file->get_value(section, key); + } + json_config_file_data[section] = section_data; + } + String json_config_file_data_as_string = JSON::stringify(json_config_file_data, String(" ").repeat(2)); + size_t json_config_file_data_len = json_config_file_data_as_string.utf8().size(); + char *returned_json_config_file_data_ptr = (char *)memalloc(sizeof(char) * json_config_file_data_len); + ERR_FAIL_NULL_V(returned_json_config_file_data_ptr, nullptr); + memcpy(returned_json_config_file_data_ptr, json_config_file_data_as_string.utf8().ptr(), json_config_file_data_len); + godot_js_string *js_string = (godot_js_string *)memalloc(sizeof(godot_js_string)); + js_string->length = json_config_file_data_len; + js_string->data = returned_json_config_file_data_ptr; + ERR_FAIL_COND_V(js_string->data != returned_json_config_file_data_ptr, nullptr); + return returned_json_config_file_data_ptr; +} + void OS_Web::force_fs_sync() { if (is_userfs_persistent()) { idb_needs_sync = true; @@ -299,6 +409,7 @@ OS_Web::OS_Web() { setenv("LANG", locale_ptr, true); godot_js_pwa_cb(&OS_Web::update_pwa_state_callback); + godot_js_runtime_set_get_config_file_as_json_cb(&OS_Web::get_config_as_json_callback); if (AudioDriverWeb::is_available()) { audio_drivers.push_back(memnew(AudioDriverWorklet)); diff --git a/platform/web/os_web.h b/platform/web/os_web.h index 2862b259b1a5..ae3cab327d5c 100644 --- a/platform/web/os_web.h +++ b/platform/web/os_web.h @@ -56,6 +56,7 @@ class OS_Web : public OS_Unix { WASM_EXPORT static void dir_access_remove_callback(const String &p_file); WASM_EXPORT static void fs_sync_callback(); WASM_EXPORT static void update_pwa_state_callback(); + WASM_EXPORT static char *get_config_as_json_callback(const char *p_config_file_data); protected: void initialize() override; @@ -91,6 +92,11 @@ class OS_Web : public OS_Unix { String get_unique_id() const override; int get_default_thread_pool_size() const override; + bool async_pck_is_supported() const override { return true; } + Error async_pck_install_file(const String &p_path) const override; + bool async_pck_is_file_installed(const String &p_path) const override; + Dictionary async_pck_install_file_get_status(const String &p_path) const override; + String get_executable_path() const override; Error shell_open(const String &p_uri) override; String get_name() const override; diff --git a/platform/windows/export/export_plugin.cpp b/platform/windows/export/export_plugin.cpp index 4a69bb44a157..88d8f8298066 100644 --- a/platform/windows/export/export_plugin.cpp +++ b/platform/windows/export/export_plugin.cpp @@ -141,7 +141,7 @@ Error EditorExportPlatformWindows::_process_icon(const Ref & valid_icon_count++; } else { int size = (icon_size[i] == 0) ? 256 : icon_size[i]; - add_message(EXPORT_MESSAGE_WARNING, TTR("Resources Modification"), vformat(TTR("Icon size \"%d\" is missing."), size)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Resources Modification"), vformat(TTR("Icon size \"%d\" is missing."), size)); } } ERR_FAIL_COND_V(valid_icon_count == 0, ERR_CANT_OPEN); @@ -215,7 +215,7 @@ Error EditorExportPlatformWindows::export_project(const Ref } else { String exe_arch = _get_exe_arch(template_path); if (arch != exe_arch) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Mismatching custom export template executable architecture: found \"%s\", expected \"%s\"."), exe_arch, arch)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Mismatching custom export template executable architecture: found \"%s\", expected \"%s\"."), exe_arch, arch)); return ERR_CANT_CREATE; } } @@ -238,7 +238,7 @@ Error EditorExportPlatformWindows::export_project(const Ref Ref tmp_app_dir = DirAccess::create_for_path(tmp_dir_path); if (export_as_zip) { if (tmp_app_dir.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), tmp_dir_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("Prepare Templates"), vformat(TTR("Could not create and open the directory: \"%s\""), tmp_dir_path)); return ERR_CANT_CREATE; } if (DirAccess::exists(tmp_dir_path)) { @@ -324,7 +324,7 @@ Error EditorExportPlatformWindows::export_project(const Ref Ref tmp_dir = DirAccess::create_for_path(path.get_base_dir()); err = tmp_dir->rename(pck_path, path); if (err != OK) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to rename temporary file \"%s\"."), pck_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to rename temporary file \"%s\"."), pck_path)); } } @@ -529,7 +529,7 @@ Error EditorExportPlatformWindows::_add_data(const Ref &p_pr String tmp_icon_path = EditorPaths::get_singleton()->get_temp_dir().path_join("_tmp.ico"); if (!icon_path.is_empty()) { if (_process_icon(p_preset, icon_path, tmp_icon_path) != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Resources Modification"), vformat(TTR("Invalid icon file \"%s\"."), icon_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Resources Modification"), vformat(TTR("Invalid icon file \"%s\"."), icon_path)); icon_path = String(); } } @@ -549,7 +549,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p #ifdef WINDOWS_ENABLED String signtool_path = EDITOR_GET("export/windows/signtool"); if (!signtool_path.is_empty() && !FileAccess::exists(signtool_path)) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Could not find signtool executable at \"%s\"."), signtool_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Could not find signtool executable at \"%s\"."), signtool_path)); return ERR_FILE_NOT_FOUND; } if (signtool_path.is_empty()) { @@ -558,7 +558,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p #else String signtool_path = EDITOR_GET("export/windows/osslsigncode"); if (!signtool_path.is_empty() && !FileAccess::exists(signtool_path)) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Could not find osslsigncode executable at \"%s\"."), signtool_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Could not find osslsigncode executable at \"%s\"."), signtool_path)); return ERR_FILE_NOT_FOUND; } if (signtool_path.is_empty()) { @@ -578,7 +578,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p args.push_back("/f"); args.push_back(p_preset->get_or_env("codesign/identity", ENV_WIN_CODESIGN_ID)); } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); return FAILED; } } else if (id_type == 2) { //Windows certificate store @@ -586,11 +586,11 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p args.push_back("/sha1"); args.push_back(p_preset->get_or_env("codesign/identity", ENV_WIN_CODESIGN_ID)); } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); return FAILED; } } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Invalid identity type.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Invalid identity type.")); return FAILED; } #else @@ -599,7 +599,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p args.push_back("-pkcs12"); args.push_back(p_preset->get_or_env("codesign/identity", ENV_WIN_CODESIGN_ID)); } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("No identity found.")); return FAILED; } #endif @@ -631,7 +631,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p args.push_back(p_preset->get("codesign/timestamp_server_url")); #endif } else { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Invalid timestamp server.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Invalid timestamp server.")); return FAILED; } } @@ -680,9 +680,9 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p Error err = OS::get_singleton()->execute(signtool_path, args, &str, nullptr, true); if (err != OK || str.contains("not found") || str.contains("not recognized")) { #ifdef WINDOWS_ENABLED - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start signtool executable. Configure signtool path in the Editor Settings (Export > Windows > signtool), or disable \"Codesign\" in the export preset.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start signtool executable. Configure signtool path in the Editor Settings (Export > Windows > signtool), or disable \"Codesign\" in the export preset.")); #else - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start osslsigncode executable. Configure signtool path in the Editor Settings (Export > Windows > osslsigncode), or disable \"Codesign\" in the export preset.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), TTR("Could not start osslsigncode executable. Configure signtool path in the Editor Settings (Export > Windows > osslsigncode), or disable \"Codesign\" in the export preset.")); #endif return err; } @@ -693,7 +693,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p #else if (str.contains("Failed")) { #endif - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Signtool failed to sign executable: %s."), str)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Signtool failed to sign executable: %s."), str)); return FAILED; } @@ -702,13 +702,13 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p err = tmp_dir->remove(p_path); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Failed to remove temporary file \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Failed to remove temporary file \"%s\"."), p_path)); return err; } err = tmp_dir->rename(p_path + "_signed", p_path); if (err != OK) { - add_message(EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Failed to rename temporary file \"%s\"."), p_path + "_signed")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_WARNING, TTR("Code Signing"), vformat(TTR("Failed to rename temporary file \"%s\"."), p_path + "_signed")); return err; } #endif @@ -810,13 +810,13 @@ Error EditorExportPlatformWindows::fixup_embedded_pck(const String &p_path, int6 // Patch the header of the "pck" section in the PE file so that it corresponds to the embedded data. if (p_embedded_size + p_embedded_start >= 0x100000000) { // Check for total executable size. - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Windows executables cannot be >= 4 GiB.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Windows executables cannot be >= 4 GiB.")); return ERR_INVALID_DATA; } Ref f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (f.is_null()) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to open executable file \"%s\"."), p_path)); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), vformat(TTR("Failed to open executable file \"%s\"."), p_path)); return ERR_CANT_OPEN; } @@ -828,7 +828,7 @@ Error EditorExportPlatformWindows::fixup_embedded_pck(const String &p_path, int6 f->seek(pe_pos); uint32_t magic = f->get_32(); if (magic != 0x00004550) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable file header corrupted.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable file header corrupted.")); return ERR_FILE_CORRUPT; } } @@ -913,7 +913,7 @@ Error EditorExportPlatformWindows::fixup_embedded_pck(const String &p_path, int6 f->close(); if (pck_old_pos == -1) { - add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable \"pck\" section not found.")); + add_message(EditorExportPlatformData::EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("Executable \"pck\" section not found.")); return ERR_FILE_CORRUPT; } return OK; @@ -1042,7 +1042,7 @@ Error EditorExportPlatformWindows::run(const Ref &p_preset, } } - const bool use_remote = p_debug_flags.has_flag(DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(DEBUG_FLAG_DUMB_CLIENT); + const bool use_remote = p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_REMOTE_DEBUG) || p_debug_flags.has_flag(EditorExportPlatformData::DEBUG_FLAG_DUMB_CLIENT); int dbg_port = EDITOR_GET("network/debug/remote_port"); print_line("Creating temporary directory..."); diff --git a/pyproject.toml b/pyproject.toml index f00b29327aaf..b17c8b36eda8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ ignore-words-list = [ "outin", "parm", "pEvent", + "pResolve", "requestor", "streamin", "te", diff --git a/scene/scene_string_names.h b/scene/scene_string_names.h index 7a45a3555016..b9285abdf5c1 100644 --- a/scene/scene_string_names.h +++ b/scene/scene_string_names.h @@ -156,6 +156,11 @@ class SceneStringNames { const StringName state_started = "state_started"; const StringName state_finished = "state_finished"; + const StringName item_edited = "item_edited"; + const StringName check_propagated_to_item = "check_propagated_to_item"; + + const StringName timeout = "timeout"; + const StringName FlatButton = "FlatButton"; };