fix(app): make instances with non-UTF8 text file encodings launcheable and importable

Previous to these changes, the app always assumed that Minecraft and
other launchers always use UTF-8, which is not necessarily always true.

Author: AlexTMjugador

Cargo.lock

@@ -36,6 +36,7 @@ base64 = "0.22.1"
 bitflags = "2.9.0"
 bytes = "1.10.1"
 censor = "0.3.0"
+chardetng = "0.1.17"
 chrono = "0.4.41"
 clap = "4.5.38"
 clickhouse = "0.13.2"
@@ -50,6 +51,7 @@ dotenv-build = "0.1.1"
 dotenvy = "0.15.7"
 dunce = "1.0.5"
 either = "1.15.0"
+encoding_rs = "0.8.35"
 enumset = "1.1.6"
 flate2 = "1.1.1"
 fs4 = { version = "0.13.1", default-features = false }

Cargo.toml

@@ -20,6 +20,8 @@ tempfile.workspace = true
 dashmap = { workspace = true, features = ["serde"] }
 quick-xml = { workspace = true, features = ["async-tokio"] }
 enumset.workspace = true
+chardetng.workspace = true
+encoding_rs.workspace = true
 
 chrono = { workspace = true, features = ["serde"] }
 daedalus.workspace = true

packages/app-lib/Cargo.toml

@@ -97,12 +97,15 @@ pub struct ATLauncherMod {
 
 // Check if folder has a instance.json that parses
 pub async fn is_valid_atlauncher(instance_folder: PathBuf) -> bool {
-    let instance: String =
-        io::read_to_string(&instance_folder.join("instance.json"))
-            .await
-            .unwrap_or("".to_string());
-    let instance: Result<ATInstance, serde_json::Error> =
-        serde_json::from_str::<ATInstance>(&instance);
+    let instance = serde_json::from_str::<ATInstance>(
+        &io::read_any_encoding_to_string(
+            &instance_folder.join("instance.json"),
+        )
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    );
+
     if let Err(e) = instance {
         tracing::warn!(
             "Could not parse instance.json at {}: {}",
@@ -124,14 +127,17 @@ pub async fn import_atlauncher(
 ) -> crate::Result<()> {
     let atlauncher_instance_path = atlauncher_base_path
         .join("instances")
-        .join(instance_folder.clone());
+        .join(&instance_folder);
 
     // Load instance.json
-    let atinstance: String =
-        io::read_to_string(&atlauncher_instance_path.join("instance.json"))
-            .await?;
-    let atinstance: ATInstance =
-        serde_json::from_str::<ATInstance>(&atinstance)?;
+    let atinstance = serde_json::from_str::<ATInstance>(
+        &io::read_any_encoding_to_string(
+            &atlauncher_instance_path.join("instance.json"),
+        )
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    )?;
 
     // Icon path should be {instance_folder}/instance.png if it exists,
     // Second possibility is ATLauncher/configs/images/{safe_pack_name}.png (safe pack name is alphanumeric lowercase)

packages/app-lib/src/api/pack/import/atlauncher.rs

@@ -36,27 +36,31 @@ pub struct InstalledModpack {
 
 // Check if folder has a minecraftinstance.json that parses
 pub async fn is_valid_curseforge(instance_folder: PathBuf) -> bool {
-    let minecraftinstance: String =
-        io::read_to_string(&instance_folder.join("minecraftinstance.json"))
-            .await
-            .unwrap_or("".to_string());
-    let minecraftinstance: Result<MinecraftInstance, serde_json::Error> =
-        serde_json::from_str::<MinecraftInstance>(&minecraftinstance);
-    minecraftinstance.is_ok()
+    let minecraft_instance = serde_json::from_str::<MinecraftInstance>(
+        &io::read_any_encoding_to_string(
+            &instance_folder.join("minecraftinstance.json"),
+        )
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    );
+    minecraft_instance.is_ok()
 }
 
 pub async fn import_curseforge(
     curseforge_instance_folder: PathBuf, // instance's folder
     profile_path: &str,                  // path to profile
 ) -> crate::Result<()> {
     // Load minecraftinstance.json
-    let minecraft_instance: String = io::read_to_string(
-        &curseforge_instance_folder.join("minecraftinstance.json"),
-    )
-    .await?;
-    let minecraft_instance: MinecraftInstance =
-        serde_json::from_str::<MinecraftInstance>(&minecraft_instance)?;
-    let override_title: Option<String> = minecraft_instance.name.clone();
+    let minecraft_instance = serde_json::from_str::<MinecraftInstance>(
+        &io::read_any_encoding_to_string(
+            &curseforge_instance_folder.join("minecraftinstance.json"),
+        )
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    )?;
+    let override_title = minecraft_instance.name;
     let backup_name = format!(
         "Curseforge-{}",
         curseforge_instance_folder

packages/app-lib/src/api/pack/import/curseforge.rs

@@ -25,12 +25,12 @@ pub struct GDLauncherLoader {
 
 // Check if folder has a config.json that parses
 pub async fn is_valid_gdlauncher(instance_folder: PathBuf) -> bool {
-    let config: String =
-        io::read_to_string(&instance_folder.join("config.json"))
+    let config = serde_json::from_str::<GDLauncherConfig>(
+        &io::read_any_encoding_to_string(&instance_folder.join("config.json"))
             .await
-            .unwrap_or("".to_string());
-    let config: Result<GDLauncherConfig, serde_json::Error> =
-        serde_json::from_str::<GDLauncherConfig>(&config);
+            .unwrap_or(("".into(), encoding_rs::UTF_8))
+            .0,
+    );
     config.is_ok()
 }
 
@@ -39,12 +39,15 @@ pub async fn import_gdlauncher(
     profile_path: &str,                  // path to profile
 ) -> crate::Result<()> {
     // Load config.json
-    let config: String =
-        io::read_to_string(&gdlauncher_instance_folder.join("config.json"))
-            .await?;
-    let config: GDLauncherConfig =
-        serde_json::from_str::<GDLauncherConfig>(&config)?;
-    let override_title: Option<String> = config.loader.source_name.clone();
+    let config = serde_json::from_str::<GDLauncherConfig>(
+        &io::read_any_encoding_to_string(
+            &gdlauncher_instance_folder.join("config.json"),
+        )
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    )?;
+    let override_title = config.loader.source_name;
     let backup_name = format!(
         "GDLauncher-{}",
         gdlauncher_instance_folder

packages/app-lib/src/api/pack/import/gdlauncher.rs

@@ -144,8 +144,8 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool {
     let instance_cfg = instance_folder.join("instance.cfg");
     let mmc_pack = instance_folder.join("mmc-pack.json");
 
-    let mmc_pack = match io::read_to_string(&mmc_pack).await {
-        Ok(mmc_pack) => mmc_pack,
+    let mmc_pack = match io::read_any_encoding_to_string(&mmc_pack).await {
+        Ok((mmc_pack, _)) => mmc_pack,
         Err(_) => return false,
     };
 
@@ -155,7 +155,7 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool {
 
 #[tracing::instrument]
 pub async fn get_instances_subpath(config: PathBuf) -> Option<String> {
-    let launcher = io::read_to_string(&config).await.ok()?;
+    let launcher = io::read_any_encoding_to_string(&config).await.ok()?.0;
     let launcher: MMCLauncherEnum = serde_ini::from_str(&launcher).ok()?;
     match launcher {
         MMCLauncherEnum::General(p) => Some(p.general.instance_dir),
@@ -165,10 +165,9 @@ pub async fn get_instances_subpath(config: PathBuf) -> Option<String> {
 
 // Loading the INI (instance.cfg) file
 async fn load_instance_cfg(file_path: &Path) -> crate::Result<MMCInstance> {
-    let instance_cfg: String = io::read_to_string(file_path).await?;
-    let instance_cfg_enum: MMCInstanceEnum =
-        serde_ini::from_str::<MMCInstanceEnum>(&instance_cfg)?;
-    match instance_cfg_enum {
+    match serde_ini::from_str::<MMCInstanceEnum>(
+        &io::read_any_encoding_to_string(file_path).await?.0,
+    )? {
         MMCInstanceEnum::General(instance_cfg) => Ok(instance_cfg.general),
         MMCInstanceEnum::Instance(instance_cfg) => Ok(instance_cfg),
     }
@@ -183,9 +182,13 @@ pub async fn import_mmc(
     let mmc_instance_path =
         mmc_base_path.join("instances").join(instance_folder);
 
-    let mmc_pack =
-        io::read_to_string(&mmc_instance_path.join("mmc-pack.json")).await?;
-    let mmc_pack: MMCPack = serde_json::from_str::<MMCPack>(&mmc_pack)?;
+    let mmc_pack = serde_json::from_str::<MMCPack>(
+        &io::read_any_encoding_to_string(
+            &mmc_instance_path.join("mmc-pack.json"),
+        )
+        .await?
+        .0,
+    )?;
 
     let instance_cfg =
         load_instance_cfg(&mmc_instance_path.join("instance.cfg")).await?;
@@ -243,7 +246,7 @@ pub async fn import_mmc(
             _ => return Err(crate::ErrorKind::InputError("Instance is managed, but managed pack type not specified in instance.cfg".to_string()).into())
         }
     } else {
-        // Direclty import unmanaged pack
+        // Directly import unmanaged pack
         import_mmc_unmanaged(
             profile_path,
             minecraft_folder,

packages/app-lib/src/api/pack/import/mmc.rs

@@ -14,6 +14,7 @@ use chrono::Utc;
 use daedalus as d;
 use daedalus::minecraft::{LoggingSide, RuleAction, VersionInfo};
 use daedalus::modded::LoaderVersion;
+use regex::Regex;
 use serde::Deserialize;
 use st::Profile;
 use std::collections::HashMap;
@@ -662,14 +663,29 @@ pub async fn launch_minecraft(
 
     // Overwrites the minecraft options.txt file with the settings from the profile
     // Uses 'a:b' syntax which is not quite yaml
-    use regex::Regex;
-
     if !mc_set_options.is_empty() {
         let options_path = instance_path.join("options.txt");
-        let mut options_string = String::new();
-        if options_path.exists() {
-            options_string = io::read_to_string(&options_path).await?;
+
+        let (mut options_string, input_encoding) = if options_path.exists() {
+            io::read_any_encoding_to_string(&options_path).await?
+        } else {
+            (String::new(), encoding_rs::UTF_8)
+        };
+
+        // UTF-16 encodings may be successfully detected and read, but we cannot encode
+        // them back, and it's technically possible that the game client strongly expects
+        // such encoding
+        if input_encoding != input_encoding.output_encoding() {
+            return Err(crate::ErrorKind::LauncherError(format!(
+                "The instance options.txt file uses an unsupported encoding: {}. \
+                Please either turn off instance options that need to modify this file, \
+                or convert the file to an encoding that both the game and this app support, \
+                such as UTF-8.",
+                input_encoding.name()
+            ))
+            .into());
         }
+
         for (key, value) in mc_set_options {
             let re = Regex::new(&format!(r"(?m)^{}:.*$", regex::escape(key)))?;
             // check if the regex exists in the file
@@ -684,7 +700,8 @@ pub async fn launch_minecraft(
             }
         }
 
-        io::write(&options_path, options_string).await?;
+        io::write(&options_path, input_encoding.encode(&options_string).0)
+            .await?;
     }
 
     crate::api::profile::edit(&profile.path, |prof| {