From 64861d81aeca416fd27bf775bd799e7809b0270e Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 14:39:29 -0500 Subject: [PATCH 1/6] Remove modio:: return type from fetch_mods_by_ids --- src/providers/modio.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/providers/modio.rs b/src/providers/modio.rs index 3bea1cd6..22bcac74 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -117,14 +117,20 @@ impl ModioMod { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct ModioModResponse { id: u32, + name_id: String, + profile_url: String, } impl From for ModioModResponse { fn from(value: modio::mods::Mod) -> Self { - Self { id: value.id } + Self { + id: value.id, + name_id: value.name_id, + profile_url: value.profile_url.to_string(), + } } } @@ -259,7 +265,7 @@ pub trait DrgModio: Sync + Send { async fn fetch_mods_by_ids( &self, filter_ids: Vec, - ) -> Result, DrgModioError>; + ) -> Result, DrgModioError>; async fn fetch_mod_updates_since( &self, mod_ids: Vec, @@ -411,7 +417,7 @@ impl DrgModio for modio::Modio { async fn fetch_mods_by_ids( &self, filter_ids: Vec, - ) -> Result, DrgModioError> { + ) -> Result, DrgModioError> { use modio::filter::In; use modio::mods::filters::Id; @@ -423,7 +429,10 @@ impl DrgModio for modio::Modio { .search(filter) .collect() .await - .context(GenericModioSnafu)?) + .context(GenericModioSnafu)? + .into_iter() + .map(|m| m.into()) + .collect()) } async fn fetch_mod_updates_since( @@ -1106,7 +1115,12 @@ mod test { .returning(move |name| { mod_names .get(name) - .map(|id| vec![ModioModResponse { id: **id }]) + .map(|id| { + vec![ModioModResponse { + id: **id, + ..Default::default() + }] + }) .ok_or(DrgModioError::GenericError { msg: "not found" }) }); mock.expect_fetch_mod().times(1).returning(move |_, id| { From a5b572744556cdcd9f50279f59f34d91879a163e Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 15:00:46 -0500 Subject: [PATCH 2/6] wip --- Cargo.lock | 1 + Cargo.toml | 1 + src/providers/modio.rs | 353 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 351 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a987b86..45baf806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2802,6 +2802,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-subscriber", "typetag", "uasset_utils", "unreal_asset", diff --git a/Cargo.toml b/Cargo.toml index 84781b0e..4d4ddbd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ snafu.workspace = true strum = { version = "0.26", features = ["derive"] } itertools.workspace = true egui_dnd = "0.6.0" +tracing-subscriber = "0.3.18" [target.'cfg(target_env = "msvc")'.dependencies] hook = { path = "hook", artifact = "cdylib", optional = true, target = "x86_64-pc-windows-msvc"} diff --git a/src/providers/modio.rs b/src/providers/modio.rs index 22bcac74..41ba4ad4 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeSet, HashSet}; use std::sync::OnceLock; use std::time::{SystemTime, UNIX_EPOCH}; +use itertools::Itertools; #[cfg(test)] use mockall::{automock, predicate::*}; @@ -9,6 +10,7 @@ use ::modio; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next}; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use task_local_extensions::Extensions; use tracing::*; @@ -26,7 +28,7 @@ const MODIO_PROVIDER_ID: &str = "modio"; inventory::submit! { super::ProviderFactory { id: MODIO_PROVIDER_ID, - new: ModioProvider::::new_provider, + new: ModioProvider::::new_provider, can_provide: |url| re_mod().is_match(url), parameters: &[ super::ProviderParameter { @@ -117,7 +119,7 @@ impl ModioMod { } } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ModioModResponse { id: u32, name_id: String, @@ -167,10 +169,10 @@ impl Middleware for LoggingMiddleware { ) -> reqwest_middleware::Result { loop { info!( - "request started {} {:?}", + "request started {} {}", self.requests .fetch_add(1, std::sync::atomic::Ordering::Relaxed), - req.url().path() + req.url() ); let res = next.clone().run(req.try_clone().unwrap(), extensions).await; if let Ok(res) = &res { @@ -194,6 +196,10 @@ pub enum DrgModioError { MissingOauthToken, #[snafu(display("mod.io error: {source}"))] GenericModioError { source: modio::Error }, + #[snafu(display("reqwest middlware error: {source}"))] + GenericReqwestMiddlewareError { source: reqwest_middleware::Error }, + #[snafu(display("reqwest error: {source}"))] + GenericReqwestError { source: reqwest::Error }, #[snafu(display("failed to perform basic mod.io probe: {source}"))] CheckFailed { source: modio::Error }, #[snafu(display("failed to fetch mod files for <{url}> (mod_id = {mod_id}): {source}"))] @@ -276,6 +282,345 @@ pub trait DrgModio: Sync + Send { modio::download::DownloadAction: From; } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ResModioMod { + name_id: String, + //name: String, + //latest_modfile: Option, + //modfiles: Vec, + //tags: HashSet, +} + +struct ModioAlt { + modio: modio::Modio, + client: reqwest_middleware::ClientWithMiddleware, + oauth: String, +} +impl ModioAlt { + async fn fetch_list( + &self, + endpoint: &str, + query: &str, + ) -> Result, DrgModioError> { + #[derive(Deserialize)] + pub struct ResultPage { + pub data: Vec, + pub result_count: i64, + pub result_offset: i64, + pub result_limit: i64, + pub result_total: i64, + } + + let mut offset = 0; + let mut results = vec![]; + + loop { + let url = + format!("https://api.mod.io/v1/games/2475/{endpoint}?_offset={offset}&{query}"); + let page = self + .client + .get(url) + .header("Authorization", format!("Bearer {}", self.oauth)) + .send() + .await + .context(GenericReqwestMiddlewareSnafu)? + .json::>() + .await + .context(GenericReqwestSnafu)?; + results.extend(page.data); + println!("req {offset} {} {}", results.len(), page.result_offset); + if results.len() == page.result_total as usize { + break; + } + offset += page.result_count; + } + Ok(results) + } +} +#[async_trait::async_trait] +impl DrgModio for ModioAlt { + fn with_parameters(parameters: &HashMap) -> Result { + let client: reqwest_middleware::ClientWithMiddleware = + reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) + .with::(Default::default()) + .build(); + let oauth = parameters + .get("oauth") + .context(MissingOauthTokenSnafu)? + .to_string(); + let modio = modio::Modio::new( + modio::Credentials::with_token( + "".to_owned(), // TODO patch modio to not use API key at all + oauth.clone(), + ), + client.clone(), + ) + .context(GenericModioSnafu)?; + + Ok(Self { + modio, + client, + oauth, + }) + } + + async fn check(&self) -> Result<(), DrgModioError> { + use modio::filter::Eq; + use modio::mods::filters::Id; + + self.modio + .game(MODIO_DRG_ID) + .mods() + .search(Id::eq(0)) + .collect() + .await + .context(CheckFailedSnafu)?; + Ok(()) + } + + async fn fetch_mod(&self, url: String, id: u32) -> Result { + use modio::filter::NotEq; + use modio::mods::filters::Id; + + let files = self + .modio + .game(MODIO_DRG_ID) + .mod_(id) + .files() + .search(Id::ne(0)) + .collect() + .await + .with_context(|_| FetchModFilesFailedSnafu { + mod_id: id, + url: url.clone(), + })?; + let r#mod = self + .modio + .game(MODIO_DRG_ID) + .mod_(id) + .get() + .await + .context(FetchModFailedSnafu { mod_id: id, url })?; + + Ok(ModioMod::new(r#mod, files)) + } + + async fn fetch_files(&self, url: String, mod_id: u32) -> Result { + use modio::filter::NotEq; + use modio::mods::filters::Id; + + let files = self + .modio + .game(MODIO_DRG_ID) + .mod_(mod_id) + .files() + .search(Id::ne(0)) + .collect() + .await + .with_context(|_| FetchModFilesFailedSnafu { + mod_id, + url: url.clone(), + })?; + let r#mod = self + .modio + .game(MODIO_DRG_ID) + .mod_(mod_id) + .get() + .await + .context(FetchModFailedSnafu { mod_id, url })?; + + Ok(ModioMod::new(r#mod, files)) + } + + async fn fetch_file( + &self, + url: String, + mod_id: u32, + modfile_id: u32, + ) -> Result { + let file = self + .modio + .game(MODIO_DRG_ID) + .mod_(mod_id) + .file(modfile_id) + .get() + .await + .with_context(|_| FetchModFileFailedSnafu { + url, + mod_id, + modfile_id, + })?; + Ok(file) + } + + async fn fetch_dependencies( + &self, + url: String, + mod_id: u32, + ) -> Result, DrgModioError> { + let res: Vec = self + .fetch_list(&format!("mods/{mod_id}/dependencies"), "") + .await + .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })?; + } + + async fn fetch_mods_by_name( + &self, + name_id: &str, + ) -> Result, DrgModioError> { + let params = vec![ + format!("name_id={name_id}"), + "visible-in=0,1".to_string() + ]; + let res: Vec = self + .fetch_list("mods", ¶ms.into_iter().join("&")) + .await?; + Ok(res) + } + + async fn fetch_mods_by_ids( + &self, + filter_ids: Vec, + ) -> Result, DrgModioError> { + let mod_list_str = filter_ids.into_iter().map(|id| id.to_string()).join(","); + let params = vec![ + format!("id-in={mod_list_str}"), + ]; + let res: Vec = self + .fetch_list("mods", ¶ms.into_iter().join("&")) + .await?; + Ok(res) + } + + async fn fetch_mod_updates_since( + &self, + mod_ids: Vec, + last_update: u64, + ) -> Result, DrgModioError> { + #[derive(Deserialize)] + struct ResultEvent { + mod_id: u32, + } + + let mod_list_str = mod_ids.into_iter().map(|id| id.to_string()).join(","); + let params = vec![ + format!("date_added-gt={last_update}"), + format!("event_type-not-in=MOD_COMMENT_ADDED%2CMOD_COMMENT_DELETED"), + format!("mod_id-in={mod_list_str}"), + ]; + let res: Vec = self + .fetch_list("mods/events", ¶ms.into_iter().join("&")) + .await?; + Ok(res.iter().map(|e| e.mod_id).collect::>()) + } + + fn download(&self, action: A) -> modio::download::Downloader + where + modio::download::DownloadAction: From, + { + self.modio.download(action) + } +} + +mod test2 { + use super::*; + + static INIT: std::sync::Once = std::sync::Once::new(); + pub fn setup_tracing() { + INIT.call_once(|| { + tracing_subscriber::fmt().init(); + }); + } + + fn client() -> T { + T::with_parameters(&HashMap::from([("oauth".to_string(), "".to_string())])) + .unwrap() + } + + #[tokio::test] + async fn test_modio_alt_dependencies() { + setup_tracing(); + + async fn get() -> Vec { + client::().fetch_dependencies("asdfa".to_string(), 1897251).await.unwrap() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_mods_by_name() { + setup_tracing(); + + async fn get() -> Vec { + client::().fetch_mods_by_name("sandbox-utilities").await.unwrap() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_mods_by_ids() { + setup_tracing(); + + async fn get() -> Vec { + let mods = vec![1620474, 3813360, 1618289]; + client::().fetch_mods_by_ids(mods).await.unwrap() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_mod_updates_since() { + setup_tracing(); + + async fn get() -> HashSet { + let mods = vec![ + 1620474, 3813360, 1618289, 1668540, 1490893, 1162391, 2434441, 2799076, 2191437, + 1839402, 1769472, 2690013, 2478655, 2334132, 1888355, 1034748, 3502887, 3418446, + 2947767, 1130085, 2247462, 1861561, 2152235, 1962912, 2101319, 2450041, 2273035, + 1645602, 1169791, 2408852, 2442215, 3148183, 2751974, 3431533, 1034830, 2101850, + 1034237, 1792770, 2905192, 2990379, 1517852, 2041465, 1770392, 2036912, 1137706, + 1157883, 3097022, 2424882, 2296587, 2348581, 1773996, 2433395, 1727746, 1499941, + 1893389, 1034161, 2771775, 2092879, 3122386, 1035598, 2728755, 1751567, 1582325, + 1130032, 2909497, 1510072, 1753813, 1150682, 1139370, 1533287, 2155079, 1034644, + 2613099, 1911144, 1158572, 2400086, 1637267, 2917335, 2939176, 2190584, 3733289, + 1750422, 2369777, 1034406, 1489631, 1038224, 1863656, 2723280, 1250251, 2503544, + 1122623, 1929699, 1166441, 1138668, 2685922, 1634437, 2125794, 3116431, 1585858, + 3096846, 3190129, 2745837, 1616717, 1399744, 2959011, 3179565, 1636619, 2030854, + 3112892, 1145532, 3146563, 2259069, 2750508, 1564609, 2148763, 3096163, 3205713, + 3363051, 1956173, 2515857, 2942827, 3135932, 1181346, 2180950, 2229868, 1035291, + 1139332, 1122785, 2434442, 2498386, 1533299, 1160469, 2525496, 2536881, 2224215, + 1372030, 3029397, 1835591, 2553961, 1365047, 1593577, 2025849, 2530578, 2435811, + 1664873, 1521216, 1807375, 2080865, 1151344, 2432311, 2133905, 1034587, 1720132, + 1751367, 1877135, 2121639, 2054975, 1960533, 1897251, 3035859, 3094888, 2044707, + 1824234, 2084461, 2986917, 2191191, 1606854, 2478398, 1157000, 1674997, 1160676, + 1831206, 1764999, 2585212, 2965513, 2038940, 1158356, 2025965, 1149629, 2566820, + 2461393, 1519511, 2814623, 3187876, 2767795, 2029130, 2909311, 2071320, 2474953, + 2032299, 2100272, 1033779, 2285617, + ]; + client::() + .fetch_mod_updates_since(mods, 1318552474) + .await + .unwrap() + } + + let a = get::().await; + let b = get::().await; + assert_eq!(a, b); + } +} + #[async_trait::async_trait] impl DrgModio for modio::Modio { fn with_parameters(parameters: &HashMap) -> Result { From 0e6f120bd6e273e9015991d724f5edcae9be2e43 Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 16:49:05 -0500 Subject: [PATCH 3/6] WIP hand rolled modio requests working except download --- src/providers/modio.rs | 341 +++++++++++++++++++++++------------------ 1 file changed, 194 insertions(+), 147 deletions(-) diff --git a/src/providers/modio.rs b/src/providers/modio.rs index 41ba4ad4..94bdf58c 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -108,12 +108,12 @@ pub struct ModioMod { } impl ModioMod { - fn new(mod_: modio::mods::Mod, files: Vec) -> Self { + fn new(mod_: ModioModResponse, modfiles: Vec) -> Self { Self { name_id: mod_.name_id, name: mod_.name, latest_modfile: mod_.modfile.map(|f| f.id), - modfiles: files.into_iter().map(ModioFile::new).collect(), + modfiles, tags: mod_.tags.into_iter().map(|t| t.name).collect(), } } @@ -123,7 +123,19 @@ impl ModioMod { pub struct ModioModResponse { id: u32, name_id: String, + name: String, profile_url: String, + modfile: Option, + tags: Vec, +} +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ModioModResponseTag { + name: String, +} +impl From for ModioModResponseTag { + fn from(value: modio::mods::Tag) -> Self { + Self { name: value.name } + } } impl From for ModioModResponse { @@ -131,7 +143,30 @@ impl From for ModioModResponse { Self { id: value.id, name_id: value.name_id, + name: value.name, profile_url: value.profile_url.to_string(), + modfile: value.modfile.map(Into::into), + tags: value.tags.into_iter().map(Into::into).collect(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ModioFileResponse { + id: u32, + date_added: u64, + version: Option, + changelog: Option, + filesize: u64, +} +impl From for ModioFileResponse { + fn from(value: modio::files::File) -> Self { + Self { + id: value.id, + date_added: value.date_added, + version: value.version, + changelog: value.changelog, + filesize: value.filesize, } } } @@ -143,16 +178,21 @@ pub struct ModioFile { version: Option, changelog: Option, } -impl ModioFile { - fn new(file: modio::files::File) -> Self { +impl From for ModioFile { + fn from(value: ModioFileResponse) -> Self { Self { - id: file.id, - date_added: file.date_added, - version: file.version, - changelog: file.changelog, + id: value.id, + date_added: value.date_added, + version: value.version, + changelog: value.changelog, } } } +impl From for ModioFile { + fn from(value: modio::files::File) -> Self { + Self::from(ModioFileResponse::from(value)) + } +} #[derive(Default)] struct LoggingMiddleware { @@ -197,14 +237,14 @@ pub enum DrgModioError { #[snafu(display("mod.io error: {source}"))] GenericModioError { source: modio::Error }, #[snafu(display("reqwest middlware error: {source}"))] - GenericReqwestMiddlewareError { source: reqwest_middleware::Error }, + GenericReqwestMiddlewareError { source: anyhow::Error }, #[snafu(display("reqwest error: {source}"))] GenericReqwestError { source: reqwest::Error }, #[snafu(display("failed to perform basic mod.io probe: {source}"))] CheckFailed { source: modio::Error }, #[snafu(display("failed to fetch mod files for <{url}> (mod_id = {mod_id}): {source}"))] FetchModFilesFailed { - source: modio::Error, + source: anyhow::Error, url: String, mod_id: u32, }, @@ -212,14 +252,14 @@ pub enum DrgModioError { "failed to fetch mod file {modfile_id} for <{url}> (mod_id = {mod_id}): {source}" ))] FetchModFileFailed { - source: modio::Error, + source: anyhow::Error, url: String, mod_id: u32, modfile_id: u32, }, #[snafu(display("failed to fetch mod <{url}> (mod_id = {mod_id}): {source}"))] FetchModFailed { - source: modio::Error, + source: anyhow::Error, url: String, mod_id: u32, }, @@ -227,7 +267,7 @@ pub enum DrgModioError { "failed to fetch dependencies for mod <{url}> (mod_id = {mod_id}): {source}" ))] FetchDependenciesFailed { - source: modio::Error, + source: anyhow::Error, url: String, mod_id: u32, }, @@ -255,13 +295,12 @@ pub trait DrgModio: Sync + Send { Self: Sized; async fn check(&self) -> Result<(), DrgModioError>; async fn fetch_mod(&self, url: String, id: u32) -> Result; - async fn fetch_files(&self, url: String, mod_id: u32) -> Result; async fn fetch_file( &self, url: String, mod_id: u32, modfile_id: u32, - ) -> Result; + ) -> Result; async fn fetch_dependencies(&self, url: String, mod_id: u32) -> Result, DrgModioError>; async fn fetch_mods_by_name( @@ -282,21 +321,28 @@ pub trait DrgModio: Sync + Send { modio::download::DownloadAction: From; } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ResModioMod { - name_id: String, - //name: String, - //latest_modfile: Option, - //modfiles: Vec, - //tags: HashSet, -} - struct ModioAlt { modio: modio::Modio, client: reqwest_middleware::ClientWithMiddleware, oauth: String, } impl ModioAlt { + async fn get(&self, endpoint: &str) -> Result { + self.client + .get(&format!("https://api.mod.io/v1/games/2475/{endpoint}")) + .header("Authorization", format!("Bearer {}", self.oauth)) + .send() + .await + // TODO clean up this nonsense + .map_err(|e| Into::::into(e)) + .context(GenericReqwestMiddlewareSnafu)? + .error_for_status() + .map_err(|e| Into::::into(e)) + .context(GenericReqwestMiddlewareSnafu)? + .json::() + .await + .context(GenericReqwestSnafu) + } async fn fetch_list( &self, endpoint: &str, @@ -315,18 +361,8 @@ impl ModioAlt { let mut results = vec![]; loop { - let url = - format!("https://api.mod.io/v1/games/2475/{endpoint}?_offset={offset}&{query}"); - let page = self - .client - .get(url) - .header("Authorization", format!("Bearer {}", self.oauth)) - .send() - .await - .context(GenericReqwestMiddlewareSnafu)? - .json::>() - .await - .context(GenericReqwestSnafu)?; + let url = format!("{endpoint}?_offset={offset}&{query}"); + let page: ResultPage = self.get(&url).await?; results.extend(page.data); println!("req {offset} {} {}", results.len(), page.result_offset); if results.len() == page.result_total as usize { @@ -365,71 +401,30 @@ impl DrgModio for ModioAlt { } async fn check(&self) -> Result<(), DrgModioError> { - use modio::filter::Eq; - use modio::mods::filters::Id; - - self.modio - .game(MODIO_DRG_ID) - .mods() - .search(Id::eq(0)) - .collect() - .await - .context(CheckFailedSnafu)?; - Ok(()) - } - - async fn fetch_mod(&self, url: String, id: u32) -> Result { - use modio::filter::NotEq; - use modio::mods::filters::Id; - - let files = self - .modio - .game(MODIO_DRG_ID) - .mod_(id) - .files() - .search(Id::ne(0)) - .collect() - .await - .with_context(|_| FetchModFilesFailedSnafu { - mod_id: id, - url: url.clone(), - })?; - let r#mod = self - .modio - .game(MODIO_DRG_ID) - .mod_(id) - .get() - .await - .context(FetchModFailedSnafu { mod_id: id, url })?; - - Ok(ModioMod::new(r#mod, files)) + #[derive(Deserialize)] + struct Empty {} + self.get::(&format!("mods?id=0")).await.map(|_| ()) } - async fn fetch_files(&self, url: String, mod_id: u32) -> Result { - use modio::filter::NotEq; - use modio::mods::filters::Id; - - let files = self - .modio - .game(MODIO_DRG_ID) - .mod_(mod_id) - .files() - .search(Id::ne(0)) - .collect() + async fn fetch_mod(&self, url: String, mod_id: u32) -> Result { + let res_files = self + .fetch_list::(&format!("mods/{mod_id}/files"), "id-not=0") .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchModFilesFailedSnafu { + url: url.to_string(), mod_id, - url: url.clone(), - })?; - let r#mod = self - .modio - .game(MODIO_DRG_ID) - .mod_(mod_id) - .get() - .await - .context(FetchModFailedSnafu { mod_id, url })?; + })? + .into_iter() + .map(Into::into) + .collect(); - Ok(ModioMod::new(r#mod, files)) + let res: ModioModResponse = self + .get(&format!("mods/{mod_id}")) + .await + .map_err(|e| Into::::into(e)) + .with_context(|_| FetchModFailedSnafu { url, mod_id })?; + Ok(ModioMod::new(res, res_files)) } async fn fetch_file( @@ -437,20 +432,17 @@ impl DrgModio for ModioAlt { url: String, mod_id: u32, modfile_id: u32, - ) -> Result { - let file = self - .modio - .game(MODIO_DRG_ID) - .mod_(mod_id) - .file(modfile_id) - .get() + ) -> Result { + let res: ModioFileResponse = self + .get(&format!("mods/{mod_id}/files/{modfile_id}")) .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchModFileFailedSnafu { url, mod_id, modfile_id, })?; - Ok(file) + Ok(res) } async fn fetch_dependencies( @@ -458,20 +450,23 @@ impl DrgModio for ModioAlt { url: String, mod_id: u32, ) -> Result, DrgModioError> { - let res: Vec = self + #[derive(Deserialize)] + struct ModDependency { + mod_id: u32, + } + let res: Vec = self .fetch_list(&format!("mods/{mod_id}/dependencies"), "") .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })?; + Ok(res.into_iter().map(|d| d.mod_id).collect()) } async fn fetch_mods_by_name( &self, name_id: &str, ) -> Result, DrgModioError> { - let params = vec![ - format!("name_id={name_id}"), - "visible-in=0,1".to_string() - ]; + let params = vec![format!("name_id={name_id}"), "visible-in=0,1".to_string()]; let res: Vec = self .fetch_list("mods", ¶ms.into_iter().join("&")) .await?; @@ -483,9 +478,7 @@ impl DrgModio for ModioAlt { filter_ids: Vec, ) -> Result, DrgModioError> { let mod_list_str = filter_ids.into_iter().map(|id| id.to_string()).join(","); - let params = vec![ - format!("id-in={mod_list_str}"), - ]; + let params = vec![format!("id-in={mod_list_str}")]; let res: Vec = self .fetch_list("mods", ¶ms.into_iter().join("&")) .await?; @@ -536,13 +529,81 @@ mod test2 { T::with_parameters(&HashMap::from([("oauth".to_string(), "".to_string())])) .unwrap() } + fn client_bad() -> T { + T::with_parameters(&HashMap::from([("oauth".to_string(), "bad".to_string())])).unwrap() + } + + #[tokio::test] + async fn test_modio_alt_check() { + setup_tracing(); + + async fn get() -> Option<()> { + client::().check().await.ok() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_check_bad() { + setup_tracing(); + + async fn get() -> Option<()> { + client_bad::().check().await.ok() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_fetch_mod() { + setup_tracing(); + + async fn get() -> ModioMod { + client::() + .fetch_mod("asdf".into(), 1897251) + .await + .unwrap() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } + + #[tokio::test] + async fn test_modio_alt_fetch_file() { + setup_tracing(); + + async fn get() -> ModioFileResponse { + client::() + .fetch_file("asdf".into(), 1897251, 4916540) + .await + .unwrap() + } + + let a = get::().await; + let b = get::().await; + dbg!(&a); + assert_eq!(a, b); + } #[tokio::test] async fn test_modio_alt_dependencies() { setup_tracing(); async fn get() -> Vec { - client::().fetch_dependencies("asdfa".to_string(), 1897251).await.unwrap() + client::() + .fetch_dependencies("asdfa".to_string(), 1897251) + .await + .unwrap() } let a = get::().await; @@ -556,7 +617,10 @@ mod test2 { setup_tracing(); async fn get() -> Vec { - client::().fetch_mods_by_name("sandbox-utilities").await.unwrap() + client::() + .fetch_mods_by_name("sandbox-utilities") + .await + .unwrap() } let a = get::().await; @@ -663,43 +727,23 @@ impl DrgModio for modio::Modio { .search(Id::ne(0)) .collect() .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchModFilesFailedSnafu { mod_id: id, url: url.clone(), - })?; + })? + .into_iter() + .map(Into::into) + .collect(); let r#mod = self .game(MODIO_DRG_ID) .mod_(id) .get() .await + .map_err(|e| Into::::into(e)) .context(FetchModFailedSnafu { mod_id: id, url })?; - Ok(ModioMod::new(r#mod, files)) - } - - async fn fetch_files(&self, url: String, mod_id: u32) -> Result { - use modio::filter::NotEq; - use modio::mods::filters::Id; - - let files = self - .game(MODIO_DRG_ID) - .mod_(mod_id) - .files() - .search(Id::ne(0)) - .collect() - .await - .with_context(|_| FetchModFilesFailedSnafu { - mod_id, - url: url.clone(), - })?; - let r#mod = self - .game(MODIO_DRG_ID) - .mod_(mod_id) - .get() - .await - .context(FetchModFailedSnafu { mod_id, url })?; - - Ok(ModioMod::new(r#mod, files)) + Ok(ModioMod::new(r#mod.into(), files)) } async fn fetch_file( @@ -707,19 +751,20 @@ impl DrgModio for modio::Modio { url: String, mod_id: u32, modfile_id: u32, - ) -> Result { + ) -> Result { let file = self .game(MODIO_DRG_ID) .mod_(mod_id) .file(modfile_id) .get() .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchModFileFailedSnafu { url, mod_id, modfile_id, })?; - Ok(file) + Ok(file.into()) } async fn fetch_dependencies( @@ -733,6 +778,7 @@ impl DrgModio for modio::Modio { .dependencies() .list() .await + .map_err(|e| Into::::into(e)) .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })? .into_iter() .map(|d| d.mod_id) @@ -923,10 +969,7 @@ impl ModProvider for ModioProvider { for m in mods { let id = m.id; // TODO avoid fetching mod a second time - let m = self - .modio - .fetch_files(m.profile_url.to_string(), id) - .await?; + let m = self.modio.fetch_mod(m.profile_url.to_string(), id).await?; write_cache(&cache, |c| { c.mod_id_map.insert(m.name_id.to_owned(), id); c.mods.insert(id, m); @@ -1101,7 +1144,11 @@ impl ModProvider for ModioProvider { .await?; let size = file.filesize; - let download: modio::download::DownloadAction = file.into(); + let download = modio::download::DownloadAction::File { + game_id: MODIO_DRG_ID, + mod_id, + file_id: file.id, + }; info!("downloading mod {url:?}..."); From 7fe66a3d6d3cb4a1c66431b3e55958828342fd1d Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 18:09:54 -0500 Subject: [PATCH 4/6] WIP download functional except for unwraps --- Cargo.lock | 7 ++- Cargo.toml | 3 +- src/providers/mod.rs | 2 +- src/providers/modio.rs | 121 ++++++++++++++++++++++++++++------------- 4 files changed, 92 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45baf806..7d9d6133 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,9 +705,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cairo-sys-rs" @@ -2759,6 +2759,7 @@ dependencies = [ "ansi_term", "anyhow", "async-trait", + "bytes", "clap", "dialoguer", "directories", @@ -4385,6 +4386,8 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d342c51730e54029130d7dc9fd735d28c4cd360f1368c01981d4f03ff207f096" dependencies = [ + "futures-core", + "pin-project", "snafu-derive", ] diff --git a/Cargo.toml b/Cargo.toml index 4d4ddbd2..b4204b33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,11 +92,12 @@ repak.workspace = true include_dir = "0.7.3" postcard.workspace = true fs-err.workspace = true -snafu.workspace = true +snafu = { workspace = true, features = ["futures"] } strum = { version = "0.26", features = ["derive"] } itertools.workspace = true egui_dnd = "0.6.0" tracing-subscriber = "0.3.18" +bytes = "1.6.0" [target.'cfg(target_env = "msvc")'.dependencies] hook = { path = "hook", artifact = "cdylib", optional = true, target = "x86_64-pc-windows-msvc"} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index dd979fd0..c66b180f 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -81,7 +81,7 @@ pub enum ProviderError { #[snafu(transparent)] DrgModioError { source: DrgModioError }, #[snafu(display("mod.io-related error encountered while working on mod {mod_id}: {source}"))] - ModCtxtModioError { source: ::modio::Error, mod_id: u32 }, + ModCtxtModioError { source: DrgModioError, mod_id: u32 }, #[snafu(display("I/O error encountered while working on mod {mod_id}: {source}"))] ModCtxtIoError { source: std::io::Error, mod_id: u32 }, #[snafu(transparent)] diff --git a/src/providers/modio.rs b/src/providers/modio.rs index 94bdf58c..ccf3bb84 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeSet, HashSet}; use std::sync::OnceLock; use std::time::{SystemTime, UNIX_EPOCH}; +use futures::TryStreamExt; use itertools::Itertools; #[cfg(test)] use mockall::{automock, predicate::*}; @@ -158,6 +159,7 @@ pub struct ModioFileResponse { version: Option, changelog: Option, filesize: u64, + download: ModioFileResponseDownload, } impl From for ModioFileResponse { fn from(value: modio::files::File) -> Self { @@ -167,6 +169,18 @@ impl From for ModioFileResponse { version: value.version, changelog: value.changelog, filesize: value.filesize, + download: value.download.into(), + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ModioFileResponseDownload { + binary_url: String, +} +impl From for ModioFileResponseDownload { + fn from(value: modio::files::Download) -> Self { + Self { + binary_url: value.binary_url.to_string(), } } } @@ -316,9 +330,11 @@ pub trait DrgModio: Sync + Send { mod_ids: Vec, last_update: u64, ) -> Result, DrgModioError>; - fn download(&self, action: A) -> modio::download::Downloader - where - modio::download::DownloadAction: From; + async fn download( + &self, + mod_id: u32, + file_id: u32, + ) -> futures::stream::BoxStream<'static, Result>; } struct ModioAlt { @@ -334,10 +350,10 @@ impl ModioAlt { .send() .await // TODO clean up this nonsense - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .context(GenericReqwestMiddlewareSnafu)? .error_for_status() - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .context(GenericReqwestMiddlewareSnafu)? .json::() .await @@ -349,12 +365,13 @@ impl ModioAlt { query: &str, ) -> Result, DrgModioError> { #[derive(Deserialize)] - pub struct ResultPage { - pub data: Vec, - pub result_count: i64, - pub result_offset: i64, - pub result_limit: i64, - pub result_total: i64, + #[allow(unused)] + struct ResultPage { + data: Vec, + result_count: i64, + result_offset: i64, + result_limit: i64, + result_total: i64, } let mut offset = 0; @@ -403,14 +420,14 @@ impl DrgModio for ModioAlt { async fn check(&self) -> Result<(), DrgModioError> { #[derive(Deserialize)] struct Empty {} - self.get::(&format!("mods?id=0")).await.map(|_| ()) + self.get::("mods?id=0").await.map(|_| ()) } async fn fetch_mod(&self, url: String, mod_id: u32) -> Result { let res_files = self .fetch_list::(&format!("mods/{mod_id}/files"), "id-not=0") .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchModFilesFailedSnafu { url: url.to_string(), mod_id, @@ -422,7 +439,7 @@ impl DrgModio for ModioAlt { let res: ModioModResponse = self .get(&format!("mods/{mod_id}")) .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchModFailedSnafu { url, mod_id })?; Ok(ModioMod::new(res, res_files)) } @@ -436,7 +453,7 @@ impl DrgModio for ModioAlt { let res: ModioFileResponse = self .get(&format!("mods/{mod_id}/files/{modfile_id}")) .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchModFileFailedSnafu { url, mod_id, @@ -457,7 +474,7 @@ impl DrgModio for ModioAlt { let res: Vec = self .fetch_list(&format!("mods/{mod_id}/dependencies"), "") .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })?; Ok(res.into_iter().map(|d| d.mod_id).collect()) } @@ -507,14 +524,38 @@ impl DrgModio for ModioAlt { Ok(res.iter().map(|e| e.mod_id).collect::>()) } - fn download(&self, action: A) -> modio::download::Downloader - where - modio::download::DownloadAction: From, - { - self.modio.download(action) + async fn download( + &self, + mod_id: u32, + file_id: u32, + ) -> futures::stream::BoxStream<'static, Result> { + use futures::StreamExt; + + let res = self + .get::(&format!("mods/{mod_id}/files/{file_id}")) + .await + .unwrap(); + + self.client + .get(res.download.binary_url) + .header("Authorization", format!("Bearer {}", self.oauth)) + .send() + .await + // TODO clean up this nonsense + //.map_err(Into::::into) + //.context(GenericReqwestMiddlewareSnafu) + .unwrap() + .error_for_status() + //.map_err(Into::::into) + //.context(GenericReqwestMiddlewareSnafu) + .unwrap() + .bytes_stream() + .context(GenericReqwestSnafu) + .boxed() } } +#[cfg(test)] mod test2 { use super::*; @@ -727,7 +768,7 @@ impl DrgModio for modio::Modio { .search(Id::ne(0)) .collect() .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchModFilesFailedSnafu { mod_id: id, url: url.clone(), @@ -740,7 +781,7 @@ impl DrgModio for modio::Modio { .mod_(id) .get() .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .context(FetchModFailedSnafu { mod_id: id, url })?; Ok(ModioMod::new(r#mod.into(), files)) @@ -758,7 +799,7 @@ impl DrgModio for modio::Modio { .file(modfile_id) .get() .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchModFileFailedSnafu { url, mod_id, @@ -778,7 +819,7 @@ impl DrgModio for modio::Modio { .dependencies() .list() .await - .map_err(|e| Into::::into(e)) + .map_err(Into::::into) .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })? .into_iter() .map(|d| d.mod_id) @@ -857,11 +898,23 @@ impl DrgModio for modio::Modio { Ok(events.iter().map(|e| e.mod_id).collect::>()) } - fn download(&self, action: A) -> modio::download::Downloader - where - modio::download::DownloadAction: From, - { - self.download(action) + async fn download( + &self, + mod_id: u32, + file_id: u32, + ) -> futures::stream::BoxStream<'static, Result> { + let download = modio::download::DownloadAction::File { + game_id: MODIO_DRG_ID, + mod_id, + file_id, + }; + + use futures::StreamExt; + + self.download(download) + .stream() + .context(GenericModioSnafu) + .boxed() } } @@ -1144,19 +1197,13 @@ impl ModProvider for ModioProvider { .await?; let size = file.filesize; - let download = modio::download::DownloadAction::File { - game_id: MODIO_DRG_ID, - mod_id, - file_id: file.id, - }; info!("downloading mod {url:?}..."); - use futures::stream::TryStreamExt; use tokio::io::AsyncWriteExt; let mut cursor = std::io::Cursor::new(vec![]); - let mut stream = Box::pin(self.modio.download(download).stream()); + let mut stream = self.modio.download(mod_id, file.id).await; while let Some(bytes) = stream .try_next() .await From 9bc38db49086b85808455b293052b7672617f280 Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 20:09:40 -0500 Subject: [PATCH 5/6] WIP working but very gross --- Cargo.lock | 212 +---------------- Cargo.toml | 3 +- src/providers/modio.rs | 508 +++++------------------------------------ 3 files changed, 72 insertions(+), 651 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d9d6133..c1762afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,21 +169,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -549,12 +534,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.5" @@ -792,19 +771,6 @@ dependencies = [ "libc", ] -[[package]] -name = "chrono" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "windows-targets 0.48.5", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1158,41 +1124,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "darling" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.48", -] - -[[package]] -name = "darling_macro" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.48", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -1200,7 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown", "lock_api", "once_cell", "parking_lot_core", @@ -1223,7 +1154,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" dependencies = [ "powerfmt", - "serde", ] [[package]] @@ -2151,19 +2081,13 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.2.6", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.3" @@ -2324,29 +2248,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "iana-time-zone" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core 0.51.1", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "iced-x86" version = "1.20.0" @@ -2367,12 +2268,6 @@ dependencies = [ "objc2 0.4.1", ] -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "0.5.0" @@ -2416,17 +2311,6 @@ dependencies = [ "quote", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - [[package]] name = "indexmap" version = "2.2.6" @@ -2434,7 +2318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown", "serde", ] @@ -2773,12 +2657,11 @@ dependencies = [ "hook", "image", "include_dir", - "indexmap 2.2.6", + "indexmap", "inventory", "itertools 0.12.0", "mint_lib", "mockall", - "modio", "obake", "opener", "path-slash", @@ -2877,30 +2760,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "modio" -version = "0.7.1" -source = "git+https://github.com/trumank/modio-rs.git?branch=dev#d979c0a1bf0fd865bb30feb850079530ec6b84ba" -dependencies = [ - "bitflags 1.3.2", - "bytes", - "futures-core", - "futures-util", - "http", - "mime", - "pin-project-lite", - "reqwest", - "reqwest-middleware", - "serde", - "serde_json", - "serde_test", - "serde_with", - "tokio", - "tokio-util", - "tracing", - "url", -] - [[package]] name = "naive-cityhash" version = "0.2.0" @@ -3243,7 +3102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4d6a8c22fc714f0c2373e6091bf6f5e9b37b1bc0b1184874b7e0a4e303d318f" dependencies = [ "dlv-list", - "hashbrown 0.14.3", + "hashbrown", ] [[package]] @@ -3833,7 +3692,7 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64 0.21.5", + "base64", "bytes", "encoding_rs", "futures-core", @@ -4013,7 +3872,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64", ] [[package]] @@ -4188,15 +4047,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_test" -version = "1.0.176" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4209,34 +4059,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" -dependencies = [ - "base64 0.13.1", - "chrono", - "hex", - "indexmap 1.9.3", - "serde", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "sha1" version = "0.10.6" @@ -4836,7 +4658,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", + "indexmap", "toml_datetime", "winnow", ] @@ -4847,7 +4669,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.2.6", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -4999,7 +4821,7 @@ dependencies = [ "anyhow", "byteorder", "cityhasher", - "indexmap 2.2.6", + "indexmap", "typed-path", "unicase", "unreal_asset", @@ -5192,7 +5014,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" dependencies = [ - "base64 0.21.5", + "base64", "flate2", "log", "once_cell", @@ -5211,7 +5033,6 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] [[package]] @@ -5579,19 +5400,10 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.52.0", + "windows-core", "windows-targets 0.52.0", ] -[[package]] -name = "windows-core" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index b4204b33..96d67741 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,13 +62,12 @@ image = { version = "0.24.7", default-features = false, features = ["png"] } indexmap = { version = "2.1.0", features = ["serde"] } inventory = "0.3.14" mint_lib = { path = "mint_lib" } -modio = { git = "https://github.com/trumank/modio-rs.git", branch = "dev", default-features = false, features = ["rustls-tls"] } obake = { version = "1.0.5", features = ["serde"] } opener = "0.6.1" path-slash = "0.2.1" rayon = "1.8.0" regex = "1.10.2" -reqwest.workspace = true +reqwest = { workspace = true, features = ["stream"] } reqwest-middleware = "0.2.4" rfd = "0.12.1" rust-ini = "0.20.0" diff --git a/src/providers/modio.rs b/src/providers/modio.rs index ccf3bb84..944d02ca 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -7,8 +7,6 @@ use itertools::Itertools; #[cfg(test)] use mockall::{automock, predicate::*}; -use ::modio; - use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next}; use serde::de::DeserializeOwned; @@ -133,24 +131,6 @@ pub struct ModioModResponse { pub struct ModioModResponseTag { name: String, } -impl From for ModioModResponseTag { - fn from(value: modio::mods::Tag) -> Self { - Self { name: value.name } - } -} - -impl From for ModioModResponse { - fn from(value: modio::mods::Mod) -> Self { - Self { - id: value.id, - name_id: value.name_id, - name: value.name, - profile_url: value.profile_url.to_string(), - modfile: value.modfile.map(Into::into), - tags: value.tags.into_iter().map(Into::into).collect(), - } - } -} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ModioFileResponse { @@ -161,29 +141,10 @@ pub struct ModioFileResponse { filesize: u64, download: ModioFileResponseDownload, } -impl From for ModioFileResponse { - fn from(value: modio::files::File) -> Self { - Self { - id: value.id, - date_added: value.date_added, - version: value.version, - changelog: value.changelog, - filesize: value.filesize, - download: value.download.into(), - } - } -} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ModioFileResponseDownload { binary_url: String, } -impl From for ModioFileResponseDownload { - fn from(value: modio::files::Download) -> Self { - Self { - binary_url: value.binary_url.to_string(), - } - } -} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ModioFile { @@ -202,11 +163,6 @@ impl From for ModioFile { } } } -impl From for ModioFile { - fn from(value: modio::files::File) -> Self { - Self::from(ModioFileResponse::from(value)) - } -} #[derive(Default)] struct LoggingMiddleware { @@ -248,14 +204,12 @@ impl Middleware for LoggingMiddleware { pub enum DrgModioError { #[snafu(display("missing OAuth token"))] MissingOauthToken, - #[snafu(display("mod.io error: {source}"))] - GenericModioError { source: modio::Error }, #[snafu(display("reqwest middlware error: {source}"))] GenericReqwestMiddlewareError { source: anyhow::Error }, #[snafu(display("reqwest error: {source}"))] GenericReqwestError { source: reqwest::Error }, #[snafu(display("failed to perform basic mod.io probe: {source}"))] - CheckFailed { source: modio::Error }, + CheckFailed { source: anyhow::Error }, #[snafu(display("failed to fetch mod files for <{url}> (mod_id = {mod_id}): {source}"))] FetchModFilesFailed { source: anyhow::Error, @@ -338,14 +292,19 @@ pub trait DrgModio: Sync + Send { } struct ModioAlt { - modio: modio::Modio, client: reqwest_middleware::ClientWithMiddleware, oauth: String, } impl ModioAlt { - async fn get(&self, endpoint: &str) -> Result { + async fn get>( + &self, + endpoint: U, + ) -> Result { self.client - .get(&format!("https://api.mod.io/v1/games/2475/{endpoint}")) + .get(&format!( + "https://api.mod.io/v1/games/{MODIO_DRG_ID}/{}", + endpoint.as_ref() + )) .header("Authorization", format!("Bearer {}", self.oauth)) .send() .await @@ -401,26 +360,18 @@ impl DrgModio for ModioAlt { .get("oauth") .context(MissingOauthTokenSnafu)? .to_string(); - let modio = modio::Modio::new( - modio::Credentials::with_token( - "".to_owned(), // TODO patch modio to not use API key at all - oauth.clone(), - ), - client.clone(), - ) - .context(GenericModioSnafu)?; - - Ok(Self { - modio, - client, - oauth, - }) + + Ok(Self { client, oauth }) } async fn check(&self) -> Result<(), DrgModioError> { #[derive(Deserialize)] struct Empty {} - self.get::("mods?id=0").await.map(|_| ()) + self.get::("mods?id=0") + .await + .map(|_| ()) + .map_err(Into::::into) + .context(CheckFailedSnafu) } async fn fetch_mod(&self, url: String, mod_id: u32) -> Result { @@ -530,391 +481,50 @@ impl DrgModio for ModioAlt { file_id: u32, ) -> futures::stream::BoxStream<'static, Result> { use futures::StreamExt; - - let res = self - .get::(&format!("mods/{mod_id}/files/{file_id}")) - .await - .unwrap(); - - self.client - .get(res.download.binary_url) - .header("Authorization", format!("Bearer {}", self.oauth)) - .send() - .await - // TODO clean up this nonsense - //.map_err(Into::::into) - //.context(GenericReqwestMiddlewareSnafu) - .unwrap() - .error_for_status() - //.map_err(Into::::into) - //.context(GenericReqwestMiddlewareSnafu) - .unwrap() - .bytes_stream() - .context(GenericReqwestSnafu) - .boxed() - } -} - -#[cfg(test)] -mod test2 { - use super::*; - - static INIT: std::sync::Once = std::sync::Once::new(); - pub fn setup_tracing() { - INIT.call_once(|| { - tracing_subscriber::fmt().init(); - }); - } - - fn client() -> T { - T::with_parameters(&HashMap::from([("oauth".to_string(), "".to_string())])) - .unwrap() - } - fn client_bad() -> T { - T::with_parameters(&HashMap::from([("oauth".to_string(), "bad".to_string())])).unwrap() - } - - #[tokio::test] - async fn test_modio_alt_check() { - setup_tracing(); - - async fn get() -> Option<()> { - client::().check().await.ok() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_check_bad() { - setup_tracing(); - - async fn get() -> Option<()> { - client_bad::().check().await.ok() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_fetch_mod() { - setup_tracing(); - - async fn get() -> ModioMod { - client::() - .fetch_mod("asdf".into(), 1897251) - .await - .unwrap() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_fetch_file() { - setup_tracing(); - - async fn get() -> ModioFileResponse { - client::() - .fetch_file("asdf".into(), 1897251, 4916540) - .await - .unwrap() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_dependencies() { - setup_tracing(); - - async fn get() -> Vec { - client::() - .fetch_dependencies("asdfa".to_string(), 1897251) - .await - .unwrap() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_mods_by_name() { - setup_tracing(); - - async fn get() -> Vec { - client::() - .fetch_mods_by_name("sandbox-utilities") - .await - .unwrap() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_mods_by_ids() { - setup_tracing(); - - async fn get() -> Vec { - let mods = vec![1620474, 3813360, 1618289]; - client::().fetch_mods_by_ids(mods).await.unwrap() - } - - let a = get::().await; - let b = get::().await; - dbg!(&a); - assert_eq!(a, b); - } - - #[tokio::test] - async fn test_modio_alt_mod_updates_since() { - setup_tracing(); - - async fn get() -> HashSet { - let mods = vec![ - 1620474, 3813360, 1618289, 1668540, 1490893, 1162391, 2434441, 2799076, 2191437, - 1839402, 1769472, 2690013, 2478655, 2334132, 1888355, 1034748, 3502887, 3418446, - 2947767, 1130085, 2247462, 1861561, 2152235, 1962912, 2101319, 2450041, 2273035, - 1645602, 1169791, 2408852, 2442215, 3148183, 2751974, 3431533, 1034830, 2101850, - 1034237, 1792770, 2905192, 2990379, 1517852, 2041465, 1770392, 2036912, 1137706, - 1157883, 3097022, 2424882, 2296587, 2348581, 1773996, 2433395, 1727746, 1499941, - 1893389, 1034161, 2771775, 2092879, 3122386, 1035598, 2728755, 1751567, 1582325, - 1130032, 2909497, 1510072, 1753813, 1150682, 1139370, 1533287, 2155079, 1034644, - 2613099, 1911144, 1158572, 2400086, 1637267, 2917335, 2939176, 2190584, 3733289, - 1750422, 2369777, 1034406, 1489631, 1038224, 1863656, 2723280, 1250251, 2503544, - 1122623, 1929699, 1166441, 1138668, 2685922, 1634437, 2125794, 3116431, 1585858, - 3096846, 3190129, 2745837, 1616717, 1399744, 2959011, 3179565, 1636619, 2030854, - 3112892, 1145532, 3146563, 2259069, 2750508, 1564609, 2148763, 3096163, 3205713, - 3363051, 1956173, 2515857, 2942827, 3135932, 1181346, 2180950, 2229868, 1035291, - 1139332, 1122785, 2434442, 2498386, 1533299, 1160469, 2525496, 2536881, 2224215, - 1372030, 3029397, 1835591, 2553961, 1365047, 1593577, 2025849, 2530578, 2435811, - 1664873, 1521216, 1807375, 2080865, 1151344, 2432311, 2133905, 1034587, 1720132, - 1751367, 1877135, 2121639, 2054975, 1960533, 1897251, 3035859, 3094888, 2044707, - 1824234, 2084461, 2986917, 2191191, 1606854, 2478398, 1157000, 1674997, 1160676, - 1831206, 1764999, 2585212, 2965513, 2038940, 1158356, 2025965, 1149629, 2566820, - 2461393, 1519511, 2814623, 3187876, 2767795, 2029130, 2909311, 2071320, 2474953, - 2032299, 2100272, 1033779, 2285617, - ]; - client::() - .fetch_mod_updates_since(mods, 1318552474) + use futures::TryFutureExt; + + let client = self.client.clone(); + let oauth = self.oauth.clone(); + + ({ + let oauth = oauth.clone(); + let client = client.clone(); + || async move { + client + .get(&format!( + "https://api.mod.io/v1/games/{MODIO_DRG_ID}/mods/{mod_id}/files/{file_id}", + )) + .header("Authorization", format!("Bearer {}", oauth)) + .send() + .await + // TODO clean up this nonsense + .map_err(Into::::into) + .context(GenericReqwestMiddlewareSnafu)? + .error_for_status() + .map_err(Into::::into) + .context(GenericReqwestMiddlewareSnafu)? + .json::() + .await + .context(GenericReqwestSnafu) + } + })() + .and_then(|res| async move { + Ok(client + .get(res.download.binary_url) + .header("Authorization", format!("Bearer {}", oauth)) + .send() .await - .unwrap() - } - - let a = get::().await; - let b = get::().await; - assert_eq!(a, b); - } -} - -#[async_trait::async_trait] -impl DrgModio for modio::Modio { - fn with_parameters(parameters: &HashMap) -> Result { - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) - .with::(Default::default()) - .build(); - let modio = modio::Modio::new( - modio::Credentials::with_token( - "".to_owned(), // TODO patch modio to not use API key at all - parameters.get("oauth").context(MissingOauthTokenSnafu)?, - ), - client, - ) - .context(GenericModioSnafu)?; - - Ok(modio) - } - - async fn check(&self) -> Result<(), DrgModioError> { - use modio::filter::Eq; - use modio::mods::filters::Id; - - self.game(MODIO_DRG_ID) - .mods() - .search(Id::eq(0)) - .collect() - .await - .context(CheckFailedSnafu)?; - Ok(()) - } - - async fn fetch_mod(&self, url: String, id: u32) -> Result { - use modio::filter::NotEq; - use modio::mods::filters::Id; - - let files = self - .game(MODIO_DRG_ID) - .mod_(id) - .files() - .search(Id::ne(0)) - .collect() - .await - .map_err(Into::::into) - .with_context(|_| FetchModFilesFailedSnafu { - mod_id: id, - url: url.clone(), - })? - .into_iter() - .map(Into::into) - .collect(); - let r#mod = self - .game(MODIO_DRG_ID) - .mod_(id) - .get() - .await - .map_err(Into::::into) - .context(FetchModFailedSnafu { mod_id: id, url })?; - - Ok(ModioMod::new(r#mod.into(), files)) - } - - async fn fetch_file( - &self, - url: String, - mod_id: u32, - modfile_id: u32, - ) -> Result { - let file = self - .game(MODIO_DRG_ID) - .mod_(mod_id) - .file(modfile_id) - .get() - .await - .map_err(Into::::into) - .with_context(|_| FetchModFileFailedSnafu { - url, - mod_id, - modfile_id, - })?; - Ok(file.into()) - } - - async fn fetch_dependencies( - &self, - url: String, - mod_id: u32, - ) -> Result, DrgModioError> { - Ok(self - .game(MODIO_DRG_ID) - .mod_(mod_id) - .dependencies() - .list() - .await - .map_err(Into::::into) - .with_context(|_| FetchDependenciesFailedSnafu { url, mod_id })? - .into_iter() - .map(|d| d.mod_id) - .collect::>()) - } - - async fn fetch_mods_by_name( - &self, - name_id: &str, - ) -> Result, DrgModioError> { - use modio::filter::{Eq, In}; - use modio::mods::filters::{NameId, Visible}; - - let filter = NameId::eq(name_id).and(Visible::_in(vec![0, 1])); - Ok(self - .game(MODIO_DRG_ID) - .mods() - .search(filter) - .collect() - .await - .context(GenericModioSnafu)? - .into_iter() - .map(|m| m.into()) - .collect()) - } - - async fn fetch_mods_by_ids( - &self, - filter_ids: Vec, - ) -> Result, DrgModioError> { - use modio::filter::In; - use modio::mods::filters::Id; - - let filter = Id::_in(filter_ids); - - Ok(self - .game(MODIO_DRG_ID) - .mods() - .search(filter) - .collect() - .await - .context(GenericModioSnafu)? - .into_iter() - .map(|m| m.into()) - .collect()) - } - - async fn fetch_mod_updates_since( - &self, - mod_ids: Vec, - last_update: u64, - ) -> Result, DrgModioError> { - use modio::filter::Cmp; - use modio::filter::In; - use modio::filter::NotIn; - - use modio::mods::filters::events::EventType; - use modio::mods::filters::events::ModId; - use modio::mods::filters::DateAdded; - use modio::mods::EventType as EventTypes; - - let events = self - .game(MODIO_DRG_ID) - .mods() - .events( - EventType::not_in(vec![ - EventTypes::ModCommentAdded, - EventTypes::ModCommentDeleted, - ]) - .and(ModId::_in(mod_ids)) - .and(DateAdded::gt(last_update)), - ) - .collect() - .await - .context(GenericModioSnafu)?; - Ok(events.iter().map(|e| e.mod_id).collect::>()) - } - - async fn download( - &self, - mod_id: u32, - file_id: u32, - ) -> futures::stream::BoxStream<'static, Result> { - let download = modio::download::DownloadAction::File { - game_id: MODIO_DRG_ID, - mod_id, - file_id, - }; - - use futures::StreamExt; - - self.download(download) - .stream() - .context(GenericModioSnafu) - .boxed() + // TODO clean up this nonsense + .map_err(Into::::into) + .context(GenericReqwestMiddlewareSnafu)? + .error_for_status() + .map_err(Into::::into) + .context(GenericReqwestMiddlewareSnafu)? + .bytes_stream() + .context(GenericReqwestSnafu)) + }) + .try_flatten_stream() + .boxed() } } From c9613cb3a85effc0d8c7f955fec7e82cb691479e Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 18 Jun 2024 21:12:01 -0500 Subject: [PATCH 6/6] remove println --- src/providers/modio.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/providers/modio.rs b/src/providers/modio.rs index 944d02ca..4060aa82 100644 --- a/src/providers/modio.rs +++ b/src/providers/modio.rs @@ -340,7 +340,6 @@ impl ModioAlt { let url = format!("{endpoint}?_offset={offset}&{query}"); let page: ResultPage = self.get(&url).await?; results.extend(page.data); - println!("req {offset} {} {}", results.len(), page.result_offset); if results.len() == page.result_total as usize { break; }