diff --git a/Cargo.lock b/Cargo.lock index 80081c23..87472d56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2886,6 +2886,27 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51066092dc750f8cbcac4afa04592377bf7463a911d1beca430ddd788deb0757" +[[package]] +name = "htmd" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eee9b00ee2e599b4f86507157e3db786e7a3319fc225f0e9584151dbea2291d" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "phf", +] + +[[package]] +name = "html5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "http" version = "0.2.12" @@ -3676,6 +3697,29 @@ dependencies = [ "libc", ] +[[package]] +name = "markup5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.38.0+unofficial" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333171ccdf66e915257740d44e38ea5b1b19ce7b45d33cc35cb6f118fbd981ff" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + [[package]] name = "matchers" version = "0.2.0" @@ -4556,6 +4600,7 @@ dependencies = [ "dunce", "futures-lite", "futures-util", + "htmd", "image", "indicatif", "interfrost", @@ -4743,6 +4788,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + [[package]] name = "phf_generator" version = "0.13.1" @@ -4910,6 +4965,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.2.37" @@ -6465,6 +6526,31 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -6579,6 +6665,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -7576,6 +7671,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "075474b12bcb3d2e3d4546580e9de478eeeead668a1761e2a8860c836b7ef297" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" @@ -8283,6 +8390,16 @@ version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" +[[package]] +name = "xml5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dc9559429edf0cd3f327cc0afd9d6b36fa8cec6d93107b7fbe64f806b5f2d9" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "y4m" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 3a93c211..f3aa2b41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,6 +122,7 @@ strum = {version = "=0.28.0", features = ["derive"]} syn = {version = "=2.0.117"} image = {version = "=0.25.10", default-features = false, features = ["png", "jpeg"]} +htmd = {version = "=0.5.4"} arboard = {version = "=3.6.1"} trash = {version = "=5.2.6"} diff --git a/packages/oneclient_app/src/hooks/queries/packages.rs b/packages/oneclient_app/src/hooks/queries/packages.rs index 1ca3c6b2..0a86189e 100644 --- a/packages/oneclient_app/src/hooks/queries/packages.rs +++ b/packages/oneclient_app/src/hooks/queries/packages.rs @@ -134,7 +134,7 @@ impl QueryCapability for PackageProjectQuery { let state = LauncherState::get()?; let provider = state.services.packages.get(keys.provider)?; provider - .get_project(&keys.project_id, &state.services) + .get_project_with_body(&keys.project_id, &state.services) .await } } diff --git a/packages/oneclient_core/Cargo.toml b/packages/oneclient_core/Cargo.toml index 0044b11a..c246a8d2 100644 --- a/packages/oneclient_core/Cargo.toml +++ b/packages/oneclient_core/Cargo.toml @@ -99,6 +99,9 @@ async-compression.workspace = true # image decode/resize image.workspace = true +# used to confirm html to markdown +htmd.workspace = true + # move-to-trash for deletions trash.workspace = true diff --git a/packages/oneclient_core/src/api_config.rs b/packages/oneclient_core/src/api_config.rs index f1182f07..2a0b97f0 100644 --- a/packages/oneclient_core/src/api_config.rs +++ b/packages/oneclient_core/src/api_config.rs @@ -36,10 +36,6 @@ pub fn meta_url_base() -> String { .to_string() } -pub fn curseforge_headers() -> Vec<(String, String)> { - vec![("x-api-key".to_string(), curseforge_api_key())] -} - pub fn modrinth_headers() -> Vec<(String, String)> { match modrinth_api_key() { Some(token) => vec![("Authorization".to_string(), token)], diff --git a/packages/oneclient_core/src/http/service.rs b/packages/oneclient_core/src/http/service.rs index d108865e..357867a2 100644 --- a/packages/oneclient_core/src/http/service.rs +++ b/packages/oneclient_core/src/http/service.rs @@ -31,6 +31,42 @@ fn backoff_delay(attempt: u32) -> std::time::Duration { std::time::Duration::from_millis(capped + jitter) } +fn is_curseforge_host(url: &reqwest::Url) -> bool { + if url.scheme() != "https" { + return false; + } + + let Some(host) = url.host_str() else { + return false; + }; + let host = host.trim_end_matches('.').to_ascii_lowercase(); + + matches!(host.as_str(), "curseforge.com" | "forgecdn.net") + || host.ends_with(".curseforge.com") + || host.ends_with(".forgecdn.net") +} + +fn apply_curseforge_auth(request: &mut reqwest::Request) -> Result<(), RequestError> { + const API_KEY_HEADER: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-api-key"); + + if !is_curseforge_host(request.url()) { + return Ok(()); + } + + let headers = request.headers_mut(); + if headers.contains_key(&API_KEY_HEADER) { + return Ok(()); + } + + let mut value = + reqwest::header::HeaderValue::try_from(crate::api_config::curseforge_api_key())?; + value.set_sensitive(true); + headers.insert(API_KEY_HEADER, value); + + Ok(()) +} + fn body_snippet(bytes: &[u8]) -> String { const MAX: usize = 240; if bytes.is_empty() { @@ -89,10 +125,12 @@ impl RequestClient { impl RequestClient { #[tracing::instrument(level = "debug", skip_all)] pub async fn send(&self, request: impl Into) -> Result { - let request: HttpRequest = request.into(); + let mut request: HttpRequest = request.into(); let mut retries = 0; let mut throttle_retries = 0u32; + apply_curseforge_auth(&mut request.request)?; + let cloned_backup = request.request.try_clone(); let cloneable = cloned_backup.is_some(); @@ -235,3 +273,59 @@ impl RequestClient { self.send_as(request).await } } + +#[cfg(test)] +mod tests { + use super::*; + + fn request(url: &str) -> reqwest::Request { + reqwest::Request::new(reqwest::Method::GET, url.parse().unwrap()) + } + + fn api_key_of(url: &str) -> Option { + let mut req = request(url); + apply_curseforge_auth(&mut req).unwrap(); + req.headers() + .get("x-api-key") + .map(|v| v.to_str().unwrap().to_string()) + } + + #[test] + fn attaches_key_to_curseforge_api_and_cdn() { + for url in [ + "https://api.curseforge.com/v1/mods/search", + "https://media.forgecdn.net/avatars/1/2/icon.png", + "https://edge.forgecdn.net/files/1/2/mod.jar", + "https://www.curseforge.com/minecraft", + "https://forgecdn.net/thing", + ] { + assert_eq!( + api_key_of(url).as_deref(), + Some(crate::constants::CURSEFORGE_API_KEY), + "expected key on {url}" + ); + } + } + + #[test] + fn ignores_unrelated_and_insecure_hosts() { + for url in [ + "https://api.modrinth.com/v2/search", + "https://cdn.modrinth.com/data/x/y.jar", + "https://notcurseforge.com/x", + "https://evil-forgecdn.net.attacker.com/x", + "http://media.forgecdn.net/insecure.png", + ] { + assert_eq!(api_key_of(url), None, "unexpected key on {url}"); + } + } + + #[test] + fn does_not_override_explicit_key() { + let mut req = request("https://api.curseforge.com/v1/mods"); + req.headers_mut() + .insert("x-api-key", reqwest::header::HeaderValue::from_static("mine")); + apply_curseforge_auth(&mut req).unwrap(); + assert_eq!(req.headers()["x-api-key"], "mine"); + } +} diff --git a/packages/oneclient_core/src/packages/provider/curseforge.rs b/packages/oneclient_core/src/packages/provider/curseforge.rs index 1108a965..905c063d 100644 --- a/packages/oneclient_core/src/packages/provider/curseforge.rs +++ b/packages/oneclient_core/src/packages/provider/curseforge.rs @@ -6,8 +6,7 @@ use serde::Deserialize; use url::Url; use super::PackageProvider; -use super::http::fetch_json_with_headers; -use crate::api_config::curseforge_headers; +use super::http::fetch_json; use crate::LauncherResult; use crate::constants::{CURSEFORGE_API_URL, CURSEFORGE_GAME_ID}; use crate::packages::domain::{ContentType, GameLoader, ProviderId}; @@ -74,12 +73,11 @@ impl PackageProvider for CurseForgeProvider { } } - let response: CfPaged> = fetch_json_with_headers( + let response: CfPaged> = fetch_json( &services.requester, Method::GET, url.as_str(), None, - &curseforge_headers(), ) .await?; @@ -101,17 +99,27 @@ impl PackageProvider for CurseForgeProvider { project_id: &str, services: &LauncherServices, ) -> LauncherResult { - let response: CfData = fetch_json_with_headers( + let response: CfData = fetch_json( &services.requester, Method::GET, &api_url(&format!("/mods/{project_id}")), None, - &curseforge_headers(), ) .await?; Ok(response.data.into_detail()) } + #[tracing::instrument(level = "debug", skip(self, services))] + async fn get_project_with_body( + &self, + project_id: &str, + services: &LauncherServices, + ) -> LauncherResult { + let mut detail = self.get_project(project_id, services).await?; + detail.body = fetch_body(project_id, &detail, services).await; + Ok(detail) + } + #[tracing::instrument(level = "debug", skip(self, project_ids, services))] async fn get_projects( &self, @@ -120,12 +128,11 @@ impl PackageProvider for CurseForgeProvider { ) -> LauncherResult> { let mod_ids: Vec = project_ids.iter().filter_map(|s| s.parse().ok()).collect(); let body = serde_json::json!({ "modIds": mod_ids }); - let response: CfData> = fetch_json_with_headers( + let response: CfData> = fetch_json( &services.requester, Method::POST, &api_url("/mods"), Some(body), - &curseforge_headers(), ) .await?; Ok(response.data.into_iter().map(CfMod::into_detail).collect()) @@ -151,12 +158,11 @@ impl PackageProvider for CurseForgeProvider { params.append_pair("modLoaderType", &t.to_string()); } } - let response: CfPaged> = fetch_json_with_headers( + let response: CfPaged> = fetch_json( &services.requester, Method::GET, url.as_str(), None, - &curseforge_headers(), ) .await?; @@ -192,12 +198,11 @@ impl PackageProvider for CurseForgeProvider { url.query_pairs_mut() .append_pair("gameId", &CURSEFORGE_GAME_ID.to_string()) .append_pair("classId", &cf_class_id(content_type).to_string()); - let response: CfData> = fetch_json_with_headers( + let response: CfData> = fetch_json( &services.requester, Method::GET, url.as_str(), None, - &curseforge_headers(), ) .await?; Ok(response.data.into_iter().map(|c| c.name).collect()) @@ -210,12 +215,11 @@ impl PackageProvider for CurseForgeProvider { version_id: &str, services: &LauncherServices, ) -> LauncherResult { - let response: CfData = fetch_json_with_headers( + let response: CfData = fetch_json( &services.requester, Method::GET, &api_url(&format!("/mods/files/{version_id}")), None, - &curseforge_headers(), ) .await?; Ok(response.data.into()) @@ -229,12 +233,11 @@ impl PackageProvider for CurseForgeProvider { ) -> LauncherResult> { let file_ids: Vec = version_ids.iter().filter_map(|s| s.parse().ok()).collect(); let body = serde_json::json!({ "fileIds": file_ids }); - let response: CfData> = fetch_json_with_headers( + let response: CfData> = fetch_json( &services.requester, Method::POST, &api_url("/mods/files"), Some(body), - &curseforge_headers(), ) .await?; Ok(response.data.into_iter().map(Into::into).collect()) @@ -270,12 +273,11 @@ impl PackageProvider for CurseForgeProvider { file: CfFile, } - let response: CfData = fetch_json_with_headers( + let response: CfData = fetch_json( &services.requester, Method::POST, &api_url(&format!("/mods/fingerprints/{CURSEFORGE_GAME_ID}")), Some(body), - &curseforge_headers(), ) .await?; @@ -310,6 +312,54 @@ struct CfPaged { pagination: CfPagination, } +#[tracing::instrument(level = "debug", skip(detail, services))] +async fn fetch_body( + project_id: &str, + detail: &ProjectDetail, + services: &LauncherServices, +) -> PackageBody { + let fallback = || { + detail + .links + .iter() + .find(|(label, _)| label == "Website") + .map(|(_, url)| PackageBody::Url(url.clone())) + .unwrap_or_else(|| PackageBody::Raw(String::new())) + }; + + let response: Result, _> = fetch_json( + &services.requester, + Method::GET, + &api_url(&format!("/mods/{project_id}/description")), + None, + ) + .await; + + match response { + Ok(response) => match html_to_markdown(&response.data) { + Some(markdown) => PackageBody::Raw(markdown), + None => fallback(), + }, + Err(err) => { + tracing::warn!(project_id, %err, "failed to fetch curseforge description"); + fallback() + } + } +} + +fn html_to_markdown(html: &str) -> Option { + let markdown = htmd::convert(html).ok()?; + let markdown = markdown.trim(); + + (!markdown.is_empty()).then(|| absolutize_links(markdown)) +} + +fn absolutize_links(markdown: &str) -> String { + markdown + .replace("](/linkout?remoteUrl=", "](https://www.curseforge.com/linkout?remoteUrl=") + .replace("](/", "](https://www.curseforge.com/") +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct CfPagination { diff --git a/packages/oneclient_core/src/packages/provider/mod.rs b/packages/oneclient_core/src/packages/provider/mod.rs index 7f08b220..46bda9ef 100644 --- a/packages/oneclient_core/src/packages/provider/mod.rs +++ b/packages/oneclient_core/src/packages/provider/mod.rs @@ -32,6 +32,18 @@ pub trait PackageProvider: Send + Sync { services: &LauncherServices, ) -> LauncherResult; + /// Same as [`PackageProvider::get_project`], but guarantees [`ProjectDetail::body`] + /// is populated when the provider has one. Only worth calling when the body is + /// actually rendered: providers that serve descriptions separately pay an extra + /// request here, so install paths should stay on `get_project`. + async fn get_project_with_body( + &self, + project_id: &str, + services: &LauncherServices, + ) -> LauncherResult { + self.get_project(project_id, services).await + } + async fn get_projects( &self, project_ids: &[String], diff --git a/packages/oneclient_core/tests/providers.rs b/packages/oneclient_core/tests/providers.rs index 39b2fbf2..e8151e22 100644 --- a/packages/oneclient_core/tests/providers.rs +++ b/packages/oneclient_core/tests/providers.rs @@ -1,7 +1,7 @@ use oneclient_core::dev; use oneclient_core::packages::domain::{ContentType, ProviderId}; -use oneclient_core::packages::types::SearchFilters; +use oneclient_core::packages::types::{PackageBody, SearchFilters}; #[test] fn registry_get_unknown_provider_errors() { @@ -116,3 +116,49 @@ async fn modrinth_lookup_version_by_sha1() { let (provider_id, _) = found.unwrap(); assert_eq!(provider_id, ProviderId::Modrinth); } + +#[tokio::test] +#[ignore = "requires network"] +async fn curseforge_get_project_fetches_markdown_body() { + let env = dev::ephemeral_services().await.unwrap(); + let provider = env.packages.get(ProviderId::CurseForge).unwrap(); + + // 238222 = Just Enough Items (JEI) + let project = provider + .get_project_with_body("238222", &env) + .await + .unwrap(); + assert_eq!(project.provider, ProviderId::CurseForge); + + let body = match &project.body { + PackageBody::Raw(body) => body, + PackageBody::Url(url) => panic!("expected a fetched body, got a link to {url}"), + }; + + assert!(!body.is_empty(), "body should not be empty"); + assert!( + !body.contains("

") && !body.contains(" assert!(!body.is_empty(), "body should not be empty"), + PackageBody::Url(url) => panic!("expected a fetched body, got a link to {url}"), + } +}