diff --git a/apps/oneclient/frontend/src/bindings.gen.ts b/apps/oneclient/frontend/src/bindings.gen.ts index 9ce578aa..e4bf07b3 100644 --- a/apps/oneclient/frontend/src/bindings.gen.ts +++ b/apps/oneclient/frontend/src/bindings.gen.ts @@ -301,8 +301,11 @@ export type VersionType = */ "old_beta" -const ARGS_MAP = { 'events':'{"message":["event"],"process":["event"],"ingress":["event"]}', 'folders':'{"fromCluster":["folder_name"],"openCluster":["folder_name"]}', 'oneclient':'{"getClustersGroupedByMajor":[],"getBundlesFor":["cluster_id"],"getVersions":[]}', 'debug':'{"openDevTools":[],"getType":[],"getOsVersion":[],"isInDev":[],"getArch":[],"getFamily":[],"getPackageVersion":[],"getLocale":[],"getPlatform":[],"getBuildTimestamp":[],"getGitCommitHash":[]}', 'core':'{"convertUsernameUUID":["username_uuid"],"getMultiplePackages":["provider","slugs"],"removeCluster":["id"],"getLinkedPackages":["cluster_id"],"getClusterById":["id"],"getDefaultUser":["fallback"],"createCluster":["options"],"getRunningProcesses":[],"openMsaLogin":[],"getGlobalProfile":[],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"getScreenshots":["id"],"changeSkin":["access_token","skin_url","skin_variant"],"getProfileOrDefault":["name"],"removeCape":["access_token"],"getClusters":[],"getUser":["uuid"],"killProcess":["pid"],"getUsers":[],"setDiscordRPCMessage":["message"],"open":["input"],"getLogs":["id"],"getPackage":["provider","slug"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"getPackageBody":["provider","body"],"isClusterRunning":["cluster_id"],"getGameVersions":[],"createSettingsProfile":["name"],"getUsersFromAuthor":["provider","author"],"getRunningProcessesByClusterId":["cluster_id"],"installModpack":["modpack","cluster_id"],"getLoadersForVersion":["mc_version"],"changeCape":["access_token","cape_uuid"],"updateClusterById":["id","request"],"removePackage":["cluster_id","package_hash"],"getWorlds":["id"],"readSettings":[],"removeUser":["uuid"],"fetchLoggedInProfile":["access_token"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getLogByName":["id","name"],"searchPackages":["provider","query"],"fetchMinecraftProfile":["uuid"],"launchCluster":["id","uuid","search_for_java"],"updateClusterProfile":["name","profile"],"writeSettings":["setting"],"setDefaultUser":["uuid"]}' } -export type Router = { 'oneclient': { getClustersGroupedByMajor: () => Promise>, +const ARGS_MAP = { 'debug':'{"getPackageVersion":[],"isInDev":[],"getType":[],"getGitCommitHash":[],"getArch":[],"getBuildTimestamp":[],"openDevTools":[],"getPlatform":[],"getFamily":[],"getOsVersion":[],"getLocale":[]}', 'folders':'{"openCluster":["folder_name"],"fromCluster":["folder_name"]}', 'oneclient':'{"getBundlesFor":["cluster_id"],"getVersions":[],"getClustersGroupedByMajor":[]}', 'events':'{"message":["event"],"process":["event"],"ingress":["event"]}', 'core':'{"installModpack":["modpack","cluster_id"],"removeCape":["access_token"],"killProcess":["pid"],"getLogByName":["id","name"],"getGlobalProfile":[],"removeCluster":["id"],"getUsersFromAuthor":["provider","author"],"getPackage":["provider","slug"],"getUser":["uuid"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getRunningProcesses":[],"readSettings":[],"getPackageBody":["provider","body"],"removePackage":["cluster_id","package_hash"],"setDiscordRPCMessage":["message"],"searchPackages":["provider","query"],"getMultiplePackages":["provider","slugs"],"updateClusterById":["id","request"],"writeSettings":["setting"],"isClusterRunning":["cluster_id"],"getProfileOrDefault":["name"],"getGameVersions":[],"getClusters":[],"getLogs":["id"],"updateClusterProfile":["name","profile"],"removeUser":["uuid"],"getDefaultUser":["fallback"],"setDefaultUser":["uuid"],"getScreenshots":["id"],"getLoadersForVersion":["mc_version"],"createCluster":["options"],"createSettingsProfile":["name"],"getWorlds":["id"],"openMsaLogin":[],"getLinkedPackages":["cluster_id"],"fetchMinecraftProfile":["uuid"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"changeCape":["access_token","cape_uuid"],"getUsers":[],"open":["input"],"fetchLoggedInProfile":["access_token"],"convertUsernameUUID":["username_uuid"],"launchCluster":["id","uuid","search_for_java"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"getRunningProcessesByClusterId":["cluster_id"],"changeSkin":["access_token","skin_url","skin_variant"],"getClusterById":["id"]}' } +export type Router = { 'events': { ingress: (event: IngressPayload) => Promise, +message: (event: MessagePayload) => Promise, +process: (event: ProcessPayload) => Promise }, +'oneclient': { getClustersGroupedByMajor: () => Promise>, getBundlesFor: (clusterId: number) => Promise, getVersions: () => Promise }, 'folders': { fromCluster: (folderName: string) => Promise, @@ -365,10 +368,7 @@ changeCape: (accessToken: string, capeUuid: string) => Promise Promise, convertUsernameUUID: (usernameUuid: string) => Promise, setDiscordRPCMessage: (message: string) => Promise, -open: (input: string) => Promise }, -'events': { ingress: (event: IngressPayload) => Promise, -message: (event: MessagePayload) => Promise, -process: (event: ProcessPayload) => Promise } }; +open: (input: string) => Promise } }; export type { InferCommandOutput } diff --git a/packages/core/src/api/packages/provider/modrinth/mod.rs b/packages/core/src/api/packages/provider/modrinth/mod.rs index 7899f6eb..5bf599d9 100644 --- a/packages/core/src/api/packages/provider/modrinth/mod.rs +++ b/packages/core/src/api/packages/provider/modrinth/mod.rs @@ -1,5 +1,7 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; use super::ProviderExt; use crate::api::packages::categories::ToProviderCategory; @@ -35,6 +37,12 @@ macro_rules! url_v3 { }; } +static PACKAGE_CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +static VERSION_CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +const CACHE_DURATION: Duration = Duration::from_secs(600); + #[derive(Default)] pub struct ModrinthProviderImpl; @@ -120,64 +128,172 @@ impl ProviderExt for ModrinthProviderImpl { } async fn get(&self, slug: &str) -> LauncherResult { - Ok( + { + let mut cache = PACKAGE_CACHE.lock().unwrap(); + if let Some((package, timestamp)) = cache.get(slug) { + if timestamp.elapsed() < CACHE_DURATION { + return Ok(package.clone()); + } else { + cache.remove(slug); + } + } + } + + let package: ManagedPackage = http::fetch_json::(Method::GET, url!("/project/{}", slug), None, None) .await? - .into(), - ) + .into(); + + { + let mut cache = PACKAGE_CACHE.lock().unwrap(); + cache.insert(package.id.clone(), (package.clone(), Instant::now())); + cache.insert(package.slug.clone(), (package.clone(), Instant::now())); + } + + Ok(package) } async fn get_multiple(&self, slugs: &[String]) -> LauncherResult> { - Ok(http::fetch_json::>( - Method::GET, - url!( - "/projects?ids=[{}]", - slugs - .iter() - .map(|id| format!("\"{id}\"")) - .collect::>() - .join(",") - ), - None, - None, - ) - .await? - .into_iter() - .map(Into::into) - .collect()) + let mut packages = Vec::new(); + let mut missing_slugs = Vec::new(); + + let unique_slugs: HashSet<&String> = slugs.iter().collect(); + + { + let mut cache = PACKAGE_CACHE.lock().unwrap(); + for slug in unique_slugs { + if let Some((package, timestamp)) = cache.get(slug) { + if timestamp.elapsed() < CACHE_DURATION { + packages.push(package.clone()); + } else { + cache.remove(slug); + missing_slugs.push(slug.clone()); + } + } else { + missing_slugs.push(slug.clone()); + } + } + } + + if !missing_slugs.is_empty() { + let fetched_packages: Vec = http::fetch_json::>( + Method::GET, + url!( + "/projects?ids=[{}]", + missing_slugs + .iter() + .map(|id| format!("\"{id}\"")) + .collect::>() + .join(",") + ), + None, + None, + ) + .await? + .into_iter() + .map(Into::into) + .collect(); + + { + let mut cache = PACKAGE_CACHE.lock().unwrap(); + for package in &fetched_packages { + cache.insert(package.id.clone(), (package.clone(), Instant::now())); + cache.insert(package.slug.clone(), (package.clone(), Instant::now())); + } + } + + packages.extend(fetched_packages); + } + + Ok(packages) } async fn get_versions_by_hashes( &self, hashes: &[String], ) -> LauncherResult> { - let body = serde_json::json!({ - "hashes": hashes, - "algorithm": "sha1" - }); + let mut results = HashMap::new(); + let mut missing_hashes = Vec::new(); + let unique_hashes: HashSet<&String> = hashes.iter().collect(); - let results = http::fetch_json::>( - Method::POST, - url!("/version_files"), - Some(body), - None, - ) - .await?; + { + let mut cache = VERSION_CACHE.lock().unwrap(); + for hash in unique_hashes { + if let Some((version, timestamp)) = cache.get(hash) { + if timestamp.elapsed() < CACHE_DURATION { + results.insert(hash.clone(), version.clone()); + } else { + cache.remove(hash); + missing_hashes.push(hash.clone()); + } + } else { + missing_hashes.push(hash.clone()); + } + } + } - Ok(results.into_iter().map(|(k, v)| (k, v.into())).collect()) + if !missing_hashes.is_empty() { + let body = serde_json::json!({ + "hashes": missing_hashes, + "algorithm": "sha1" + }); + + let fetched_results = http::fetch_json::>( + Method::POST, + url!("/version_files"), + Some(body), + None, + ) + .await?; + + { + let mut cache = VERSION_CACHE.lock().unwrap(); + for (hash, version) in fetched_results { + let managed: ManagedVersion = version.into(); + cache.insert(hash.clone(), (managed.clone(), Instant::now())); + cache.insert( + managed.version_id.clone(), + (managed.clone(), Instant::now()), + ); + results.insert(hash, managed); + } + } + } + + Ok(results) } async fn get_version_by_hash(&self, hash: &str) -> LauncherResult> { - Ok(Some( - http::fetch_json::( - Method::GET, - url!("/version_file/{hash}"), - None, - None, - ) - .await? - .into(), - )) + { + let mut cache = VERSION_CACHE.lock().unwrap(); + if let Some((version, timestamp)) = cache.get(hash) { + if timestamp.elapsed() < CACHE_DURATION { + return Ok(Some(version.clone())); + } else { + cache.remove(hash); + } + } + } + + let version: ManagedVersion = http::fetch_json::( + Method::GET, + url!("/version_file/{hash}"), + None, + None, + ) + .await? + .into(); + + { + let mut cache = VERSION_CACHE.lock().unwrap(); + cache.insert(hash.to_string(), (version.clone(), Instant::now())); + cache.insert( + version.version_id.clone(), + (version.clone(), Instant::now()), + ); + } + + Ok(Some(version)) } // async fn get_org_projects(&self, slug: &str) -> LauncherResult> { @@ -301,38 +417,92 @@ impl ProviderExt for ModrinthProviderImpl { let response = http::fetch_json::>(Method::GET, url.as_str(), None, None).await?; - let paginated = response + let mut items = Vec::new(); + { + let mut cache = VERSION_CACHE.lock().unwrap(); + for version in response { + let managed: ManagedVersion = version.into(); + cache.insert( + managed.version_id.clone(), + (managed.clone(), Instant::now()), + ); + for file in &managed.files { + cache.insert(file.sha1.clone(), (managed.clone(), Instant::now())); + } + items.push(managed); + } + } + + let paginated = items .into_iter() .skip(offset) .take(limit) - .collect::>(); + .collect::>(); Ok(Paginated { offset, limit, total: paginated.len(), - items: paginated.into_iter().map(Into::into).collect(), + items: paginated, }) } async fn get_versions(&self, slugs: &[String]) -> LauncherResult> { - Ok(http::fetch_json::>( - Method::GET, - url!( - "/versions?ids=[{}]", - slugs - .iter() - .map(|v| format!("\"{v}\"")) - .collect::>() - .join(",") - ), - None, - None, - ) - .await? - .into_iter() - .map(Into::into) - .collect()) + let mut versions = Vec::new(); + let mut missing_slugs = Vec::new(); + let unique_slugs: HashSet<&String> = slugs.iter().collect(); + + { + let mut cache = VERSION_CACHE.lock().unwrap(); + for slug in unique_slugs { + if let Some((version, timestamp)) = cache.get(slug) { + if timestamp.elapsed() < CACHE_DURATION { + versions.push(version.clone()); + } else { + cache.remove(slug); + missing_slugs.push(slug.clone()); + } + } else { + missing_slugs.push(slug.clone()); + } + } + } + + if !missing_slugs.is_empty() { + let fetched_versions = http::fetch_json::>( + Method::GET, + url!( + "/versions?ids=[{}]", + missing_slugs + .iter() + .map(|v| format!("\"{v}\"")) + .collect::>() + .join(",") + ), + None, + None, + ) + .await? + .into_iter() + .map(Into::into) + .collect::>(); + + { + let mut cache = VERSION_CACHE.lock().unwrap(); + for version in &fetched_versions { + cache.insert( + version.version_id.clone(), + (version.clone(), Instant::now()), + ); + for file in &version.files { + cache.insert(file.sha1.clone(), (version.clone(), Instant::now())); + } + } + } + versions.extend(fetched_versions); + } + + Ok(versions) } } diff --git a/packages/core/src/utils/http.rs b/packages/core/src/utils/http.rs index 48157145..d3f4f91e 100644 --- a/packages/core/src/utils/http.rs +++ b/packages/core/src/utils/http.rs @@ -76,9 +76,18 @@ pub fn build_request(method: Method, url: &str) -> RequestBuilder { client.request(method.clone(), url) } +fn get_ratelimit_reset(headers: &reqwest::header::HeaderMap) -> Option { + headers + .get("X-Ratelimit-Reset") + .or(headers.get("retry-after")) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map(|s| s.ceil() as u64) +} + pub async fn send_request(request: reqwest::Request) -> LauncherResult { if let Some(host) = request.url().host_str() { - if host.ends_with("modrinth.com") { + if host == "api.modrinth.com" { let start = std::time::Instant::now(); MODRINTH_RATE_LIMITER.until_ready().await; tracing::debug!("waited {:?} for modrinth rate limiter", start.elapsed()); @@ -92,6 +101,12 @@ pub async fn send_request(request: reqwest::Request) -> LauncherResult LauncherResult break res, + Ok(res) => { + if res.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { + if let Some(reset) = get_ratelimit_reset(res.headers()) { + tracing::warn!( + "Rate limited. Waiting for {reset} seconds before retrying..." + ); + tokio::time::sleep(std::time::Duration::from_secs(reset)).await; + continue; + } + } + break res; + } } }; @@ -134,7 +160,7 @@ pub async fn fetch_advanced( ) -> LauncherResult { if let Ok(parsed_url) = reqwest::Url::parse(url) { if let Some(host) = parsed_url.host_str() { - if host.ends_with("modrinth.com") { + if host == "api.modrinth.com" { let start = std::time::Instant::now(); MODRINTH_RATE_LIMITER.until_ready().await; tracing::debug!("waited {:?} for modrinth rate limiter", start.elapsed()); @@ -148,6 +174,7 @@ pub async fn fetch_advanced( let client = &REQWEST_CLIENT; let mut attempt = 0; + tracing::debug!("fetching {url} with method {method:?}"); let res = loop { attempt += 1; let mut req = client.request(method.clone(), url); @@ -173,7 +200,18 @@ pub async fn fetch_advanced( return Err(err.into()); } - Ok(res) => break res, + Ok(res) => { + if res.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { + if let Some(reset) = get_ratelimit_reset(res.headers()) { + tracing::warn!( + "Rate limited. Waiting for {reset} seconds before retrying..." + ); + tokio::time::sleep(std::time::Duration::from_secs(reset)).await; + continue; + } + } + break res; + } } };