oiupoyt/OneLauncher-Cracked
1
1
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity

implement caching for modrinth projects/versions

Browse source
This commit is contained in:
Wyvest 2025-12-26 21:46:05 +09:00
parent d69b652726
commit 5db97bc9ed
3 changed files with 281 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
apps/oneclient/frontend/src/bindings.gen.ts
View file

@ -301,8 +301,11 @@ export type VersionType =
*/
"old_beta"
const ARGS_MAP = { 'events':'{"message":["event"],"process":["event"],"ingress":["event"]}', 'folders':'{"fromCluster":["folder_name"],"openCluster":["folder_name"]}', 'oneclient':'{"getClustersGroupedByMajor":[],"getBundlesFor":["cluster_id"],"getVersions":[]}', 'debug':'{"openDevTools":[],"getType":[],"getOsVersion":[],"isInDev":[],"getArch":[],"getFamily":[],"getPackageVersion":[],"getLocale":[],"getPlatform":[],"getBuildTimestamp":[],"getGitCommitHash":[]}', 'core':'{"convertUsernameUUID":["username_uuid"],"getMultiplePackages":["provider","slugs"],"removeCluster":["id"],"getLinkedPackages":["cluster_id"],"getClusterById":["id"],"getDefaultUser":["fallback"],"createCluster":["options"],"getRunningProcesses":[],"openMsaLogin":[],"getGlobalProfile":[],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"getScreenshots":["id"],"changeSkin":["access_token","skin_url","skin_variant"],"getProfileOrDefault":["name"],"removeCape":["access_token"],"getClusters":[],"getUser":["uuid"],"killProcess":["pid"],"getUsers":[],"setDiscordRPCMessage":["message"],"open":["input"],"getLogs":["id"],"getPackage":["provider","slug"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"getPackageBody":["provider","body"],"isClusterRunning":["cluster_id"],"getGameVersions":[],"createSettingsProfile":["name"],"getUsersFromAuthor":["provider","author"],"getRunningProcessesByClusterId":["cluster_id"],"installModpack":["modpack","cluster_id"],"getLoadersForVersion":["mc_version"],"changeCape":["access_token","cape_uuid"],"updateClusterById":["id","request"],"removePackage":["cluster_id","package_hash"],"getWorlds":["id"],"readSettings":[],"removeUser":["uuid"],"fetchLoggedInProfile":["access_token"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getLogByName":["id","name"],"searchPackages":["provider","query"],"fetchMinecraftProfile":["uuid"],"launchCluster":["id","uuid","search_for_java"],"updateClusterProfile":["name","profile"],"writeSettings":["setting"],"setDefaultUser":["uuid"]}' }
export type Router = { 'oneclient': { getClustersGroupedByMajor: () => Promise<Partial<{ [key in number]: ClusterModel[] }>>,
const ARGS_MAP = { 'debug':'{"getPackageVersion":[],"isInDev":[],"getType":[],"getGitCommitHash":[],"getArch":[],"getBuildTimestamp":[],"openDevTools":[],"getPlatform":[],"getFamily":[],"getOsVersion":[],"getLocale":[]}', 'folders':'{"openCluster":["folder_name"],"fromCluster":["folder_name"]}', 'oneclient':'{"getBundlesFor":["cluster_id"],"getVersions":[],"getClustersGroupedByMajor":[]}', 'events':'{"message":["event"],"process":["event"],"ingress":["event"]}', 'core':'{"installModpack":["modpack","cluster_id"],"removeCape":["access_token"],"killProcess":["pid"],"getLogByName":["id","name"],"getGlobalProfile":[],"removeCluster":["id"],"getUsersFromAuthor":["provider","author"],"getPackage":["provider","slug"],"getUser":["uuid"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getRunningProcesses":[],"readSettings":[],"getPackageBody":["provider","body"],"removePackage":["cluster_id","package_hash"],"setDiscordRPCMessage":["message"],"searchPackages":["provider","query"],"getMultiplePackages":["provider","slugs"],"updateClusterById":["id","request"],"writeSettings":["setting"],"isClusterRunning":["cluster_id"],"getProfileOrDefault":["name"],"getGameVersions":[],"getClusters":[],"getLogs":["id"],"updateClusterProfile":["name","profile"],"removeUser":["uuid"],"getDefaultUser":["fallback"],"setDefaultUser":["uuid"],"getScreenshots":["id"],"getLoadersForVersion":["mc_version"],"createCluster":["options"],"createSettingsProfile":["name"],"getWorlds":["id"],"openMsaLogin":[],"getLinkedPackages":["cluster_id"],"fetchMinecraftProfile":["uuid"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"changeCape":["access_token","cape_uuid"],"getUsers":[],"open":["input"],"fetchLoggedInProfile":["access_token"],"convertUsernameUUID":["username_uuid"],"launchCluster":["id","uuid","search_for_java"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"getRunningProcessesByClusterId":["cluster_id"],"changeSkin":["access_token","skin_url","skin_variant"],"getClusterById":["id"]}' }
export type Router = { 'events': { ingress: (event: IngressPayload) => Promise<void>,
message: (event: MessagePayload) => Promise<void>,
process: (event: ProcessPayload) => Promise<void> },
'oneclient': { getClustersGroupedByMajor: () => Promise<Partial<{ [key in number]: ClusterModel[] }>>,
getBundlesFor: (clusterId: number) => Promise<ModpackArchive[]>,
getVersions: () => Promise<OnlineClusterManifest> },
'folders': { fromCluster: (folderName: string) => Promise<string>,
@ -365,10 +368,7 @@ changeCape: (accessToken: string, capeUuid: string) => Promise<MojangFullPlayerP
removeCape: (accessToken: string) => Promise<MojangFullPlayerProfile>,
convertUsernameUUID: (usernameUuid: string) => Promise<MowojangProfile>,
setDiscordRPCMessage: (message: string) => Promise<null>,
open: (input: string) => Promise<null> },
'events': { ingress: (event: IngressPayload) => Promise<void>,
message: (event: MessagePayload) => Promise<void>,
process: (event: ProcessPayload) => Promise<void> } };
open: (input: string) => Promise<null> } };
export type { InferCommandOutput }

296
packages/core/src/api/packages/provider/modrinth/mod.rs
View file

@ -1,5 +1,7 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::{LazyLock, Mutex};
use std::time::{Duration, Instant};
use super::ProviderExt;
use crate::api::packages::categories::ToProviderCategory;
@ -35,6 +37,12 @@ macro_rules! url_v3 {
};
}
static PACKAGE_CACHE: LazyLock<Mutex<HashMap<String, (ManagedPackage, Instant)>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
static VERSION_CACHE: LazyLock<Mutex<HashMap<String, (ManagedVersion, Instant)>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
const CACHE_DURATION: Duration = Duration::from_secs(600);
#[derive(Default)]
pub struct ModrinthProviderImpl;
@ -120,64 +128,172 @@ impl ProviderExt for ModrinthProviderImpl {
}
async fn get(&self, slug: &str) -> LauncherResult<ManagedPackage> {
Ok(
{
let mut cache = PACKAGE_CACHE.lock().unwrap();
if let Some((package, timestamp)) = cache.get(slug) {
if timestamp.elapsed() < CACHE_DURATION {
return Ok(package.clone());
} else {
cache.remove(slug);
}
}
}
let package: ManagedPackage =
http::fetch_json::<ModrinthPackage>(Method::GET, url!("/project/{}", slug), None, None)
.await?
.into(),
)
.into();
{
let mut cache = PACKAGE_CACHE.lock().unwrap();
cache.insert(package.id.clone(), (package.clone(), Instant::now()));
cache.insert(package.slug.clone(), (package.clone(), Instant::now()));
}
Ok(package)
}
async fn get_multiple(&self, slugs: &[String]) -> LauncherResult<Vec<ManagedPackage>> {
Ok(http::fetch_json::<Vec<ModrinthPackage>>(
Method::GET,
url!(
"/projects?ids=[{}]",
slugs
.iter()
.map(|id| format!("\"{id}\""))
.collect::<Vec<String>>()
.join(",")
),
None,
None,
)
.await?
.into_iter()
.map(Into::into)
.collect())
let mut packages = Vec::new();
let mut missing_slugs = Vec::new();
let unique_slugs: HashSet<&String> = slugs.iter().collect();
{
let mut cache = PACKAGE_CACHE.lock().unwrap();
for slug in unique_slugs {
if let Some((package, timestamp)) = cache.get(slug) {
if timestamp.elapsed() < CACHE_DURATION {
packages.push(package.clone());
} else {
cache.remove(slug);
missing_slugs.push(slug.clone());
}
} else {
missing_slugs.push(slug.clone());
}
}
}
if !missing_slugs.is_empty() {
let fetched_packages: Vec<ManagedPackage> = http::fetch_json::<Vec<ModrinthPackage>>(
Method::GET,
url!(
"/projects?ids=[{}]",
missing_slugs
.iter()
.map(|id| format!("\"{id}\""))
.collect::<Vec<String>>()
.join(",")
),
None,
None,
)
.await?
.into_iter()
.map(Into::into)
.collect();
{
let mut cache = PACKAGE_CACHE.lock().unwrap();
for package in &fetched_packages {
cache.insert(package.id.clone(), (package.clone(), Instant::now()));
cache.insert(package.slug.clone(), (package.clone(), Instant::now()));
}
}
packages.extend(fetched_packages);
}
Ok(packages)
}
async fn get_versions_by_hashes(
&self,
hashes: &[String],
) -> LauncherResult<HashMap<String, ManagedVersion>> {
let body = serde_json::json!({
"hashes": hashes,
"algorithm": "sha1"
});
let mut results = HashMap::new();
let mut missing_hashes = Vec::new();
let unique_hashes: HashSet<&String> = hashes.iter().collect();
let results = http::fetch_json::<HashMap<String, ModrinthVersion>>(
Method::POST,
url!("/version_files"),
Some(body),
None,
)
.await?;
{
let mut cache = VERSION_CACHE.lock().unwrap();
for hash in unique_hashes {
if let Some((version, timestamp)) = cache.get(hash) {
if timestamp.elapsed() < CACHE_DURATION {
results.insert(hash.clone(), version.clone());
} else {
cache.remove(hash);
missing_hashes.push(hash.clone());
}
} else {
missing_hashes.push(hash.clone());
}
}
}
Ok(results.into_iter().map(|(k, v)| (k, v.into())).collect())
if !missing_hashes.is_empty() {
let body = serde_json::json!({
"hashes": missing_hashes,
"algorithm": "sha1"
});
let fetched_results = http::fetch_json::<HashMap<String, ModrinthVersion>>(
Method::POST,
url!("/version_files"),
Some(body),
None,
)
.await?;
{
let mut cache = VERSION_CACHE.lock().unwrap();
for (hash, version) in fetched_results {
let managed: ManagedVersion = version.into();
cache.insert(hash.clone(), (managed.clone(), Instant::now()));
cache.insert(
managed.version_id.clone(),
(managed.clone(), Instant::now()),
);
results.insert(hash, managed);
}
}
}
Ok(results)
}
async fn get_version_by_hash(&self, hash: &str) -> LauncherResult<Option<ManagedVersion>> {
Ok(Some(
http::fetch_json::<ModrinthVersion>(
Method::GET,
url!("/version_file/{hash}"),
None,
None,
)
.await?
.into(),
))
{
let mut cache = VERSION_CACHE.lock().unwrap();
if let Some((version, timestamp)) = cache.get(hash) {
if timestamp.elapsed() < CACHE_DURATION {
return Ok(Some(version.clone()));
} else {
cache.remove(hash);
}
}
}
let version: ManagedVersion = http::fetch_json::<ModrinthVersion>(
Method::GET,
url!("/version_file/{hash}"),
None,
None,
)
.await?
.into();
{
let mut cache = VERSION_CACHE.lock().unwrap();
cache.insert(hash.to_string(), (version.clone(), Instant::now()));
cache.insert(
version.version_id.clone(),
(version.clone(), Instant::now()),
);
}
Ok(Some(version))
}
// async fn get_org_projects(&self, slug: &str) -> LauncherResult<Vec<ManagedPackage>> {
@ -301,38 +417,92 @@ impl ProviderExt for ModrinthProviderImpl {
let response =
http::fetch_json::<Vec<ModrinthVersion>>(Method::GET, url.as_str(), None, None).await?;
let paginated = response
let mut items = Vec::new();
{
let mut cache = VERSION_CACHE.lock().unwrap();
for version in response {
let managed: ManagedVersion = version.into();
cache.insert(
managed.version_id.clone(),
(managed.clone(), Instant::now()),
);
for file in &managed.files {
cache.insert(file.sha1.clone(), (managed.clone(), Instant::now()));
}
items.push(managed);
}
}
let paginated = items
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<ModrinthVersion>>();
.collect::<Vec<ManagedVersion>>();
Ok(Paginated {
offset,
limit,
total: paginated.len(),
items: paginated.into_iter().map(Into::into).collect(),
items: paginated,
})
}
async fn get_versions(&self, slugs: &[String]) -> LauncherResult<Vec<ManagedVersion>> {
Ok(http::fetch_json::<Vec<ModrinthVersion>>(
Method::GET,
url!(
"/versions?ids=[{}]",
slugs
.iter()
.map(|v| format!("\"{v}\""))
.collect::<Vec<String>>()
.join(",")
),
None,
None,
)
.await?
.into_iter()
.map(Into::into)
.collect())
let mut versions = Vec::new();
let mut missing_slugs = Vec::new();
let unique_slugs: HashSet<&String> = slugs.iter().collect();
{
let mut cache = VERSION_CACHE.lock().unwrap();
for slug in unique_slugs {
if let Some((version, timestamp)) = cache.get(slug) {
if timestamp.elapsed() < CACHE_DURATION {
versions.push(version.clone());
} else {
cache.remove(slug);
missing_slugs.push(slug.clone());
}
} else {
missing_slugs.push(slug.clone());
}
}
}
if !missing_slugs.is_empty() {
let fetched_versions = http::fetch_json::<Vec<ModrinthVersion>>(
Method::GET,
url!(
"/versions?ids=[{}]",
missing_slugs
.iter()
.map(|v| format!("\"{v}\""))
.collect::<Vec<String>>()
.join(",")
),
None,
None,
)
.await?
.into_iter()
.map(Into::into)
.collect::<Vec<ManagedVersion>>();
{
let mut cache = VERSION_CACHE.lock().unwrap();
for version in &fetched_versions {
cache.insert(
version.version_id.clone(),
(version.clone(), Instant::now()),
);
for file in &version.files {
cache.insert(file.sha1.clone(), (version.clone(), Instant::now()));
}
}
}
versions.extend(fetched_versions);
}
Ok(versions)
}
}

46
packages/core/src/utils/http.rs
View file

@ -76,9 +76,18 @@ pub fn build_request(method: Method, url: &str) -> RequestBuilder {
client.request(method.clone(), url)
}
fn get_ratelimit_reset(headers: &reqwest::header::HeaderMap) -> Option<u64> {
headers
.get("X-Ratelimit-Reset")
.or(headers.get("retry-after"))
.and_then(|h| h.to_str().ok())
.and_then(|s| s.parse::<f64>().ok())
.map(|s| s.ceil() as u64)
}
pub async fn send_request(request: reqwest::Request) -> LauncherResult<reqwest::Response> {
if let Some(host) = request.url().host_str() {
if host.ends_with("modrinth.com") {
if host == "api.modrinth.com" {
let start = std::time::Instant::now();
MODRINTH_RATE_LIMITER.until_ready().await;
tracing::debug!("waited {:?} for modrinth rate limiter", start.elapsed());
@ -92,6 +101,12 @@ pub async fn send_request(request: reqwest::Request) -> LauncherResult<reqwest::
let mut attempt = 0;
tracing::debug!(
"fetching {:?} with method {:?}",
request.url(),
request.method()
);
let res = loop {
attempt += 1;
@ -112,7 +127,18 @@ pub async fn send_request(request: reqwest::Request) -> LauncherResult<reqwest::
return Err(err.into());
}
Ok(res) => break res,
Ok(res) => {
if res.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
if let Some(reset) = get_ratelimit_reset(res.headers()) {
tracing::warn!(
"Rate limited. Waiting for {reset} seconds before retrying..."
);
tokio::time::sleep(std::time::Duration::from_secs(reset)).await;
continue;
}
}
break res;
}
}
};
@ -134,7 +160,7 @@ pub async fn fetch_advanced(
) -> LauncherResult<Bytes> {
if let Ok(parsed_url) = reqwest::Url::parse(url) {
if let Some(host) = parsed_url.host_str() {
if host.ends_with("modrinth.com") {
if host == "api.modrinth.com" {
let start = std::time::Instant::now();
MODRINTH_RATE_LIMITER.until_ready().await;
tracing::debug!("waited {:?} for modrinth rate limiter", start.elapsed());
@ -148,6 +174,7 @@ pub async fn fetch_advanced(
let client = &REQWEST_CLIENT;
let mut attempt = 0;
tracing::debug!("fetching {url} with method {method:?}");
let res = loop {
attempt += 1;
let mut req = client.request(method.clone(), url);
@ -173,7 +200,18 @@ pub async fn fetch_advanced(
return Err(err.into());
}
Ok(res) => break res,
Ok(res) => {
if res.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
if let Some(reset) = get_ratelimit_reset(res.headers()) {
tracing::warn!(
"Rate limited. Waiting for {reset} seconds before retrying..."
);
tokio::time::sleep(std::time::Duration::from_secs(reset)).await;
continue;
}
}
break res;
}
}
};