add concurrency request limits
This commit is contained in:
parent
403a9f4be8
commit
f8b307d005
3 changed files with 49 additions and 17 deletions
|
|
@ -301,8 +301,19 @@ export type VersionType =
|
|||
*/
|
||||
"old_beta"
|
||||
|
||||
const ARGS_MAP = { 'core':'{"launchCluster":["id","uuid"],"getGlobalProfile":[],"setDefaultUser":["uuid"],"getPackage":["provider","slug"],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getPackageBody":["provider","body"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"writeSettings":["setting"],"removePackage":["cluster_id","package_hash"],"open":["input"],"killProcess":["pid"],"getLinkedPackages":["cluster_id"],"getLogs":["id"],"setDiscordRPCMessage":["message"],"removeUser":["uuid"],"readSettings":[],"getDefaultUser":["fallback"],"getLogByName":["id","name"],"openMsaLogin":[],"getMultiplePackages":["provider","slugs"],"getGameVersions":[],"createSettingsProfile":["name"],"updateClusterById":["id","request"],"changeSkin":["access_token","skin_url","skin_variant"],"getLoadersForVersion":["mc_version"],"isClusterRunning":["cluster_id"],"getRunningProcesses":[],"getProfileOrDefault":["name"],"createCluster":["options"],"getClusters":[],"getClusterById":["id"],"getScreenshots":["id"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"removeCluster":["id"],"getUsers":[],"changeCape":["access_token","cape_uuid"],"getWorlds":["id"],"searchPackages":["provider","query"],"removeCape":["access_token"],"fetchLoggedInProfile":["access_token"],"convertUsernameUUID":["username_uuid"],"getUser":["uuid"],"updateClusterProfile":["name","profile"],"getUsersFromAuthor":["provider","author"],"installModpack":["modpack","cluster_id"],"fetchMinecraftProfile":["uuid"],"getRunningProcessesByClusterId":["cluster_id"]}', 'events':'{"message":["event"],"process":["event"],"ingress":["event"]}', 'oneclient':'{"getVersions":[],"getClustersGroupedByMajor":[],"getBundlesFor":["cluster_id"]}', 'folders':'{"fromCluster":["folder_name"],"openCluster":["folder_name"]}', 'debug':'{"getOsVersion":[],"getArch":[],"getPackageVersion":[],"getBuildTimestamp":[],"getFamily":[],"getLocale":[],"getPlatform":[],"getType":[],"isInDev":[],"getGitCommitHash":[],"openDevTools":[]}' }
|
||||
export type Router = { 'folders': { fromCluster: (folderName: string) => Promise<string>,
|
||||
const ARGS_MAP = { 'debug':'{"getType":[],"openDevTools":[],"getFamily":[],"getOsVersion":[],"getBuildTimestamp":[],"isInDev":[],"getLocale":[],"getPlatform":[],"getPackageVersion":[],"getArch":[],"getGitCommitHash":[]}', 'oneclient':'{"getVersions":[],"getBundlesFor":["cluster_id"],"getClustersGroupedByMajor":[]}', 'events':'{"message":["event"],"ingress":["event"],"process":["event"]}', 'core':'{"getRunningProcesses":[],"searchPackages":["provider","query"],"getMultiplePackages":["provider","slugs"],"isClusterRunning":["cluster_id"],"getPackage":["provider","slug"],"downloadPackage":["provider","package_id","version_id","cluster_id","skip_compatibility"],"fetchMinecraftProfile":["uuid"],"getGameVersions":[],"readSettings":[],"getWorlds":["id"],"uploadSkinBytes":["access_token","skin_data","image_format","skin_variant"],"setDiscordRPCMessage":["message"],"open":["input"],"getGlobalProfile":[],"getUser":["uuid"],"getPackageBody":["provider","body"],"changeCape":["access_token","cape_uuid"],"removePackage":["cluster_id","package_hash"],"removeCape":["access_token"],"removeCluster":["id"],"getClusterById":["id"],"getLoadersForVersion":["mc_version"],"getUsers":[],"installModpack":["modpack","cluster_id"],"createCluster":["options"],"createSettingsProfile":["name"],"convertUsernameUUID":["username_uuid"],"changeSkin":["access_token","skin_url","skin_variant"],"getLogByName":["id","name"],"killProcess":["pid"],"getPackageVersions":["provider","slug","mc_version","loader","offset","limit"],"getProfileOrDefault":["name"],"getRunningProcessesByClusterId":["cluster_id"],"openMsaLogin":[],"getScreenshots":["id"],"getUsersFromAuthor":["provider","author"],"removeUser":["uuid"],"launchCluster":["id","uuid"],"writeSettings":["setting"],"getClusters":[],"downloadExternalPackage":["package","cluster_id","force","skip_compatibility"],"setDefaultUser":["uuid"],"updateClusterById":["id","request"],"getLogs":["id"],"fetchLoggedInProfile":["access_token"],"getLinkedPackages":["cluster_id"],"getDefaultUser":["fallback"],"updateClusterProfile":["name","profile"]}', 'folders':'{"openCluster":["folder_name"],"fromCluster":["folder_name"]}' }
|
||||
export type Router = { 'debug': { openDevTools: () => Promise<void>,
|
||||
isInDev: () => Promise<boolean>,
|
||||
getArch: () => Promise<string>,
|
||||
getFamily: () => Promise<string>,
|
||||
getLocale: () => Promise<string | null>,
|
||||
getType: () => Promise<string>,
|
||||
getPlatform: () => Promise<string>,
|
||||
getOsVersion: () => Promise<string>,
|
||||
getGitCommitHash: () => Promise<string>,
|
||||
getBuildTimestamp: () => Promise<string>,
|
||||
getPackageVersion: () => Promise<string> },
|
||||
'folders': { fromCluster: (folderName: string) => Promise<string>,
|
||||
openCluster: (folderName: string) => Promise<null> },
|
||||
'oneclient': { getClustersGroupedByMajor: () => Promise<Partial<{ [key in number]: ClusterModel[] }>>,
|
||||
getBundlesFor: (clusterId: number) => Promise<ModpackArchive[]>,
|
||||
|
|
@ -355,17 +366,6 @@ removeCape: (accessToken: string) => Promise<MojangFullPlayerProfile>,
|
|||
convertUsernameUUID: (usernameUuid: string) => Promise<MowojangProfile>,
|
||||
setDiscordRPCMessage: (message: string) => Promise<null>,
|
||||
open: (input: string) => Promise<null> },
|
||||
'debug': { openDevTools: () => Promise<void>,
|
||||
isInDev: () => Promise<boolean>,
|
||||
getArch: () => Promise<string>,
|
||||
getFamily: () => Promise<string>,
|
||||
getLocale: () => Promise<string | null>,
|
||||
getType: () => Promise<string>,
|
||||
getPlatform: () => Promise<string>,
|
||||
getOsVersion: () => Promise<string>,
|
||||
getGitCommitHash: () => Promise<string>,
|
||||
getBuildTimestamp: () => Promise<string>,
|
||||
getPackageVersion: () => Promise<string> },
|
||||
'events': { ingress: (event: IngressPayload) => Promise<void>,
|
||||
message: (event: MessagePayload) => Promise<void>,
|
||||
process: (event: ProcessPayload) => Promise<void> } };
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ function DownloadingMods({ mods, setOpen, nextPath }: { mods: ModDataArray; setO
|
|||
useEffect(() => {
|
||||
const downloadAll = async () => {
|
||||
if (useParallelModDownloading)
|
||||
await downloadModsParallel(mods, 25, async (mod) => {
|
||||
await downloadModsParallel(mods, 4, async (mod) => {
|
||||
setModName(mod.name);
|
||||
try {
|
||||
await download.mutateAsync(mod);
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::sync::{Arc, LazyLock, Mutex};
|
||||
|
||||
use bytes::Bytes;
|
||||
use reqwest::{Method, RequestBuilder};
|
||||
use reqwest::{Method, RequestBuilder, StatusCode};
|
||||
use serde::de::DeserializeOwned;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_stream::StreamExt;
|
||||
|
|
@ -36,7 +36,7 @@ pub(crate) static REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
|||
.default_headers({
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
let header = reqwest::header::HeaderValue::from_str(&format!(
|
||||
"{}/{} ({})",
|
||||
"Polyfrost/{}/{} ({})",
|
||||
Core::get().launcher_name,
|
||||
Core::get().launcher_version,
|
||||
Core::get().launcher_website,
|
||||
|
|
@ -49,6 +49,18 @@ pub(crate) static REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
|||
.expect("failed to build reqwest client!")
|
||||
});
|
||||
|
||||
pub(crate) static DOMAIN_SEMAPHORES: LazyLock<Mutex<HashMap<String, Arc<Semaphore>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
const MAX_CONCURRENT_REQUESTS_PER_DOMAIN: usize = 3;
|
||||
|
||||
fn get_domain_semaphore(host: &str) -> Arc<Semaphore> {
|
||||
let mut map = DOMAIN_SEMAPHORES.lock().unwrap();
|
||||
map.entry(host.to_string())
|
||||
.or_insert_with(|| Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS_PER_DOMAIN)))
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub async fn request(method: Method, url: &str) -> LauncherResult<reqwest::Response> {
|
||||
let req = build_request(method, url).build()?;
|
||||
send_request(req).await
|
||||
|
|
@ -61,6 +73,13 @@ pub fn build_request(method: Method, url: &str) -> RequestBuilder {
|
|||
}
|
||||
|
||||
pub async fn send_request(request: reqwest::Request) -> LauncherResult<reqwest::Response> {
|
||||
let domain_permit = if let Some(host) = request.url().host_str() {
|
||||
let sem = get_domain_semaphore(host);
|
||||
Some(sem.acquire_owned().await.map_err(|e| anyhow::anyhow!(e))?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let semaphore = SemaphoreStore::fetch().await;
|
||||
let permit = semaphore.acquire().await;
|
||||
|
||||
|
|
@ -93,6 +112,7 @@ pub async fn send_request(request: reqwest::Request) -> LauncherResult<reqwest::
|
|||
};
|
||||
|
||||
// Drop the fetch permit as we are no longer fetching
|
||||
drop(domain_permit);
|
||||
drop(permit);
|
||||
|
||||
Ok(res)
|
||||
|
|
@ -108,6 +128,17 @@ pub async fn fetch_advanced(
|
|||
headers: Option<HashMap<&str, &str>>,
|
||||
ingress: Option<&SubIngress<'_>>,
|
||||
) -> LauncherResult<Bytes> {
|
||||
let domain_permit = if let Ok(parsed_url) = reqwest::Url::parse(url) {
|
||||
if let Some(host) = parsed_url.host_str() {
|
||||
let sem = get_domain_semaphore(host);
|
||||
Some(sem.acquire_owned().await.map_err(|e| anyhow::anyhow!(e))?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let semaphore = SemaphoreStore::fetch().await;
|
||||
let permit = semaphore.acquire().await;
|
||||
|
||||
|
|
@ -144,6 +175,7 @@ pub async fn fetch_advanced(
|
|||
};
|
||||
|
||||
// Drop the fetch permit as we are no longer fetching
|
||||
drop(domain_permit);
|
||||
drop(permit);
|
||||
|
||||
let bytes = if let Some(ingress) = ingress {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue