oiupoyt/OneLauncher-Cracked
1
1
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity

feat: Package inferring from hash

Browse source
This commit is contained in:
LynithDev 2024-11-16 14:40:41 +01:00
parent c07d538575
commit 1521bbb96a
17 changed files with 432 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Cargo.lock generated
View file

@ -3336,6 +3336,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "murmur2"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb585ade2549a017db2e35978b77c319214fa4b37cede841e27954dd6e8f3ca8"
[[package]]
name = "ndk"
version = "0.9.0"
@ -3813,6 +3819,7 @@ dependencies = [
"interpulse",
"iota_stronghold",
"lazy_static",
"murmur2",
"notify",
"notify-debouncer-mini",
"onelauncher_gamemode",

1
Cargo.toml
View file

@ -133,6 +133,7 @@ async-compression = { version = "0.4.12", features = [ "flate2", "tokio" ] }
flate2 = { version = "1.0" }
sha1_smol = { version = "1.0", features = [ "std" ] }
sha2 = { version = "0.10" }
murmur2 = { version = "0.1.0" }
bytes = { version = "1.7" }
# # ============= UTILITY ============= # #

3
apps/desktop/src/api/commands/package.rs
View file

@ -213,7 +213,8 @@ pub async fn sync_cluster_packages(cluster_path: ClusterPath) -> Result<(), Stri
pub async fn sync_cluster_packages_by_type(
cluster_path: ClusterPath,
package_type: PackageType,
clear: Option<bool>,
) -> Result<(), String> {
package::sync_packages_by_type(&cluster_path, package_type).await?;
package::sync_packages_by_type(&cluster_path, package_type, clear).await?;
Ok(())
}

113
apps/frontend/src/ui/pages/cluster/ClusterMods.tsx
View file

@ -1,6 +1,6 @@
import type { ManagedPackage, Package, Providers } from '@onelauncher/client/bindings';
import type { Package } from '@onelauncher/client/bindings';
import UFuzzy from '@leeoniya/ufuzzy';
import { FilterFunnel01Icon, SearchMdIcon, Trash03Icon } from '@untitled-theme/icons-solid';
import { FilterFunnel01Icon, RefreshCw01Icon, SearchMdIcon, Trash03Icon } from '@untitled-theme/icons-solid';
import { bridge } from '~imports';
import Button from '~ui/components/base/Button';
import TextField from '~ui/components/base/TextField';
@ -8,39 +8,13 @@ import ScrollableContainer from '~ui/components/ScrollableContainer';
import Sidebar from '~ui/components/Sidebar';
import useBrowser from '~ui/hooks/useBrowser';
import useClusterContext from '~ui/hooks/useCluster';
import useCommand, { tryResult } from '~ui/hooks/useCommand';
import { createEffect, createResource, createSignal, For, Match, Show, Switch } from 'solid-js';
import useCommand from '~ui/hooks/useCommand';
import { createEffect, createSignal, For, Match, Show, Switch } from 'solid-js';
// TODO: Possibly optimise this as it has 2 cloned lists, and another containing only the names
// TODO: This needs a rewrite.
function ClusterMods() {
const [cluster] = useClusterContext();
const [mods, { refetch }] = useCommand(cluster, () => bridge.commands.getClusterPackages(cluster()?.path || '', 'mod'));
const [managedMods] = createResource(mods, async () => {
const list = mods();
if (!list)
return Promise.resolve(new Map<Providers, ManagedPackage[]>());
const providers: Map<Providers, string[]> = new Map();
list.forEach((mod) => {
if (mod.meta.type === 'managed')
providers.set(mod.meta.provider, [
...(providers.get(mod.meta.provider) || []),
mod.meta.package_id,
]);
});
const formatted: Map<Providers, ManagedPackage[]> = new Map();
for (const [provider, packageIds] of providers.entries()) {
const result = await tryResult(() => bridge.commands.getProviderPackages(provider, packageIds));
formatted.set(provider, result);
}
return formatted;
}, {
initialValue: new Map<Providers, ManagedPackage[]>(),
});
const [mods, { refetch: refetchMods }] = useCommand(cluster, () => bridge.commands.getClusterPackages(cluster()?.path || '', 'mod'));
// Mods to display, should be a clone of the mods array but sorted
const [displayedMods, setDisplayedMods] = createSignal<Package[]>([]);
@ -101,7 +75,15 @@ function ClusterMods() {
<div class="w-full flex flex-row justify-between">
<h1>Mods</h1>
<div>
<div class="flex flex-row items-stretch justify-end gap-x-2">
<div class="h-8 flex flex-row items-stretch justify-end gap-x-2">
<Button
buttonStyle="iconSecondary"
children={<RefreshCw01Icon />}
onClick={() => {
bridge.commands.syncClusterPackagesByType(cluster()?.path || '', 'mod', true).then(refetchMods);
}}
/>
<Button
buttonStyle="secondary"
children={sortingAtoZ() ? 'A-Z' : 'Z-A'}
@ -119,7 +101,7 @@ function ClusterMods() {
<Match when={displayedMods().length > 0}>
<For each={displayedMods()}>
{mod => (
<ModEntry managedMods={managedMods()} pkg={mod} refetch={refetch} />
<ModEntry pkg={mod} refetch={refetchMods} />
)}
</For>
</Match>
@ -138,23 +120,12 @@ export default ClusterMods;
interface ModEntryProps {
pkg: Package;
refetch: () => void;
managedMods?: Map<Providers, ManagedPackage[]>;
};
function ModEntry(props: ModEntryProps) {
const [cluster] = useClusterContext();
const browser = useBrowser();
const managed = () => {
const pkg = props.pkg;
if (props.managedMods && pkg.meta.type === 'managed') {
const package_id = pkg.meta.package_id;
return props.managedMods.get(pkg.meta.provider)?.find(p => p.id === package_id);
}
return undefined;
};
const name = () => {
if (props.pkg.meta.type === 'unknown')
return props.pkg.file_name;
@ -167,38 +138,45 @@ function ModEntry(props: ModEntryProps) {
if (props.pkg.meta.type === 'managed')
formatted = props.pkg.meta.version_formatted;
else if (props.pkg.meta.type === 'mapped')
formatted = props.pkg.meta.version || formatted;
return formatted;
};
const tag = () => {
let tag = 'Unknown';
const tags = () => {
type TagStatus = 'normal' | 'warning' | 'danger';
const tags: [string, TagStatus][] = [];
const meta = props.pkg.meta;
if (props.pkg.meta.type === 'managed')
tag = props.pkg.meta.provider;
else if (props.pkg.meta.type === 'mapped')
tag = 'Mapped';
if (meta.type === 'managed') {
tags[0] = [meta.provider, 'normal'];
return tag;
const supportedVersions = meta.mc_versions || [];
if (supportedVersions.length === 0)
tags.push(['Unknown Version', 'warning']);
else if (!supportedVersions.includes(cluster()!.meta.mc_version))
tags.push(['Incompatible', 'danger']);
}
else {
tags[0] = ['Unknown', 'normal'];
}
return tags;
};
const icon = () => {
if (props.pkg.meta.type === 'managed')
return managed()?.icon_url;
// if (props.pkg.meta.type === 'managed')
// return props.pkg.meta.;
return undefined;
};
function onClick(e: MouseEvent) {
const mngd = managed();
if (mngd) {
const meta = props.pkg.meta;
if (meta.type === 'managed') {
e.preventDefault();
e.stopPropagation();
browser.displayPackage(mngd.id, mngd.provider);
browser.displayPackage(meta.package_id, meta.provider);
}
}
@ -232,7 +210,20 @@ function ModEntry(props: ModEntryProps) {
<div class="flex flex-col items-start justify-center gap-y-2">
<span class="flex flex-row items-center justify-start gap-x-1">
<h4>{name()}</h4>
<span class="rounded-xl bg-gray-05 px-1.5 py-0.5 text-xs text-white/60">{tag()}</span>
<For each={tags()}>
{([tag, status]) => (
<span
class="rounded-xl px-1.5 py-1 text-xs"
classList={{
'bg-gray-05 text-white/60': status === 'normal',
'bg-amber/10 text-amber': status === 'warning',
'bg-danger/15 text-danger': status === 'danger',
}}
>
{tag}
</span>
)}
</For>
</span>
<span class="h-2 flex flex-row items-center justify-start text-xs text-fg-secondary/50 font-600">
{version()}

6
apps/testing/examples/cluster_packages.rs
View file

@ -26,7 +26,7 @@ async fn main() -> Result<()> {
true,
)
.await?;
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod)
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod, None)
.await?;
// Print Packages again
@ -40,7 +40,7 @@ async fn main() -> Result<()> {
false,
)
.await?;
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod)
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod, None)
.await?;
// Print Packages again
@ -53,7 +53,7 @@ async fn main() -> Result<()> {
PackageType::Mod,
)
.await?;
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod)
cluster::content::package::sync_packages_by_type(&cluster.cluster_path(), PackageType::Mod, None)
.await?;
Ok(())

14
apps/testing/examples/murmur2_hash.rs Normal file
View file

@ -0,0 +1,14 @@
use onelauncher::utils::crypto;
const CONTENT: &str = "
this is my
text content
";
fn main() {
let hash = crypto::murmur2(CONTENT.as_bytes());
dbg!(hash);
dbg!(hash as u64);
assert_eq!(hash, 661461369);
}

8
packages/client/src/bindings.ts generated
View file

@ -474,9 +474,9 @@ export const commands = {
else return { status: 'error', error: e as any };
}
},
async syncClusterPackagesByType(clusterPath: ClusterPath, packageType: PackageType): Promise<Result<null, string>> {
async syncClusterPackagesByType(clusterPath: ClusterPath, packageType: PackageType, clear: boolean | null): Promise<Result<null, string>> {
try {
return { status: 'ok', data: await TAURI_INVOKE('sync_cluster_packages_by_type', { clusterPath, packageType }) };
return { status: 'ok', data: await TAURI_INVOKE('sync_cluster_packages_by_type', { clusterPath, packageType, clear }) };
}
catch (e) {
if (e instanceof Error)
@ -885,7 +885,7 @@ export interface OfflinePayload { offline: boolean };
/**
* A struct that represents a Package.
*/
export interface Package { sha512: string; meta: PackageMetadata; file_name: string; disabled: boolean };
export interface Package { sha1: string; meta: PackageMetadata; file_name: string; disabled: boolean };
export type PackageBody = { Url: string } | { Markdown: string };
/**
* Optional data used to link a specific cluster to a package project.
@ -915,7 +915,7 @@ export type PackageFile = 'required_pack' | 'optional_pack' | 'unknown';
/**
* Metadata that represents a [`Package`].
*/
export type PackageMetadata = { type: 'managed'; package_id: string; provider: Providers; package_type: PackageType; title: string; version_id: string; version_formatted: string } | { type: 'mapped'; title: string | null; description: string | null; authors: string[]; version: string | null; icon: string | null; package_type: PackageType | null } | { type: 'unknown' };
export type PackageMetadata = { type: 'managed'; package_id: string; provider: Providers; package_type: PackageType; title: string; version_id: string; version_formatted: string; mc_versions: string[] | null } | { type: 'unknown' };
/**
* Relative [`PathBuf`] for a specific [`Package`] of a [`Cluster`].
*/

1
packages/core/Cargo.toml
View file

@ -50,6 +50,7 @@ reqwest = { workspace = true }
dirs = { workspace = true }
sha1_smol = { workspace = true }
sha2 = { workspace = true }
murmur2 = { workspace = true }
async_zip = { workspace = true }
discord-rich-presence = { workspace = true }
regex = { workspace = true }

25
packages/core/src/api/cluster/content/package.rs
View file

@ -92,15 +92,11 @@ pub async fn download_package(
);
let package_path = download_file(file, &package.package_type, cluster).await?;
let sha512 = file
.hashes
.get("sha512")
.unwrap_or(&"unknown".to_string())
.to_owned();
let sha1 = file.hashes.get("sha1").unwrap_or(&"".to_string()).clone();
let package = Package {
file_name: file.file_name.clone(),
sha512,
sha1,
meta: PackageMetadata::from_managed_package(package.clone(), managed_version),
disabled: false,
};
@ -192,13 +188,14 @@ pub async fn get_package(
package_type: PackageType,
) -> Result<Package> {
let state = State::get().await?;
let manager = state.packages.read().await;
let manager = manager
.get(cluster_path)
let mut store = state.packages.write().await;
let manager = store
.get_mut(cluster_path)
.ok_or_else(|| anyhow::anyhow!("cluster not found in packages map"))?;
Ok(manager
.get(package_type)
.await
.packages
.get(package_path)
.cloned()
@ -212,13 +209,14 @@ pub async fn get_packages(
package_type: PackageType,
) -> Result<Vec<Package>> {
let state = State::get().await?;
let manager = state.packages.read().await;
let manager = manager
.get(cluster_path)
let mut store = state.packages.write().await;
let manager = store
.get_mut(cluster_path)
.ok_or_else(|| anyhow::anyhow!("cluster not found in packages map"))?;
Ok(manager
.get(package_type)
.await
.packages
.values()
.cloned()
@ -244,6 +242,7 @@ pub async fn sync_packages(cluster_path: &ClusterPath) -> Result<()> {
pub async fn sync_packages_by_type(
cluster_path: &ClusterPath,
package_type: PackageType,
clear: Option<bool>
) -> Result<()> {
let state = State::get().await?;
let mut manager = state.packages.write().await;
@ -252,7 +251,7 @@ pub async fn sync_packages_by_type(
.ok_or_else(|| anyhow::anyhow!("cluster not found in packages map"))?;
manager
.sync_packages_by_type(&state.directories, package_type)
.sync_packages_by_type(&state.directories, package_type, clear)
.await?;
Ok(())

4
packages/core/src/api/cluster/create.rs
View file

@ -67,7 +67,7 @@ pub async fn create_unfinished_cluster(
State::sync().await?;
let state = State::get().await?;
let mut packages = state.packages.write().await;
packages.add_cluster(&state.directories, cluster.cluster_path()).await;
packages.add_cluster(cluster.cluster_path()).await;
Ok(cluster.cluster_path())
}
@ -173,7 +173,7 @@ pub async fn create_cluster(
State::sync().await?;
let state = State::get().await?;
let mut packages = state.packages.write().await;
packages.add_cluster(&state.directories, cluster.cluster_path()).await;
packages.add_cluster(cluster.cluster_path()).await;
Ok(cluster.cluster_path())
}

46
packages/core/src/api/package/content/curseforge.rs
View file

@ -4,6 +4,7 @@ use bytes::Bytes;
use chrono::{DateTime, Utc};
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::json;
use serde_repr::{Deserialize_repr, Serialize_repr};
use url::Url;
@ -301,7 +302,7 @@ pub struct ModFile {
// pub server_pack_file_id: u32,
// pub is_early_access_content: bool,
// pub early_access_end_date: String,
// pub file_fingerprint: u32,
pub file_fingerprint: u32,
// pub modules: Vec<Module>,
}
@ -657,11 +658,10 @@ pub async fn get_all_versions(
pub async fn get_versions(
versions: Vec<String>,
) -> Result<Vec<ManagedVersion>> {
let mut map = serde_json::Map::new();
map.insert("fileIds".to_string(), versions.iter().map(|v| v.parse::<u32>().unwrap()).collect::<Vec<u32>>().into());
let json_body = serde_json::Value::Object(map);
) -> Result<Vec<ModFile>> {
let json_body = json!({
"fileIds": versions.iter().map(|v| v.parse::<u32>().unwrap()).collect::<Vec<u32>>()
});
Ok(serde_json::from_slice::<CFData<_>>(
&fetch_advanced(
@ -676,4 +676,38 @@ pub async fn get_versions(
.data)
}
pub async fn get_versions_by_hashes(
hashes: Vec<String>,
) -> Result<HashMap<String, ModFile>> {
let body = json!({
"fingerprints": hashes.into_iter().map(|hash| hash.parse::<u32>().unwrap_or_default()).collect::<Vec<u32>>()
});
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Response {
exact_matches: Vec<FingerprintMatch>,
exact_fingerprints: Vec<u32>
}
#[derive(Debug, Serialize, Deserialize)]
struct FingerprintMatch {
id: u32,
file: ModFile
}
let files = serde_json::from_slice::<CFData<Response>>(
&fetch_advanced(
Method::POST,
format!("/v1/fingerprints/{}", crate::constants::CURSEFORGE_GAME_ID).as_str(),
None::<fn(&mut Url)>,
None,
Some(body),
)
.await?,
)?;
Ok(files.data.exact_matches.into_iter().map(|m| (m.file.file_fingerprint.to_string(), m.file)).collect())
}

21
packages/core/src/api/package/content/mod.rs
View file

@ -2,6 +2,8 @@
//!
//! Utilities for searching and downloading content packages to `OneLauncher`.
use std::collections::HashMap;
use modrinth::{Facet, FacetOperation};
use serde::{Deserialize, Serialize};
@ -47,6 +49,10 @@ impl Providers {
}
}
pub const fn get_providers() -> &'static [Providers] {
&[Self::Modrinth, Self::Curseforge]
}
#[allow(clippy::too_many_arguments)]
pub async fn search(
&self,
@ -195,6 +201,21 @@ impl Providers {
PackageBody::Markdown(markdown) => markdown.to_owned(),
})
}
pub async fn get_versions_by_hashes(&self, hashes: Vec<String>) -> Result<HashMap<String, ManagedVersion>> {
Ok(match self {
Self::Modrinth => modrinth::get_versions_by_hashes(hashes)
.await?
.into_iter()
.map(|(hash, version)| (hash, version.into()))
.collect(),
Self::Curseforge => curseforge::get_versions_by_hashes(hashes)
.await?
.into_iter()
.map(|(hash, version)| (hash, version.into()))
.collect()
})
}
}
fn build_facets(

26
packages/core/src/api/package/content/modrinth.rs
View file

@ -5,11 +5,12 @@ use crate::data::{Loader, ManagedPackage, ManagedUser, ManagedVersion, PackageTy
use crate::store::{
Author, License, ManagedVersionFile, ManagedVersionReleaseType, PackageFile, PackageSide, ProviderSearchResults, SearchResult
};
use crate::utils::http::fetch;
use crate::utils::http::{fetch, fetch_advanced};
use crate::utils::pagination::Pagination;
use crate::{Result, State};
use chrono::{DateTime, Utc};
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::Value;
@ -147,6 +148,7 @@ pub struct File {
pub primary: bool,
pub size: u64,
pub file_type: Option<PackageFile>,
pub hashes: HashMap<String, String>
}
impl From<File> for ManagedVersionFile {
@ -157,7 +159,7 @@ impl From<File> for ManagedVersionFile {
primary: value.primary,
size: value.size,
file_type: value.file_type,
hashes: HashMap::default(), // TODO
hashes: value.hashes
}
}
}
@ -510,3 +512,23 @@ pub async fn get_versions(versions: Vec<String>) -> Result<Vec<ModrinthVersion>>
.await?,
)?)
}
pub async fn get_versions_by_hashes(hashes: Vec<String>) -> Result<HashMap<String, ModrinthVersion>> {
let body = serde_json::json!({
"hashes": hashes,
"algorithm": "sha1"
});
Ok(serde_json::from_slice(
&fetch_advanced(
Method::POST,
format_url!("/version_files").as_str(),
None,
Some(body),
None,
None,
&State::get().await?.fetch_semaphore,
)
.await?
)?)
}

2
packages/core/src/store/mod.rs
View file

@ -144,7 +144,7 @@ impl State {
}?;
send_ingress(&ingress, 5.0, Some("loading packages")).await?;
let packages = Packages::initialize(&directories, &clusters).await;
let packages = Packages::initialize(&clusters).await;
let ingress_processor = IngressProcessor::new();
let discord_rpc = DiscordRPC::initialize(is_offline || settings.disable_discord)?;

268
packages/core/src/store/package.rs
View file

@ -2,7 +2,9 @@
use crate::package::content::Providers;
use crate::store::Loader;
use crate::utils::crypto;
use crate::utils::http::{write_icon, IoSemaphore};
use crate::State;
use async_zip::tokio::read::fs::ZipFileReader;
use chrono::{DateTime, Utc};
use onelauncher_utils::io;
@ -166,7 +168,7 @@ pub struct ProviderSearchResults {
#[derive(Serialize, Deserialize, Clone, Debug)]
#[cfg_attr(feature = "specta", derive(specta::Type))]
pub struct Package {
pub sha512: String,
pub sha1: String,
pub meta: PackageMetadata,
pub file_name: String,
pub disabled: bool,
@ -180,11 +182,11 @@ impl Package {
.ok_or_else(|| crate::ErrorKind::AnyhowError(anyhow::anyhow!("no file name")))?
.to_string_lossy()
.to_string();
// TODO: sha512
let sha512 = String::from("unknown");
let sha1 = crypto::sha1_file(&path.0)?;
Ok(Self {
sha512,
sha1,
meta,
file_name,
disabled: false,
@ -199,32 +201,20 @@ pub struct Packages {
}
impl Packages {
pub async fn initialize(dirs: &Directories, clusters: &Clusters) -> Self {
pub async fn initialize(clusters: &Clusters) -> Self {
let mut this = Self { managers: HashMap::new() };
// TODO: This should probably not clone and store the cluster path in like 2 areas
for cluster_path in clusters.0.keys() {
this.add_cluster(dirs, cluster_path.clone()).await;
this.add_cluster(cluster_path.clone()).await;
}
this
}
pub async fn add_cluster(&mut self, dirs: &Directories, cluster_path: ClusterPath) {
let mgr = PackageManager::initialize(dirs, cluster_path.clone()).await;
match mgr {
Ok(mgr) => {
self.managers.insert(cluster_path, mgr);
}
Err(e) => {
tracing::error!(
"failed to initialize package manager for cluster {}: {}. skipping",
cluster_path,
e
);
}
};
pub async fn add_cluster(&mut self, cluster_path: ClusterPath) {
let mgr = PackageManager::new(cluster_path.clone());
self.managers.insert(cluster_path, mgr);
}
#[must_use]
@ -257,6 +247,7 @@ impl PackagesMeta {
#[derive(Debug)]
pub struct PackageManager {
pub cluster_path: ClusterPath,
synced: bool,
modpacks: PackagesMeta,
mods: PackagesMeta,
datapacks: PackagesMeta,
@ -265,18 +256,12 @@ pub struct PackageManager {
}
impl PackageManager {
pub async fn initialize(dirs: &Directories, cluster_path: ClusterPath) -> crate::Result<Self> {
let mut manager = Self::new(cluster_path);
manager.sync_packages(dirs).await;
Ok(manager)
}
#[must_use]
pub fn new(cluster_path: ClusterPath) -> Self {
Self {
cluster_path,
modpacks: PackagesMeta::new(PackageType::ModPack),
synced: false,
mods: PackagesMeta::new(PackageType::Mod),
datapacks: PackagesMeta::new(PackageType::DataPack),
resourcepacks: PackagesMeta::new(PackageType::ResourcePack),
@ -296,9 +281,27 @@ impl PackageManager {
.join(package_type.get_meta()))
}
/// Get the `PackagesMeta` for a specific package type. Does not sync.
#[async_recursion::async_recursion]
async fn get_and_sync(&mut self, dirs: Option<&Directories>) -> &mut Self {
if !self.synced {
self.synced = true; // early set to prevent infinite loop
let state = State::get().await.expect("couldn't get state when syncing packages");
let dirs = match dirs {
Some(dirs) => dirs,
None => &state.directories,
};
self.sync_packages(dirs).await;
}
self
}
/// Get the `PackagesMeta` for a specific package type. Syncs.
#[must_use]
pub const fn get(&self, package_type: PackageType) -> &PackagesMeta {
pub async fn get(&mut self, package_type: PackageType) -> &PackagesMeta {
self.get_and_sync(None).await;
match package_type {
PackageType::Mod => &self.mods,
PackageType::DataPack => &self.datapacks,
@ -396,7 +399,7 @@ impl PackageManager {
];
for package_type in package_types {
if let Err(err) = &self.sync_packages_by_type(dirs, package_type).await {
if let Err(err) = &self.sync_packages_by_type(dirs, package_type, None).await {
tracing::error!("failed to sync package {}: {}", package_type.get_name(), err);
}
}
@ -418,11 +421,11 @@ impl PackageManager {
/// sync packages from the manager to the metafile
#[tracing::instrument(skip(self, dirs))]
async fn sync_to_file_by_type(
&self,
&mut self,
dirs: &Directories,
package_type: PackageType,
) -> crate::Result<()> {
let packages_meta = self.get(package_type);
let packages_meta = self.get(package_type).await;
let packages_meta = serde_json::to_string(&packages_meta)?;
io::write(self.get_meta_file(dirs, package_type).await?, packages_meta).await?;
@ -431,7 +434,7 @@ impl PackageManager {
}
/// returns a list of packages that have a file but are not synced in the manager
#[tracing::instrument]
#[tracing::instrument(skip(self, dirs, packages))]
async fn sync_packages_from_local(
&self,
dirs: &Directories,
@ -447,13 +450,11 @@ impl PackageManager {
.await?;
let mut packages_to_keep = HashSet::<PackagePath>::new();
let mut packages_to_infer = HashSet::<PathBuf>::new();
while let Some(file) = files.next_entry().await? {
// Skip .packages.json meta file
if file
.file_name()
.to_string_lossy()
.eq(&package_type.get_meta_file_name())
if file.file_name().to_string_lossy().eq(&package_type.get_meta_file_name()) || file.file_type().await?.is_dir()
{
continue;
}
@ -461,29 +462,24 @@ impl PackageManager {
let package_path = PackagePath::new(&file.path());
// Check if the file is in the packages list already
if let Some(_package) = packages.get(&package_path) {
// Package path is in manager and file system
// match package.meta {
// PackageMetadata::Unknown => {
// // TODO: Infer
// }
// _ => {}
// }
} else {
// Package path is not in the packages list but exists on file system, lets add it to the manager
// TODO: Infer package
let meta = PackageMetadata::Unknown;
let package = Package::new(&package_path, meta)?;
packages.insert(package_path.clone(), package);
if let None = packages.get(&package_path) {
// Package path is not in the packages list but exists on file system, lets try to infer it later on
packages_to_infer.insert(package_path.0.clone());
}
// Add the package to the set to not remove it from the manager
packages_to_keep.insert(package_path);
}
// Infer any packages
if packages_to_infer.len() > 0 {
let not_found = infer_modrinth(packages, packages_to_infer, package_type).await;
if let Some(not_found) = not_found {
infer_curseforge(packages, not_found, package_type).await;
}
}
if packages.len() != packages_to_keep.len() {
packages.retain(|pkg_path, _| packages_to_keep.contains(pkg_path));
}
@ -492,30 +488,165 @@ impl PackageManager {
}
// sync packages in a cluster
#[tracing::instrument]
#[tracing::instrument(skip(self, dirs))]
pub async fn sync_packages_by_type(
&mut self,
dirs: &Directories,
package_type: PackageType,
clear: Option<bool>,
) -> crate::Result<()> {
if clear.unwrap_or(false) {
self.get_mut(package_type).packages.clear();
}
// Clone the current packages and merge them with the local package list
let mut packages = self.get(package_type).packages.clone();
let mut packages = self.get(package_type).await.packages.clone();
let mut new_packages = self.get_from_file(dirs, package_type).await?.packages;
packages.extend(new_packages.drain());
// Sync the packages from FS
self.sync_packages_from_local(dirs, &mut packages, package_type)
.await?;
self.sync_packages_from_local(dirs, &mut packages, package_type).await?;
// Finally store the new list in memory and on disk
// TODO: Should probably only do this if the packages have changed
self.get_mut(package_type).packages = packages;
self.sync_to_file_by_type(dirs, package_type).await?;
let stored_packages = &self.get_mut(package_type).packages;
if stored_packages.len() != packages.len() {
self.get_mut(package_type).packages = packages;
self.sync_to_file_by_type(dirs, package_type).await?;
}
Ok(())
}
}
#[tracing::instrument(skip(packages, to_infer))]
async fn infer_modrinth(packages: &mut PackagesMap, to_infer: HashSet<PathBuf>, package_type: PackageType) -> Option<HashSet<PathBuf>> {
let mut sha1_map = HashMap::<String, PathBuf>::new();
to_infer.iter().for_each(|path| {
if let Ok(hash) = crypto::sha1_file(path) {
sha1_map.insert(hash, path.clone());
}
});
let sha1_len = sha1_map.len();
match Providers::Modrinth.get_versions_by_hashes(sha1_map.iter().map(|hash| hash.0.clone()).collect()).await {
Ok(versions) => {
let found_all = (sha1_len - versions.len()) <= 0;
let havent_found = if !found_all {
Some(sha1_map.iter().filter_map(|(hash, path)| {
if versions.contains_key(hash) {
None
} else {
Some(path.clone())
}
}).collect::<HashSet<PathBuf>>())
} else {
None
};
let package_ids = versions.iter().map(|(hash, version)| (version.package_id.clone(), hash.clone())).collect::<HashMap<String, String>>();
match Providers::Modrinth.get_multiple(package_ids.clone().into_keys().collect::<Vec<String>>().as_slice()).await {
Ok(managed_packages) => {
for managed in managed_packages {
let hash = match package_ids.get(&managed.id) {
Some(hash) => hash,
None => continue,
};
let version = match versions.get(hash) {
Some(version) => version,
None => continue,
};
if let Some(package_path) = sha1_map.get(hash) {
let meta = PackageMetadata::Managed {
package_id: managed.id,
provider: Providers::Modrinth,
package_type,
title: managed.title,
version_id: version.id.clone(),
version_formatted: version.version_display.clone(),
mc_versions: Some(version.game_versions.clone()),
};
let package_path = &PackagePath::new(package_path);
match Package::new(package_path, meta) {
Ok(package) => {
packages.insert(package_path.clone(), package);
},
Err(err) => {
tracing::error!("failed to create package from Modrinth: {}", err);
},
};
}
}
},
Err(err) => {
tracing::error!("failed to get versions from Modrinth: {}", err);
}
}
return havent_found;
},
Err(err) => {
tracing::error!("failed to get versions from Modrinth: {}", err);
}
};
Some(to_infer)
}
#[tracing::instrument(skip(packages, to_infer))]
async fn infer_curseforge(packages: &mut PackagesMap, to_infer: HashSet<PathBuf>, package_type: PackageType) {
let provider = Providers::Curseforge;
let mut murmur_map = HashMap::<String, PathBuf>::new();
to_infer.iter().for_each(|path| {
if let Ok(hash) = crypto::murmur2_file(path) {
murmur_map.insert(hash.to_string(), path.clone());
}
});
let sha1_len = murmur_map.len();
match provider.get_versions_by_hashes(murmur_map.iter().map(|hash| hash.0.clone()).collect()).await {
Ok(versions) => {
let found_all = (sha1_len - versions.len()) <= 0;
if !found_all {
tracing::warn!("failed to infer all packages from Curseforge");
}
for (hash, version) in versions {
let package_path = match murmur_map.get(&hash) {
Some(path) => PackagePath::new(path),
None => continue,
};
let meta = PackageMetadata::Managed {
package_id: version.package_id,
provider: provider.clone(),
package_type,
title: version.name,
version_id: version.id,
version_formatted: version.version_display,
mc_versions: Some(version.game_versions),
};
if let Ok(package) = Package::new(&package_path, meta) {
packages.insert(package_path, package);
}
}
},
Err(err) => {
tracing::error!("failed to get versions from Curseforge: {}", err);
}
};
}
/// Metadata that represents a [`Package`].
#[cfg_attr(feature = "specta", derive(specta::Type))]
#[derive(Serialize, Deserialize, Clone, Debug)]
@ -528,14 +659,7 @@ pub enum PackageMetadata {
title: String,
version_id: String,
version_formatted: String,
},
Mapped {
title: Option<String>,
description: Option<String>,
authors: Vec<String>,
version: Option<String>,
icon: Option<PathBuf>,
package_type: Option<PackageType>,
mc_versions: Option<Vec<String>>,
},
Unknown,
}
@ -550,6 +674,7 @@ impl PackageMetadata {
title: package.title,
provider: package.provider,
package_type: package.package_type,
mc_versions: Some(version.game_versions)
}
}
@ -557,8 +682,7 @@ impl PackageMetadata {
pub const fn get_package_type(&self) -> Option<PackageType> {
match self {
Self::Managed { package_type, .. } => Some(*package_type),
Self::Mapped { package_type, .. } => *package_type,
Self::Unknown => None,
_ => None,
}
}
}

51
packages/core/src/utils/crypto.rs Normal file
View file

@ -0,0 +1,51 @@
use std::path::PathBuf;
use whoami::Result;
pub fn sha1(bytes: &[u8]) -> String {
sha1_smol::Sha1::from(bytes).digest().to_string()
}
pub fn sha1_file(file: &PathBuf) -> Result<String> {
Ok(sha1(&std::fs::read(file)?))
}
pub fn sha1_files(files: Vec<&PathBuf>) -> Vec<Option<String>> {
let mut hashes = Vec::new();
for file in files {
hashes.push(sha1_file(file).ok());
}
hashes
}
const CURSEFORGE_FINGERPRINT_SEED: u32 = 1;
pub fn murmur2(bytes: &[u8]) -> u32 {
let normalized = normalize_byte_array(bytes);
murmur2::murmur2(normalized.as_slice(), CURSEFORGE_FINGERPRINT_SEED)
}
pub fn murmur2_file(file: &PathBuf) -> Result<u32> {
Ok(murmur2(&std::fs::read(file)?))
}
pub fn murmur2_files(files: Vec<&PathBuf>) -> Vec<Option<u32>> {
let mut hashes = Vec::new();
for file in files {
hashes.push(murmur2_file(file).ok());
}
hashes
}
// Curseforge and their documented code :D (it wasn't documented)
// https://github.com/CurseForgeCommunity/.NET-APIClient/blob/2c4f5f613d20025f9286fdd53592f8519022918f/Murmur2.cs#L8-L21
// https://github.com/comp500/infra.link/blob/a298502fdb15b8939fe0effe71ed42ad06f3946f/murmur2/go/main.go#L10-L40
fn normalize_byte_array(bytes: &[u8]) -> Vec<u8> {
let normalized = bytes
.iter()
.filter(|byte| !matches!(byte, 9 | 10 | 13 | 32))
.copied()
.collect::<Vec<u8>>();
// println!("Old len {}", bytes.len());
// println!("New len {}", normalized.len());
normalized
}

1
packages/core/src/utils/mod.rs
View file

@ -9,6 +9,7 @@ pub mod http;
pub mod java;
pub mod pagination;
pub mod watcher;
pub mod crypto;
#[cfg(feature = "tauri")]
pub mod window;