diff --git a/Cargo.lock b/Cargo.lock index 27b5d525..b1f2240e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4583,6 +4583,7 @@ dependencies = [ "sqlx", "thiserror 2.0.18", "tokio", + "toml", "tracing", "url", "uuid 1.23.1", diff --git a/Cargo.toml b/Cargo.toml index ab0d1c44..b3b9168a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ serde = {version = "=1.0.228"} serde_json = {version = "=1.0.150"} serde_repr = {version = "=0.1.20"} sysinfo = {version = "=0.39.5"} +toml = {version = "=1.1.4", default-features = false, features = ["parse", "serde"]} trash = {version = "=5.2.6"} url = {version = "=2.5.8", features = ["serde"]} uuid = {version = "=1.23.1", features = ["serde", "v4"]} diff --git a/packages/oneclient_app/src/components/file_drop.rs b/packages/oneclient_app/src/components/file_drop.rs index d4f6c6a8..e10dc522 100644 --- a/packages/oneclient_app/src/components/file_drop.rs +++ b/packages/oneclient_app/src/components/file_drop.rs @@ -330,9 +330,7 @@ fn prompt_body( let import_list = resolved.clone(); let import = move |_| { - for (path, content_type) in &import_list { - dispatch.import_local_file(cluster_id, *content_type, path.clone()); - } + dispatch.import_local_files(cluster_id, import_list.clone()); pending.set(Vec::new()); }; diff --git a/packages/oneclient_app/src/components/package_row.rs b/packages/oneclient_app/src/components/package_row.rs index 026ee8c8..40dba488 100644 --- a/packages/oneclient_app/src/components/package_row.rs +++ b/packages/oneclient_app/src/components/package_row.rs @@ -219,11 +219,7 @@ pub(crate) fn grid_card( ) -> Element { let remote = item.is_remote(); let enabled = item.enabled; - let title = if remote { - item.name.clone() - } else { - item.file_name.clone() - }; + let title = item.name.clone(); let (bg, border, content_alpha) = if enabled { (colors::brand().with_a(38), colors::brand(), 255) @@ -345,11 +341,7 @@ fn package_info( let provider = item.provider; let package_id = item.package_id.clone(); let package_type = package_type.to_string(); - let title = if remote { - item.name.clone() - } else { - item.file_name.clone() - }; + let title = item.name.clone(); rect() .horizontal() @@ -420,18 +412,7 @@ pub(crate) fn package_icon( icon_query: &freya::query::UseQuery, size: f32, ) -> Element { - if item.is_remote() { - remote_icon(&item.icon_url, icon_query, size) - } else { - local_icon(size) - } -} - -fn remote_icon( - icon_url: &Option, - icon_query: &freya::query::UseQuery, - size: f32, -) -> Element { + let icon_url = &item.icon_url; let loaded = loaded_image(icon_url.as_deref(), icon_query); match loaded { @@ -442,14 +423,11 @@ fn remote_icon( .corner_radius(CornerRadius::new_all(8.)) .into_element(), - None => icon_box(IconType::DotsGrid, size), + None if icon_url.is_some() => icon_box(IconType::DotsGrid, size), + None => icon_box(IconType::HelpCircle, size), } } -fn local_icon(size: f32) -> Element { - icon_box(IconType::HelpCircle, size) -} - fn icon_box(icon: IconType, size: f32) -> Element { rect() .center() diff --git a/packages/oneclient_app/src/hooks/actions.rs b/packages/oneclient_app/src/hooks/actions.rs index 0298234b..1e4d8efd 100644 --- a/packages/oneclient_app/src/hooks/actions.rs +++ b/packages/oneclient_app/src/hooks/actions.rs @@ -14,6 +14,7 @@ use oneclient_cluster::{ ClusterStage, ClusterUpdate, GameSettingsProfile, PackageUpdateMode, ProfileUpdate, }; use oneclient_common::domain::{ContentType, ProviderId}; +use oneclient_content::packages::{LiveSync, LocalImportReport}; use oneclient_core::relocate::RelocationPlan; use oneclient_core::settings::LauncherSettings; use oneclient_core::settings::store::{save_global_profile, save_settings_and_apply}; @@ -911,36 +912,43 @@ impl Actions { } pub fn import_local_file(&self, cluster_id: ClusterId, content_type: ContentType, path: PathBuf) { + self.import_local_files(cluster_id, vec![(path, content_type)]); + } + + pub fn import_local_files( + &self, + cluster_id: ClusterId, + files: Vec<(PathBuf, ContentType)>, + ) { + if files.is_empty() { + return; + } + spawn_forever(async move { let Ok(state) = launcher::state() else { return }; let events = state.services.events.clone(); - match oneclient_content::packages::PackageStore::import_local_file( - &path, - content_type, + match oneclient_content::packages::PackageStore::import_local_files( + &files, cluster_id, &state.services.content(), ) .await { - Ok(row) => { - let live = oneclient_content::packages::PackageStore::sync_live_content( - cluster_id, - &row, - &state.services.content(), - ) - .await - .unwrap_or(oneclient_content::packages::LiveSync::Skipped); + Ok(report) => { + let mut deferred = false; + for row in &report.imported { + let live = oneclient_content::packages::PackageStore::sync_live_content( + cluster_id, + row, + &state.services.content(), + ) + .await + .unwrap_or(LiveSync::Skipped); - let mut body = format!("Added {}", row.file_name); - if live == oneclient_content::packages::LiveSync::Deferred - && state.games.is_active(cluster_id) - { - body.push_str( - ". Minecraft is running, so it will be there at the next launch", - ); + deferred |= live == LiveSync::Deferred; } - events.notify("Imported").body(body).send(); + notify_import(&events, &report, deferred && state.games.is_active(cluster_id)); super::invalidate_cluster_queries().await; } Err(err) => events @@ -1813,6 +1821,39 @@ async fn repair_and_relaunch( } } +fn notify_import(events: &oneclient_events::EventBus, report: &LocalImportReport, deferred: bool) { + let failed = report.failed.len(); + + let Some(mut body) = (match report.imported.as_slice() { + [] => None, + [only] => Some(format!("Added {}", only.file_name)), + rows => Some(format!("Added {} files", rows.len())), + }) else { + let reason = report + .failed + .first() + .map_or_else(|| "Nothing could be read".to_string(), |(_, err)| err.to_string()); + + events.notify("Import failed").body(reason).error().send(); + return; + }; + + if failed > 0 { + body.push_str(&format!(", {failed} could not be read")); + } + + if deferred { + body.push_str(". Minecraft is running, so it will be there at the next launch"); + } + + if failed == 0 { + events.notify("Imported").body(body).send(); + return; + } + + events.notify("Imported").body(body).error().send(); +} + #[must_use = "the notification is not raised until `.send()` is called"] pub struct NotificationBuilder { actions: Actions, diff --git a/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs b/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs index c5a060b1..29153440 100644 --- a/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs +++ b/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs @@ -52,6 +52,14 @@ fn provider_project_ids( ids } +fn local_project_ids(content: &[LinkedArtifactInfo], content_type: ContentType) -> Vec { + content + .iter() + .filter(|info| info.content_type == content_type && info.provider.is_none()) + .map(|info| info.hash.clone()) + .collect() +} + pub fn use_content_meta( content: &[LinkedArtifactInfo], bundles: &[BundleWithUpdateStatus], @@ -65,6 +73,15 @@ pub fn use_content_meta( out.insert((provider, project_id), meta); } } + + let local = use_package_meta_batch( + ProviderId::Local, + local_project_ids(content, content_type), + ); + for (hash, meta) in package_meta_batch(&local) { + out.insert((ProviderId::Local, hash), meta); + } + out } diff --git a/packages/oneclient_app/src/view/app/cluster/package_manager/views.rs b/packages/oneclient_app/src/view/app/cluster/package_manager/views.rs index 1f6e2535..0c3548c4 100644 --- a/packages/oneclient_app/src/view/app/cluster/package_manager/views.rs +++ b/packages/oneclient_app/src/view/app/cluster/package_manager/views.rs @@ -53,13 +53,7 @@ impl SortMode { } pub(super) fn sort(self, rows: &mut [PackageEntry]) { - let title = |p: &PackageEntry| { - if p.is_remote() { - p.name.to_lowercase() - } else { - p.file_name.to_lowercase() - } - }; + let title = |p: &PackageEntry| p.name.to_lowercase(); match self { SortMode::NameDesc => rows.sort_by_key(|p| std::cmp::Reverse(title(p))), @@ -521,13 +515,13 @@ fn add_from_file_button( .pick_files() .await { - for handle in handles { - dispatch.import_local_file( - cluster_id, - content_type, - handle.path().to_path_buf(), - ); - } + dispatch.import_local_files( + cluster_id, + handles + .iter() + .map(|handle| (handle.path().to_path_buf(), content_type)) + .collect(), + ); } }); }) diff --git a/packages/oneclient_common/src/paths.rs b/packages/oneclient_common/src/paths.rs index c16140d6..10c3a754 100644 --- a/packages/oneclient_common/src/paths.rs +++ b/packages/oneclient_common/src/paths.rs @@ -155,6 +155,22 @@ pub fn images_cache_dir() -> PathsResult { Ok(caches_dir()?.join("images")) } +pub const LOCAL_IMAGE_SCHEME: &str = "local:"; + +pub fn local_icons_dir() -> PathsResult { + Ok(images_cache_dir()?.join("local")) +} + +pub fn local_image_path(reference: &str) -> Option { + let name = reference.strip_prefix(LOCAL_IMAGE_SCHEME)?; + + if name.is_empty() || name.contains(['/', '\\']) || name.contains("..") { + return None; + } + + Some(local_icons_dir().ok()?.join(name)) +} + pub fn profiles_cache_dir() -> PathsResult { Ok(caches_dir()?.join("profiles")) } diff --git a/packages/oneclient_content/Cargo.toml b/packages/oneclient_content/Cargo.toml index ae88b929..960914e6 100644 --- a/packages/oneclient_content/Cargo.toml +++ b/packages/oneclient_content/Cargo.toml @@ -35,6 +35,7 @@ serde.workspace = true serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true +toml.workspace = true tracing.workspace = true url.workspace = true uuid.workspace = true diff --git a/packages/oneclient_content/src/packages/local_manifest.rs b/packages/oneclient_content/src/packages/local_manifest.rs new file mode 100644 index 00000000..619b72a2 --- /dev/null +++ b/packages/oneclient_content/src/packages/local_manifest.rs @@ -0,0 +1,528 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use serde::Deserialize; + +const MAX_DESCRIPTION: usize = 600; +const MAX_DESCRIPTION_WINDOW: usize = MAX_DESCRIPTION * 8; + +const FABRIC: &str = "fabric.mod.json"; +const QUILT: &str = "quilt.mod.json"; +const NEOFORGE: &str = "META-INF/neoforge.mods.toml"; +const FORGE: &str = "META-INF/mods.toml"; +const LEGACY_FORGE: &str = "mcmod.info"; + +const MANIFESTS: [&str; 5] = [FABRIC, QUILT, NEOFORGE, FORGE, LEGACY_FORGE]; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct JarManifest { + pub name: Option, + pub description: Option, + pub authors: Vec, + pub icon_entry: Option, +} + +impl JarManifest { + #[must_use] + pub fn is_empty(&self) -> bool { + self.name.is_none() + && self.description.is_none() + && self.authors.is_empty() + && self.icon_entry.is_none() + } + + #[must_use] + pub fn author_line(&self) -> String { + self.authors.join(", ") + } + + fn fill_from(&mut self, other: Self) { + if self.name.is_none() { + self.name = other.name; + } + if self.description.is_none() { + self.description = other.description; + } + if self.authors.is_empty() { + self.authors = other.authors; + } + if self.icon_entry.is_none() { + self.icon_entry = other.icon_entry; + } + } +} + +#[tracing::instrument(level = "debug", fields(jar = %jar.display()))] +pub async fn read_jar_manifest(jar: &Path) -> JarManifest { + let entries = match polyio::read_zip_file_entries(jar, |name| MANIFESTS.contains(&name)).await { + Ok(entries) => entries, + Err(err) => { + tracing::debug!("could not read {}: {err}", jar.display()); + return JarManifest::default(); + } + }; + + let mut out = JarManifest::default(); + for wanted in MANIFESTS { + let Some((_, bytes)) = entries.iter().find(|(name, _)| name == wanted) else { + continue; + }; + let Ok(text) = std::str::from_utf8(bytes) else { + continue; + }; + + let parsed = match wanted { + FABRIC => parse_fabric(text), + QUILT => parse_quilt(text), + NEOFORGE | FORGE => parse_forge(text), + _ => parse_legacy_forge(text), + }; + + match parsed { + Some(parsed) => out.fill_from(parsed), + None => tracing::debug!("{wanted} in {} did not parse", jar.display()), + } + } + + out +} + +#[tracing::instrument(level = "debug", fields(jar = %jar.display()))] +pub async fn read_jar_icon(jar: &Path, entry: &str) -> Option> { + let entry = entry.to_string(); + let found = polyio::read_zip_file_entries(jar, |name| name == entry) + .await + .inspect_err(|err| tracing::debug!("no icon in {}: {err}", jar.display())) + .ok()?; + + found.into_iter().next().map(|(_, bytes)| bytes) +} + +#[must_use] +pub fn parse_fabric(text: &str) -> Option { + #[derive(Deserialize)] + struct FabricMod { + name: Option, + description: Option, + #[serde(default)] + authors: Vec, + icon: Option, + } + + let parsed: FabricMod = serde_json::from_str(text).ok()?; + + Some(JarManifest { + name: clean(parsed.name), + description: clean_description(parsed.description), + authors: people(parsed.authors), + icon_entry: parsed.icon.and_then(Icon::path).and_then(entry_path), + }) +} + +#[must_use] +pub fn parse_quilt(text: &str) -> Option { + #[derive(Deserialize)] + struct QuiltMod { + quilt_loader: QuiltLoader, + } + + #[derive(Deserialize)] + struct QuiltLoader { + metadata: Option, + } + + #[derive(Deserialize)] + struct QuiltMetadata { + name: Option, + description: Option, + #[serde(default)] + contributors: BTreeMap, + icon: Option, + } + + let parsed: QuiltMod = serde_json::from_str(text).ok()?; + let metadata = parsed.quilt_loader.metadata?; + + Some(JarManifest { + name: clean(metadata.name), + description: clean_description(metadata.description), + authors: metadata + .contributors + .into_keys() + .filter_map(|name| clean(Some(name))) + .collect(), + icon_entry: metadata.icon.and_then(Icon::path).and_then(entry_path), + }) +} + +#[must_use] +pub fn parse_forge(text: &str) -> Option { + #[derive(Deserialize)] + struct ForgeManifest { + #[serde(default)] + mods: Vec, + /// Some older jars put it above the mod list rather than inside it + #[serde(rename = "logoFile")] + logo_file: Option, + } + + #[derive(Deserialize)] + struct ForgeMod { + #[serde(rename = "displayName")] + display_name: Option, + description: Option, + authors: Option, + credits: Option, + #[serde(rename = "logoFile")] + logo_file: Option, + } + + let parsed: ForgeManifest = toml::from_str(text).ok()?; + let first = parsed.mods.into_iter().next()?; + + Some(JarManifest { + name: clean(first.display_name), + description: clean_description(first.description), + authors: first + .authors + .or(first.credits) + .map(StringOrList::into_people) + .unwrap_or_default(), + icon_entry: first.logo_file.or(parsed.logo_file).and_then(entry_path), + }) +} + +#[must_use] +pub fn parse_legacy_forge(text: &str) -> Option { + #[derive(Deserialize)] + #[serde(untagged)] + enum McModInfo { + Bare(Vec), + Wrapped { + #[serde(rename = "modList")] + mod_list: Vec, + }, + } + + #[derive(Deserialize)] + struct LegacyMod { + name: Option, + description: Option, + #[serde(rename = "authorList", default)] + author_list: Vec, + credits: Option, + #[serde(rename = "logoFile")] + logo_file: Option, + } + + let mods = match serde_json::from_str::(text).ok()? { + McModInfo::Bare(mods) | McModInfo::Wrapped { mod_list: mods } => mods, + }; + let first = mods.into_iter().next()?; + + let authors: Vec = first + .author_list + .into_iter() + .filter_map(|name| clean(Some(name))) + .collect(); + + Some(JarManifest { + name: clean(first.name), + description: clean_description(first.description), + authors: if authors.is_empty() { + clean(first.credits).into_iter().collect() + } else { + authors + }, + icon_entry: first.logo_file.and_then(entry_path), + }) +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum Icon { + Path(String), + Sizes(BTreeMap), +} + +impl Icon { + fn path(self) -> Option { + match self { + Self::Path(path) => Some(path), + Self::Sizes(sizes) => sizes + .into_iter() + .max_by_key(|(size, _)| size.parse::().unwrap_or(0)) + .map(|(_, path)| path), + } + } +} + +fn people(list: Vec) -> Vec { + list.into_iter() + .filter_map(|person| match person { + serde_json::Value::String(name) => clean(Some(name)), + serde_json::Value::Object(mut fields) => match fields.remove("name") { + Some(serde_json::Value::String(name)) => clean(Some(name)), + _ => None, + }, + _ => None, + }) + .collect() +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum StringOrList { + One(String), + Many(Vec), +} + +impl StringOrList { + fn into_people(self) -> Vec { + match self { + Self::One(text) => clean(Some(text)).into_iter().collect(), + Self::Many(list) => list.into_iter().filter_map(|name| clean(Some(name))).collect(), + } + } +} + +fn clean(value: Option) -> Option { + let value = value?.trim().to_string(); + (!value.is_empty()).then_some(value) +} + +fn clean_description(value: Option) -> Option { + let raw = clean(value)?; + + // Collapsing whitespace only shortens so a window this wide still holds + // words to spare and a megabyte long blurb is never copied whole + let end = raw + .char_indices() + .nth(MAX_DESCRIPTION_WINDOW) + .map_or(raw.len(), |(idx, _)| idx); + + let mut text = raw[..end].split_whitespace().collect::>().join(" "); + if text.is_empty() { + return None; + } + + if text.chars().count() > MAX_DESCRIPTION { + let cut = text + .char_indices() + .nth(MAX_DESCRIPTION) + .map_or(text.len(), |(idx, _)| idx); + text.truncate(cut); + text.push('…'); + } else if end < raw.len() { + text.push('…'); + } + + Some(text) +} + +fn entry_path(raw: String) -> Option { + let path = raw + .trim() + .replace('\\', "/") + .trim_start_matches('/') + .trim_start_matches("./") + .to_string(); + + if path.is_empty() || path.split('/').any(|part| part == "..") { + return None; + } + + Some(path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fabric_reads_every_field() { + let parsed = parse_fabric( + r#"{ + "schemaVersion": 1, + "id": "sodium", + "name": "Sodium", + "description": "A modern\n rendering engine", + "authors": ["JellySquid", {"name": "Someone", "contact": {}}], + "icon": "assets/sodium/icon.png" + }"#, + ) + .expect("valid fabric manifest"); + + assert_eq!(parsed.name.as_deref(), Some("Sodium")); + assert_eq!( + parsed.description.as_deref(), + Some("A modern rendering engine"), + "hard wrapping collapses so the card lays out its own lines" + ); + assert_eq!(parsed.author_line(), "JellySquid, Someone"); + assert_eq!(parsed.icon_entry.as_deref(), Some("assets/sodium/icon.png")); + } + + #[test] + fn fabric_icon_map_takes_the_largest() { + let parsed = parse_fabric( + r#"{"id": "x", "icon": {"32": "small.png", "128": "big.png", "64": "mid.png"}}"#, + ) + .expect("valid fabric manifest"); + + assert_eq!(parsed.icon_entry.as_deref(), Some("big.png")); + } + + #[test] + fn fabric_survives_an_author_shaped_oddly() { + let parsed = parse_fabric(r#"{"id": "x", "name": "X", "authors": [["nested"], "Real"]}"#) + .expect("one odd author must not sink the manifest"); + + assert_eq!(parsed.author_line(), "Real"); + } + + #[test] + fn quilt_reads_contributor_names() { + let parsed = parse_quilt( + r#"{ + "schema_version": 1, + "quilt_loader": { + "id": "example", + "metadata": { + "name": "Example", + "description": "Does things", + "contributors": {"Alice": "Owner", "Bob": "Author"}, + "icon": "assets/example/icon.png" + } + } + }"#, + ) + .expect("valid quilt manifest"); + + assert_eq!(parsed.name.as_deref(), Some("Example")); + assert_eq!(parsed.author_line(), "Alice, Bob"); + assert_eq!(parsed.icon_entry.as_deref(), Some("assets/example/icon.png")); + } + + #[test] + fn forge_reads_the_first_mod_entry() { + let parsed = parse_forge( + r#" +modLoader="javafml" +loaderVersion="[47,)" +license="MIT" + +[[mods]] +modId="examplemod" +version="1.0.0" +displayName="Example Mod" +authors="Someone" +logoFile="examplemod.png" +description=''' +A mod that +spans lines +''' + +[[dependencies.examplemod]] +modId="forge" +mandatory=true +"#, + ) + .expect("valid forge manifest"); + + assert_eq!(parsed.name.as_deref(), Some("Example Mod")); + assert_eq!(parsed.description.as_deref(), Some("A mod that spans lines")); + assert_eq!(parsed.author_line(), "Someone"); + assert_eq!(parsed.icon_entry.as_deref(), Some("examplemod.png")); + } + + #[test] + fn forge_falls_back_to_credits_and_a_root_logo() { + let parsed = parse_forge( + r#" +logoFile="logo.png" + +[[mods]] +modId="x" +displayName="X" +credits="Thanks to everyone" +"#, + ) + .expect("valid forge manifest"); + + assert_eq!(parsed.author_line(), "Thanks to everyone"); + assert_eq!(parsed.icon_entry.as_deref(), Some("logo.png")); + } + + #[test] + fn legacy_forge_reads_both_shapes() { + let bare = parse_legacy_forge( + r#"[{"modid":"x","name":"X","description":"Old","authorList":["A","B"],"logoFile":"/logo.png"}]"#, + ) + .expect("valid mcmod.info"); + + assert_eq!(bare.name.as_deref(), Some("X")); + assert_eq!(bare.author_line(), "A, B"); + assert_eq!( + bare.icon_entry.as_deref(), + Some("logo.png"), + "the leading slash is jar relative not absolute" + ); + + let wrapped = parse_legacy_forge( + r#"{"modListVersion":2,"modList":[{"modid":"y","name":"Y","authorList":["C"]}]}"#, + ) + .expect("valid v2 mcmod.info"); + + assert_eq!(wrapped.name.as_deref(), Some("Y")); + } + + #[test] + fn a_second_manifest_only_fills_gaps() { + let mut fabric = JarManifest { + name: Some("Fabric Name".into()), + ..JarManifest::default() + }; + fabric.fill_from(JarManifest { + name: Some("Forge Name".into()), + description: Some("From forge".into()), + authors: vec!["Someone".into()], + icon_entry: Some("logo.png".into()), + }); + + assert_eq!(fabric.name.as_deref(), Some("Fabric Name")); + assert_eq!(fabric.description.as_deref(), Some("From forge")); + assert_eq!(fabric.author_line(), "Someone"); + assert_eq!(fabric.icon_entry.as_deref(), Some("logo.png")); + } + + #[test] + fn traversing_icon_paths_are_refused() { + assert_eq!(entry_path("../../etc/passwd".into()), None); + assert_eq!(entry_path(" ".into()), None); + assert_eq!(entry_path("./icon.png".into()).as_deref(), Some("icon.png")); + } + + #[test] + fn long_descriptions_are_cut_on_a_char_boundary() { + let long = "ż".repeat(MAX_DESCRIPTION + 50); + let cut = clean_description(Some(long)).expect("non empty"); + + assert_eq!( + cut.chars().count(), + MAX_DESCRIPTION + 1, + "the ellipsis is the extra one" + ); + assert!(cut.ends_with('…')); + } + + #[test] + fn garbage_is_not_a_manifest() { + assert!(parse_fabric("not json").is_none()); + assert!(parse_forge("[[mods").is_none()); + assert!( + parse_forge("modLoader=\"javafml\"").is_none(), + "no mod entry to describe" + ); + } +} diff --git a/packages/oneclient_content/src/packages/metadata_cache.rs b/packages/oneclient_content/src/packages/metadata_cache.rs index e9a5c88f..89a39537 100644 --- a/packages/oneclient_content/src/packages/metadata_cache.rs +++ b/packages/oneclient_content/src/packages/metadata_cache.rs @@ -192,6 +192,10 @@ pub async fn fetch_package_meta( ); } + if provider == ProviderId::Local { + return Ok(out); + } + let missing: Vec = project_ids .iter() .filter(|id| out.get(*id).is_none_or(|m| m.author.is_empty())) diff --git a/packages/oneclient_content/src/packages/mod.rs b/packages/oneclient_content/src/packages/mod.rs index 9f8d4962..cf97c60c 100644 --- a/packages/oneclient_content/src/packages/mod.rs +++ b/packages/oneclient_content/src/packages/mod.rs @@ -1,6 +1,7 @@ pub mod activity; pub mod dependencies; pub mod error; +pub mod local_manifest; pub mod metadata_cache; pub mod modpack; pub mod provider; @@ -21,9 +22,10 @@ pub use dependencies::{ DependencyResolution, ResolvedDependency, resolve_required, resolves_dependencies, }; pub use file_identity::{curseforge_fingerprint, FileIdentity}; +pub use local_manifest::{JarManifest, read_jar_icon, read_jar_manifest}; pub use error::{PackageError, PackageResult}; pub use provider::{PackageProvider, PackageProviderRegistry}; -pub use store::{LiveSync, PackageStore}; +pub use store::{LiveSync, LocalImportReport, PackageStore}; pub use types::*; pub use updates::{ BrowserPackageUpdate, BrowserUpdateCheck, apply_browser_package_update, diff --git a/packages/oneclient_content/src/packages/provider/registry.rs b/packages/oneclient_content/src/packages/provider/registry.rs index da7b7071..f7b2cfa7 100644 --- a/packages/oneclient_content/src/packages/provider/registry.rs +++ b/packages/oneclient_content/src/packages/provider/registry.rs @@ -8,7 +8,7 @@ use oneclient_common::domain::ProviderId; use crate::packages::error::PackageError; use crate::packages::file_identity::FileIdentity; use crate::packages::store::artifact_absolute_path; -use crate::packages::types::{VersionDetail, VersionLookup}; +use crate::packages::types::{ProviderVersionLookup, VersionDetail}; use crate::ctx::ContentCtx; #[derive(Clone)] @@ -57,22 +57,37 @@ impl PackageProviderRegistry { &self, identities: &[FileIdentity], ctx: &ContentCtx, - ) -> ContentResult { + ) -> ContentResult { if identities.is_empty() { return Ok(HashMap::new()); } let mut enriched: Vec = identities.to_vec(); - for identity in &mut enriched { - enrich_curseforge_fingerprint(identity, ctx).await?; - } + let mut merged: ProviderVersionLookup = HashMap::new(); - let mut merged = HashMap::new(); for id in self.remote_ids() { + if id == ProviderId::CurseForge { + for identity in enriched + .iter_mut() + .filter(|identity| !merged.contains_key(&identity.sha1)) + { + if let Err(err) = enrich_curseforge_fingerprint(identity, ctx).await { + tracing::debug!(sha1 = %identity.sha1, "could not fingerprint: {err}"); + } + } + } + let provider = self.get(id)?; - let found = provider.lookup_versions(&enriched, ctx).await?; + let found = match provider.lookup_versions(&enriched, ctx).await { + Ok(found) => found, + Err(err) => { + tracing::debug!(provider = ?id, "batch lookup failed: {err}"); + continue; + } + }; + for (sha1, version) in found { - merged.entry(sha1).or_insert(version); + merged.entry(sha1).or_insert((id, version)); } } Ok(merged) @@ -84,19 +99,8 @@ impl PackageProviderRegistry { sha1: impl AsRef, ctx: &ContentCtx, ) -> ContentResult> { - let mut identity = FileIdentity::from_sha1(sha1); - enrich_curseforge_fingerprint(&mut identity, ctx).await?; - - for id in self.remote_ids() { - let provider = self.get(id)?; - let mut found = provider - .lookup_versions(std::slice::from_ref(&identity), ctx) - .await?; - if let Some(version) = found.remove(&identity.sha1) { - return Ok(Some((id, version))); - } - } - Ok(None) + let identity = FileIdentity::from_sha1(sha1); + self.first_match(identity, ctx).await } #[tracing::instrument(level = "debug", skip(self, ctx), fields(sha1 = %identity.sha1))] @@ -105,10 +109,19 @@ impl PackageProviderRegistry { identity: &FileIdentity, ctx: &ContentCtx, ) -> ContentResult> { - let mut identity = identity.clone(); - enrich_curseforge_fingerprint(&mut identity, ctx).await?; + self.first_match(identity.clone(), ctx).await + } + async fn first_match( + &self, + mut identity: FileIdentity, + ctx: &ContentCtx, + ) -> ContentResult> { for id in self.remote_ids() { + if id == ProviderId::CurseForge { + enrich_curseforge_fingerprint(&mut identity, ctx).await?; + } + let provider = self.get(id)?; let mut found = provider .lookup_versions(std::slice::from_ref(&identity), ctx) @@ -147,9 +160,15 @@ async fn enrich_curseforge_fingerprint( return Ok(()); } - let bytes = polyio::read(&path).await?; - identity.cf_fingerprint = Some(crate::packages::file_identity::curseforge_fingerprint( - &bytes, - )); + // Whole jar read plus a per byte murmur2 the caller runs on freya's + // UI-thread executor so it goes to the blocking pool + let fingerprint = tokio::task::spawn_blocking(move || { + std::fs::read(&path) + .map(|bytes| crate::packages::file_identity::curseforge_fingerprint(&bytes)) + }) + .await + .map_err(std::io::Error::other)??; + + identity.cf_fingerprint = Some(fingerprint); Ok(()) } diff --git a/packages/oneclient_content/src/packages/store/mod.rs b/packages/oneclient_content/src/packages/store/mod.rs index 4217086e..7ad450d9 100644 --- a/packages/oneclient_content/src/packages/store/mod.rs +++ b/packages/oneclient_content/src/packages/store/mod.rs @@ -15,20 +15,34 @@ pub use link::{ }; pub use paths::{artifact_absolute_path, cache_file_path, relative_cache_path}; -use oneclient_db::dao::{artifact as artifact_dao, cluster as cluster_dao}; +use oneclient_db::dao::{ + artifact as artifact_dao, cluster as cluster_dao, package_metadata as meta_dao, +}; use oneclient_db::models::{ArtifactRow, ClusterRow, SeenStatus}; use oneclient_common::domain::{ContentType, GameLoader, ProviderId}; +// `paths` alone is this module's own cache-path helpers +use oneclient_common::paths as common_paths; use super::error::PackageError; +use super::file_identity::FileIdentity; +use super::{local_manifest, metadata_cache}; use super::types::{CachedArtifact, ProjectDetail, ProviderReleaseInfo, VersionDetail, LinkedArtifactInfo}; use polyio::{normalize_hash, sha1_file}; use oneclient_events::GroupedProgressChild; use crate::ctx::ContentCtx; use crate::error::{ContentError, ContentResult}; -use std::path::Path; +use std::path::{Path, PathBuf}; pub struct PackageStore; +/// One bad file in a selection must not sink the rest so failures ride along +/// with the successes instead of replacing them +#[derive(Debug, Default)] +pub struct LocalImportReport { + pub imported: Vec, + pub failed: Vec<(PathBuf, ContentError)>, +} + impl PackageStore { #[tracing::instrument(level = "debug", skip(ctx))] pub async fn get_cluster( @@ -299,6 +313,8 @@ impl PackageStore { Ok((enabled, live)) } + /// Prefer [`Self::import_local_files`] for anything the user selected in one + /// go it asks the providers about the whole set in a single request #[tracing::instrument(level = "debug", skip(ctx))] pub async fn import_local_file( path: &Path, @@ -306,50 +322,46 @@ impl PackageStore { cluster_id: i64, ctx: &ContentCtx, ) -> ContentResult { - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| PackageError::InvalidLocalFile(path.display().to_string()))? - .to_string(); - - let hash = normalize_hash(&sha1_file(path).await?); - - if let Some(row) = artifact_dao::get_artifact_by_hash(&ctx.db, &hash).await? { - let cluster = Self::get_cluster(cluster_id, ctx).await?; - Self::link_artifact(&row, &cluster, None, ctx).await?; - return Ok(row); - } - - let dest = cache_file_path( - content_type, - ProviderId::Local, - "imported", - &hash[..hash.len().min(16)], - &file_name, - )?; - if let Some(parent) = dest.parent() { - polyio::create_dir_all(parent).await?; - } - polyio::copy(path, &dest).await?; - - let size = polyio::stat(&dest).await?.len(); - let stored_path = relative_cache_path(&dest)?; - - let row = artifact_dao::insert_artifact( - &ctx.db, - &hash, - content_type as i64, - &stored_path, - &file_name, - Some(size as i64), - ) - .await?; - let cluster = Self::get_cluster(cluster_id, ctx).await?; - Self::link_artifact(&row, &cluster, None, ctx).await?; + let row = store_local_file(path, content_type, &cluster, ctx).await?; + + describe_imports(&[(row.clone(), content_type)], ctx).await; Ok(row) } + /// A whole drop at once so identifying twenty jars costs the two requests + /// one jar would rather than forty + /// + /// A file that cannot be stored is reported rather than returned one + /// unreadable jar must not sink the rest of the selection + #[tracing::instrument(level = "debug", skip(files, ctx), fields(files = files.len()))] + pub async fn import_local_files( + files: &[(PathBuf, ContentType)], + cluster_id: i64, + ctx: &ContentCtx, + ) -> ContentResult { + let cluster = Self::get_cluster(cluster_id, ctx).await?; + + let mut report = LocalImportReport::default(); + let mut stored = Vec::with_capacity(files.len()); + + for (path, content_type) in files { + match store_local_file(path, *content_type, &cluster, ctx).await { + Ok(row) => { + stored.push((row.clone(), *content_type)); + report.imported.push(row); + } + Err(err) => { + tracing::warn!("could not import {}: {err}", path.display()); + report.failed.push((path.clone(), err)); + } + } + } + + describe_imports(&stored, ctx).await; + Ok(report) + } + #[tracing::instrument(level = "debug", skip(ctx))] pub async fn resolve_or_download( provider_id: ProviderId, @@ -369,6 +381,215 @@ impl PackageStore { } } +/// Copies the file into the cache unless an identical one is already there and +/// links it to the cluster working out what it is comes after +async fn store_local_file( + path: &Path, + content_type: ContentType, + cluster: &ClusterRow, + ctx: &ContentCtx, +) -> ContentResult { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| PackageError::InvalidLocalFile(path.display().to_string()))? + .to_string(); + + let hash = normalize_hash(&sha1_file(path).await?); + + if let Some(row) = artifact_dao::get_artifact_by_hash(&ctx.db, &hash).await? { + PackageStore::link_artifact(&row, cluster, None, ctx).await?; + return Ok(row); + } + + let dest = cache_file_path( + content_type, + ProviderId::Local, + "imported", + &hash[..hash.len().min(16)], + &file_name, + )?; + if let Some(parent) = dest.parent() { + polyio::create_dir_all(parent).await?; + } + polyio::copy(path, &dest).await?; + + let size = polyio::stat(&dest).await?.len(); + let stored_path = relative_cache_path(&dest)?; + + let row = artifact_dao::insert_artifact( + &ctx.db, + &hash, + content_type as i64, + &stored_path, + &file_name, + Some(size as i64), + ) + .await?; + + PackageStore::link_artifact(&row, cluster, None, ctx).await?; + Ok(row) +} + +/// A dropped file arrives as a file name and nothing else so it is worth +/// finding out what it actually is before it lands in the list +/// +/// The providers get asked first and for the whole batch at once because a jar +/// downloaded by hand from Modrinth or CurseForge is the same file the browser +/// would have installed and deserves the same card an update check included +/// the jar's own manifest only answers for what neither of them recognises +/// +/// Never fails an unidentified mod is still a perfectly good import +#[tracing::instrument(level = "debug", skip(imports, ctx), fields(files = imports.len()))] +async fn describe_imports(imports: &[(ArtifactRow, ContentType)], ctx: &ContentCtx) { + let mut unknown = Vec::new(); + for (row, content_type) in imports { + if !already_described(row, ctx).await { + unknown.push((row, *content_type)); + } + } + + if unknown.is_empty() { + return; + } + + let identities: Vec = unknown + .iter() + .map(|(row, _)| FileIdentity::from_sha1(&row.hash)) + .collect(); + + // Offline or rate limited every jar then falls through to its own manifest + let found = ctx + .providers + .lookup_versions(&identities, ctx) + .await + .inspect_err(|err| tracing::debug!("provider lookup failed: {err}")) + .unwrap_or_default(); + + for (row, content_type) in unknown { + if let Some((provider, version)) = found.get(&row.hash) { + match record_release(*provider, version, &row.hash, ctx).await { + Ok(()) => continue, + Err(err) => { + tracing::debug!("could not record a release for {}: {err}", row.file_name); + } + } + } + + if let Err(err) = record_manifest(row, content_type, ctx).await { + tracing::debug!("could not read {} for metadata: {err}", row.file_name); + } + } +} + +/// The same file may already have arrived through the browser or an earlier +/// import and describing it again buys nothing +async fn already_described(row: &ArtifactRow, ctx: &ContentCtx) -> bool { + if artifact_dao::get_release_by_hash(&ctx.db, &row.hash) + .await + .ok() + .flatten() + .is_some() + { + return true; + } + + !metadata_cache::read_cached_package_meta( + ctx, + ProviderId::Local, + std::slice::from_ref(&row.hash), + ) + .await + .is_empty() +} + +/// Written exactly as a download would have so the row joins the update flow +/// rather than sitting outside it +async fn record_release( + provider: ProviderId, + version: &VersionDetail, + hash: &str, + ctx: &ContentCtx, +) -> ContentResult<()> { + let published_at = version.published.to_rfc3339(); + + artifact_dao::upsert_provider_release( + &ctx.db, + provider as i64, + &version.project_id, + &version.version_id, + hash, + &version.name, + &version.version_number, + Some(published_at.as_str()), + &serde_json::to_string(&version.game_versions)?, + &serde_json::to_string(&version.loaders)?, + ) + .await?; + + Ok(()) +} + +/// Cached under the artifact hash because a mod nobody recognises has no +/// project id to be keyed by +async fn record_manifest( + row: &ArtifactRow, + content_type: ContentType, + ctx: &ContentCtx, +) -> ContentResult<()> { + // Only mods carry a loader manifest resource packs and shaders describe + // themselves too but in formats that name neither an author nor a mod + if content_type != ContentType::Mod { + return Ok(()); + } + + let jar = artifact_absolute_path(&row.path)?; + let manifest = local_manifest::read_jar_manifest(&jar).await; + + let icon = match &manifest.icon_entry { + Some(entry) => store_local_icon(&row.hash, &jar, entry).await, + None => None, + }; + + // A jar that parsed to nothing still gets a row so `already_described` stops + // both providers being asked about it on every later import + meta_dao::upsert_package_metadata( + &ctx.db, + ProviderId::Local as i64, + &row.hash, + manifest.name.as_deref().unwrap_or(&row.file_name), + manifest.description.as_deref().unwrap_or_default(), + &manifest.author_line(), + icon.as_deref(), + ) + .await?; + + Ok(()) +} + +/// Kept out of the package cache which the collector sweeps by artifact path +/// and would take an icon sitting next to its jar for an orphan +async fn store_local_icon(hash: &str, jar: &std::path::Path, entry: &str) -> Option { + let bytes = local_manifest::read_jar_icon(jar, entry).await?; + if bytes.is_empty() { + return None; + } + + let extension = std::path::Path::new(entry) + .extension() + .and_then(|ext| ext.to_str()) + .filter(|ext| ext.len() <= 4 && ext.chars().all(|c| c.is_ascii_alphanumeric())) + .unwrap_or("png") + .to_ascii_lowercase(); + let name = format!("{hash}.{extension}"); + + let dir = common_paths::local_icons_dir().ok()?; + polyio::create_dir_all(&dir).await.ok()?; + polyio::write(dir.join(&name), &bytes).await.ok()?; + + Some(format!("{}{name}", common_paths::LOCAL_IMAGE_SCHEME)) +} + #[tracing::instrument(level = "debug", skip(row, ctx), fields(hash = %row.hash))] async fn row_to_cached( row: ArtifactRow, diff --git a/packages/oneclient_content/src/packages/types.rs b/packages/oneclient_content/src/packages/types.rs index b5457a81..2a5d7b2f 100644 --- a/packages/oneclient_content/src/packages/types.rs +++ b/packages/oneclient_content/src/packages/types.rs @@ -221,6 +221,7 @@ pub struct ProviderReleaseInfo { } pub type VersionLookup = HashMap; +pub type ProviderVersionLookup = HashMap; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LinkedArtifactInfo { diff --git a/packages/oneclient_core/src/images/mod.rs b/packages/oneclient_core/src/images/mod.rs index 67fc5bfd..67eee52b 100644 --- a/packages/oneclient_core/src/images/mod.rs +++ b/packages/oneclient_core/src/images/mod.rs @@ -44,7 +44,11 @@ impl ImageCacheStore { return Ok(bytes); } - let raw = download(net, url).await?; + // icon unpacked for a jar is already on disk + let raw = match paths::local_image_path(url) { + Some(source) => Bytes::from(polyio::read(&source).await?), + None => download(net, url).await?, + }; let bytes = downscale_if_oversized(raw, max_edge).await; if let Some(parent) = path.parent() { @@ -87,8 +91,26 @@ async fn downscale_if_oversized(bytes: Bytes, max_edge: u32) -> Bytes { } } +/// Icons come out of user supplied jars so the decoder gets a ceiling rather +/// than whatever the file's header claims +const MAX_DECODE_EDGE: u32 = 8192; +const MAX_DECODE_ALLOC: u64 = 128 * 1024 * 1024; + fn downscale(bytes: &[u8], max_edge: u32) -> Option { - let img = image::load_from_memory(bytes).ok()?; + let mut limits = image::Limits::default(); + limits.max_image_width = Some(MAX_DECODE_EDGE); + limits.max_image_height = Some(MAX_DECODE_EDGE); + limits.max_alloc = Some(MAX_DECODE_ALLOC); + + let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)); + reader.limits(limits); + + let img = reader + .with_guessed_format() + .ok()? + .decode() + .inspect_err(|err| tracing::warn!("refused to decode an image: {err}")) + .ok()?; if img.width().max(img.height()) <= max_edge { return None; } diff --git a/packages/polyio/src/archive.rs b/packages/polyio/src/archive.rs index 948b16f7..c29d964a 100644 --- a/packages/polyio/src/archive.rs +++ b/packages/polyio/src/archive.rs @@ -201,8 +201,14 @@ pub async fn extract_zip_filtered( Ok(()) } +/// Ceiling on a single entry read by [`read_zip_file_entries`] mod manifests, +/// icons and bundle overrides sit far below it a decompression bomb does not +pub const MAX_ZIP_ENTRY_BYTES: u64 = 16 * 1024 * 1024; + /// Each matching entry is read fully into memory so this is for small files /// not large archives +/// +/// Entries claiming or producing more than [`MAX_ZIP_ENTRY_BYTES`] are skipped #[tracing::instrument( level = "debug", skip(zip_path, filter), @@ -228,9 +234,31 @@ pub async fn read_zip_file_entries( continue; } - let mut entry_reader = reader.reader_without_entry(index).await?; + if entry.uncompressed_size() > MAX_ZIP_ENTRY_BYTES { + tracing::warn!( + "skipping '{name}' in {}: declares {} bytes over the {MAX_ZIP_ENTRY_BYTES} byte entry cap", + zip_path.display(), + entry.uncompressed_size() + ); + continue; + } + + let entry_reader = reader.reader_without_entry(index).await?; let mut data = Vec::new(); - futures_lite::AsyncReadExt::read_to_end(&mut entry_reader, &mut data).await?; + futures_lite::AsyncReadExt::read_to_end( + &mut futures_lite::AsyncReadExt::take(entry_reader, MAX_ZIP_ENTRY_BYTES + 1), + &mut data, + ) + .await?; + + if data.len() as u64 > MAX_ZIP_ENTRY_BYTES { + tracing::warn!( + "skipping '{name}' in {}: decompressed past the {MAX_ZIP_ENTRY_BYTES} byte entry cap", + zip_path.display() + ); + continue; + } + out.push((name, data)); }