From b3b436bc3e572d83916bcf07862bfa2f4597fbd0 Mon Sep 17 00:00:00 2001 From: Maciek <108873945+rozwader@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:30:50 +0200 Subject: [PATCH] feat: added support for github hosted bundle mods (#883) * feat: added support for github hosted bundle mods * fix: keep external bundle mods when their manifest entry gains an id * fix: match a pending external bundle update to its installed jar * chore: parse the bundle file host with url instead of by hand * fix: namespace external bundle file ids under ext: * chore: rename the Browser tab to Online and list unbundled GitHub mods there --------- Co-authored-by: LynithDev <61880709+LynithDev@users.noreply.github.com> --- .../oneclient_app/assets/icons/github.svg | 1 + .../src/components/cluster_update_popup.rs | 6 +- .../oneclient_app/src/components/icons.rs | 1 + packages/oneclient_app/src/components/mod.rs | 4 +- .../src/components/optional_mods_popup.rs | 7 +- .../src/components/package_row.rs | 16 +- packages/oneclient_app/src/install.rs | 7 +- packages/oneclient_app/src/notifications.rs | 2 + .../view/app/cluster/package_manager/mod.rs | 70 +++++-- .../src/view/onboarding/bundles/mod.rs | 38 ++-- .../src/view/onboarding/bundles/row.rs | 27 ++- .../src/view/onboarding/test_support.rs | 18 +- .../oneclient_content/src/bundles/install.rs | 171 +++++++++++++----- .../oneclient_content/src/bundles/manager.rs | 42 ++++- .../src/bundles/polymrpack.rs | 65 ++++++- .../oneclient_content/src/bundles/types.rs | 73 +++++++- .../oneclient_content/src/bundles/updates.rs | 152 +++++++--------- 17 files changed, 484 insertions(+), 216 deletions(-) create mode 100644 packages/oneclient_app/assets/icons/github.svg diff --git a/packages/oneclient_app/assets/icons/github.svg b/packages/oneclient_app/assets/icons/github.svg new file mode 100644 index 00000000..1d0c61b7 --- /dev/null +++ b/packages/oneclient_app/assets/icons/github.svg @@ -0,0 +1 @@ + diff --git a/packages/oneclient_app/src/components/cluster_update_popup.rs b/packages/oneclient_app/src/components/cluster_update_popup.rs index 156a52f4..eee9c61c 100644 --- a/packages/oneclient_app/src/components/cluster_update_popup.rs +++ b/packages/oneclient_app/src/components/cluster_update_popup.rs @@ -110,7 +110,11 @@ impl Component for ClusterUpdatePopup { }) .collect(); let mut meta = MetaMap::new(); - for provider in ProviderId::REMOTE_PROVIDERS.iter().copied() { + for provider in ProviderId::REMOTE_PROVIDERS + .iter() + .copied() + .chain([ProviderId::Local]) + { let ids: Vec = all_items .iter() .filter(|i| i.provider == provider) diff --git a/packages/oneclient_app/src/components/icons.rs b/packages/oneclient_app/src/components/icons.rs index 0675cf11..0f65f866 100644 --- a/packages/oneclient_app/src/components/icons.rs +++ b/packages/oneclient_app/src/components/icons.rs @@ -115,6 +115,7 @@ pub enum IconType { Folder, FolderCheck, FolderDownload, + Github, Globe01, HelpCircle, IconLogo, diff --git a/packages/oneclient_app/src/components/mod.rs b/packages/oneclient_app/src/components/mod.rs index 6f1ac851..fffaa212 100644 --- a/packages/oneclient_app/src/components/mod.rs +++ b/packages/oneclient_app/src/components/mod.rs @@ -81,7 +81,9 @@ pub use optional_mods_popup::OptionalModsPopup; pub use overlay_popup::{OVERLAY_BASE_LEVEL, OVERLAY_MAX_LEVEL, OverlayPopup, overlay_is_topmost}; pub use package_delete_dialog::use_shared_delete; pub(crate) use package_row::{CARD_GRID_H, GRID_GAP, GRID_MIN_W, grid_card, package_icon}; -pub use package_row::{CardLayout, PackageEntry, PackageRow, package_context_menu, provider_badge}; +pub use package_row::{ + CardLayout, PackageEntry, PackageRow, github_badge, package_context_menu, provider_badge, +}; pub use package_update_popup::PackageUpdatePopup; pub use pagination::Pagination; pub use player_model::PlayerModel; diff --git a/packages/oneclient_app/src/components/optional_mods_popup.rs b/packages/oneclient_app/src/components/optional_mods_popup.rs index f0dc3cc5..1a53f157 100644 --- a/packages/oneclient_app/src/components/optional_mods_popup.rs +++ b/packages/oneclient_app/src/components/optional_mods_popup.rs @@ -42,7 +42,11 @@ impl Component for OptionalModsPopup { let groups = snapshot.optional_mods.clone(); let mut meta = MetaMap::new(); - for provider in ProviderId::REMOTE_PROVIDERS.iter().copied() { + for provider in ProviderId::REMOTE_PROVIDERS + .iter() + .copied() + .chain([ProviderId::Local]) + { let ids: Vec = groups .iter() .flatten() @@ -100,6 +104,7 @@ fn entry_from_item( package_id, bundle_name: item.offer.as_ref().map(|(bundle, _)| bundle.clone()), provider: item.provider, + github_hosted: item.github_hosted, name: cached .map(|cached| cached.name.clone()) .filter(|name| !name.is_empty()) diff --git a/packages/oneclient_app/src/components/package_row.rs b/packages/oneclient_app/src/components/package_row.rs index 5e4525be..0aed716f 100644 --- a/packages/oneclient_app/src/components/package_row.rs +++ b/packages/oneclient_app/src/components/package_row.rs @@ -37,6 +37,7 @@ pub struct PackageEntry { pub package_id: String, pub bundle_name: Option, pub provider: ProviderId, + pub github_hosted: bool, pub name: String, pub file_name: String, pub author: String, @@ -398,7 +399,9 @@ fn grid_meta( ) -> Element { let muted = CARD_NAME.with_a(scale_a(alpha, 0.5)); - let source = if item.is_remote() && navigable { + let source = if item.github_hosted { + meta_text("GitHub".to_string(), muted) + } else if item.is_remote() && navigable { SourceLink { provider: item.provider, package_id: item.package_id.clone(), @@ -615,7 +618,9 @@ fn package_info( .max_width(Size::percent(60.)) .color(CARD_NAME), ) - .child(if remote { + .child(if item.github_hosted { + github_badge() + } else if remote { provider_badge(item.provider) } else { local_badge() @@ -704,6 +709,13 @@ pub fn provider_badge(provider: ProviderId) -> Element { ) } +pub fn github_badge() -> Element { + badge( + Icon::new(IconType::Github).size(12.).into_element(), + "GitHub".to_string(), + ) +} + fn outdated_badge() -> Element { accent_badge( Icon::new(IconType::RefreshCw01) diff --git a/packages/oneclient_app/src/install.rs b/packages/oneclient_app/src/install.rs index 770ae4b7..4355b548 100644 --- a/packages/oneclient_app/src/install.rs +++ b/packages/oneclient_app/src/install.rs @@ -46,14 +46,16 @@ pub fn item_from_bundle_file(file: &oneclient_core::BundleFile) -> ClusterUpdate .. } => ClusterUpdateItem { provider: *provider, + github_hosted: false, project_id: Some(project_id.clone()), fallback: file.display_name(), offer: None, status: None, }, - oneclient_core::BundleFileKind::External(_) => ClusterUpdateItem { + oneclient_core::BundleFileKind::External { .. } => ClusterUpdateItem { provider: oneclient_content::packages::ProviderId::Local, - project_id: None, + github_hosted: file.is_github_hosted(), + project_id: Some(file.kind.metadata_id()), fallback: file.display_name(), offer: None, status: None, @@ -90,6 +92,7 @@ async fn cluster_update_summary( provider: r .provider .unwrap_or(oneclient_content::packages::ProviderId::Local), + github_hosted: false, project_id: r.project_id.clone(), fallback: r .display_name diff --git a/packages/oneclient_app/src/notifications.rs b/packages/oneclient_app/src/notifications.rs index af634ebb..21330f41 100644 --- a/packages/oneclient_app/src/notifications.rs +++ b/packages/oneclient_app/src/notifications.rs @@ -47,6 +47,7 @@ pub enum OptionalModsOutcome { #[derive(Clone, Debug, PartialEq)] pub struct ClusterUpdateItem { pub provider: ProviderId, + pub github_hosted: bool, pub project_id: Option, /// Used when the meta cache has no entry (file name / package id) pub fallback: String, @@ -58,6 +59,7 @@ impl ClusterUpdateItem { pub fn from_name(name: impl Into) -> Self { Self { provider: ProviderId::Local, + github_hosted: false, project_id: None, fallback: name.into(), offer: None, diff --git a/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs b/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs index cb4ab4d0..81a9c5b0 100644 --- a/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs +++ b/packages/oneclient_app/src/view/app/cluster/package_manager/mod.rs @@ -3,7 +3,9 @@ use std::collections::{HashMap, HashSet}; use freya::prelude::*; use oneclient_common::search::{MatchScore, SearchQuery}; use oneclient_content::packages::{CachedPackageMeta, ContentType, ProviderId}; -use oneclient_core::{BundleFileKind, BundleWithUpdateStatus, LinkedArtifactInfo}; +use oneclient_core::{ + BundleFileKind, BundleWithUpdateStatus, FileUpdateStatus, LinkedArtifactInfo, +}; use oneclient_db::models::OverrideType; use crate::components::{CARD_GRID_H, CardLayout, GRID_GAP, GRID_MIN_W, PackageEntry}; @@ -52,11 +54,22 @@ fn provider_project_ids( ids } -fn local_project_ids(content: &[LinkedArtifactInfo], content_type: ContentType) -> Vec { +fn local_project_ids( + content: &[LinkedArtifactInfo], + bundles: &[BundleWithUpdateStatus], + content_type: ContentType, +) -> Vec { + let bundled = bundles + .iter() + .flat_map(|bundle| &bundle.files) + .filter(|(file, _status)| file.content_type() == content_type) + .filter(|(file, _status)| matches!(file.kind, BundleFileKind::External { .. })) + .map(|(file, _status)| file.kind.metadata_id()); content .iter() .filter(|info| info.content_type == content_type && info.provider.is_none()) .map(|info| info.hash.clone()) + .chain(bundled) .collect() } @@ -74,7 +87,10 @@ pub fn use_content_meta( } } - let local = use_package_meta_batch(ProviderId::Local, local_project_ids(content, content_type)); + let local = use_package_meta_batch( + ProviderId::Local, + local_project_ids(content, bundles, content_type), + ); for (hash, meta) in package_meta_batch(&local) { out.insert((ProviderId::Local, hash), meta); } @@ -117,7 +133,7 @@ pub fn bundle_packages( for bundle in bundles { let bundle_name = &bundle.archive.manifest.name; let category = bundle.archive.manifest.category.clone(); - for (file, _status) in &bundle.files { + for (file, status) in &bundle.files { if file.content_type() != content_type { continue; } @@ -130,14 +146,24 @@ pub fn bundle_packages( continue; } - let provider = match &file.kind { - BundleFileKind::Managed { provider, .. } => *provider, - BundleFileKind::External(_) => ProviderId::Local, - }; - let installed_info = by_project - .get(pid.as_str()) - .or_else(|| by_hash.get(pid.as_str())) - .copied(); + let provider = file.kind.metadata_provider(); + let installed_info = match &file.kind { + BundleFileKind::Managed { .. } => by_project + .get(pid.as_str()) + .or_else(|| by_hash.get(pid.as_str())), + BundleFileKind::External { file: ext, .. } => { + let hash = match status { + FileUpdateStatus::UpdateAvailable { + installed_version_id, + .. + } => installed_version_id, + _ => &ext.sha1, + }; + seen.insert(hash.clone()); + by_hash.get(hash.as_str()) + } + } + .copied(); let ov = overrides .get(&(bundle_name.clone(), pid.clone())) .map(String::as_str); @@ -160,8 +186,9 @@ pub fn bundle_packages( categories, enabled, file.enabled, + file.is_github_hosted(), installed_info, - meta, + meta.get(&(provider, file.kind.metadata_id())), file.display_name(), false, // Flagged rather than dropped `HiddenFilter` filters on the row and the seen id stops the loose-content pass resurrecting it as a local file @@ -179,6 +206,7 @@ pub fn bundle_packages( let provider = info.provider.unwrap_or(ProviderId::Local); let pid = info.project_id.clone().unwrap_or_else(|| info.hash.clone()); let outdated = stale.contains(&info.hash); + let row_meta = meta.get(&(provider, pid.clone())); rows.push(make_row( pid, None, @@ -187,8 +215,9 @@ pub fn bundle_packages( Vec::new(), info.enabled, true, + false, Some(info), - meta, + row_meta, info.display_name .clone() .unwrap_or_else(|| info.file_name.clone()), @@ -209,13 +238,13 @@ fn make_row( categories: Vec, enabled: bool, manifest_default: bool, + github_hosted: bool, installed_info: Option<&LinkedArtifactInfo>, - meta: &PackageMetaMap, + m: Option<&CachedPackageMeta>, fallback_name: String, update_available: bool, hidden: bool, ) -> PackageEntry { - let m = meta.get(&(provider, package_id.clone())); let name = m .map(|p| p.name.clone()) .filter(|s| !s.is_empty()) @@ -237,6 +266,7 @@ fn make_row( package_id, bundle_name, provider, + github_hosted, name, file_name, author, @@ -268,7 +298,7 @@ impl Tab { match self { Tab::All => "All".to_string(), Tab::Category(c) => c.clone(), - Tab::Browser => "Browser".to_string(), + Tab::Browser => "Online".to_string(), Tab::Local => "Local".to_string(), } } @@ -277,8 +307,8 @@ impl Tab { match self { Tab::All => true, Tab::Category(c) => p.categories.iter().any(|pc| pc == c), - Tab::Browser => p.is_remote() && !p.in_bundle(), - Tab::Local => !p.is_remote(), + Tab::Browser => (p.is_remote() || p.github_hosted) && !p.in_bundle(), + Tab::Local => !p.is_remote() && !p.github_hosted, } } } @@ -331,7 +361,7 @@ fn build_tabs(categories: &[String], items: &[PackageEntry], hidden: HiddenFilte } } - // Category tabs are hidden when empty All + Browser + Local are always shown + // Category tabs are hidden when empty All + Online + Local are always shown let mut tabs: Vec = vec![Tab::All]; tabs.extend( cats.into_iter() diff --git a/packages/oneclient_app/src/view/onboarding/bundles/mod.rs b/packages/oneclient_app/src/view/onboarding/bundles/mod.rs index 647d13d3..03276484 100644 --- a/packages/oneclient_app/src/view/onboarding/bundles/mod.rs +++ b/packages/oneclient_app/src/view/onboarding/bundles/mod.rs @@ -2,8 +2,8 @@ use std::collections::{HashMap, HashSet}; use freya::prelude::*; use oneclient_content::packages::ProviderId; +use oneclient_core::BundleArchive; use oneclient_core::clusters::Cluster; -use oneclient_core::{BundleArchive, BundleFile, BundleFileKind}; use crate::components::ScrollArea; use crate::hooks::{ @@ -67,13 +67,6 @@ fn fps_warning_banner() -> impl IntoElement { .child(text) } -fn file_provider(file: &BundleFile) -> ProviderId { - match &file.kind { - BundleFileKind::Managed { provider, .. } => *provider, - BundleFileKind::External(_) => ProviderId::Local, - } -} - fn bundle_display_name(archive: &BundleArchive) -> String { let category = archive.manifest.category.trim(); if category.is_empty() { @@ -149,7 +142,9 @@ fn opt_in_bundles(clusters: &[ClusterBundles]) -> Vec { #[derive(Clone)] struct OptionalMod { package_id: String, + metadata_id: String, provider: ProviderId, + github_hosted: bool, fallback_name: String, size: u64, /// `(cluster_id, bundle_name)` pairs this mod is available from @@ -186,7 +181,9 @@ fn optional_mods(clusters: &[ClusterBundles]) -> Vec { order.push(package_id.clone()); OptionalMod { package_id: package_id.clone(), - provider: file_provider(file), + metadata_id: file.kind.metadata_id(), + provider: file.kind.metadata_provider(), + github_hosted: file.is_github_hosted(), fallback_name: file.display_name(), size: file.size, locations: Vec::new(), @@ -259,9 +256,10 @@ impl Component for OnboardingBundles { let extras = optional_mods(&clusters); // Hooks so they run every render regardless of whether there are extras yet - let (mr_ids, cf_ids) = collect_ids(&extras); + let (mr_ids, cf_ids, local_ids) = collect_ids(&extras); let mr_meta = package_meta_batch(&use_package_meta_batch(ProviderId::Modrinth, mr_ids)); let cf_meta = package_meta_batch(&use_package_meta_batch(ProviderId::CurseForge, cf_ids)); + let local_meta = package_meta_batch(&use_package_meta_batch(ProviderId::Local, local_ids)); let mut sections: Vec = Vec::new(); @@ -278,6 +276,7 @@ impl Component for OnboardingBundles { user_touched, &mr_meta, &cf_meta, + &local_meta, )); } @@ -441,14 +440,15 @@ fn extras_section( user_touched: State, mr_meta: &MetaMap, cf_meta: &MetaMap, + local_meta: &MetaMap, ) -> Element { let cards: Vec = extras .iter() .map(|entry| { let meta = match entry.provider { - ProviderId::Modrinth => mr_meta.get(&entry.package_id), - ProviderId::CurseForge => cf_meta.get(&entry.package_id), - ProviderId::Local => None, + ProviderId::Modrinth => mr_meta.get(&entry.metadata_id), + ProviderId::CurseForge => cf_meta.get(&entry.metadata_id), + ProviderId::Local => local_meta.get(&entry.metadata_id), }; let name = meta .map(|m| m.name.clone()) @@ -461,6 +461,7 @@ fn extras_section( OnboardingModCard { provider: entry.provider, + github_hosted: entry.github_hosted, name, author, description, @@ -518,15 +519,16 @@ fn extras_section( .into_element() } -fn collect_ids(extras: &[OptionalMod]) -> (Vec, Vec) { +fn collect_ids(extras: &[OptionalMod]) -> (Vec, Vec, Vec) { let mut mr = Vec::new(); let mut cf = Vec::new(); + let mut local = Vec::new(); for entry in extras { match entry.provider { - ProviderId::Modrinth => mr.push(entry.package_id.clone()), - ProviderId::CurseForge => cf.push(entry.package_id.clone()), - ProviderId::Local => {} + ProviderId::Modrinth => mr.push(entry.metadata_id.clone()), + ProviderId::CurseForge => cf.push(entry.metadata_id.clone()), + ProviderId::Local => local.push(entry.metadata_id.clone()), } } - (mr, cf) + (mr, cf, local) } diff --git a/packages/oneclient_app/src/view/onboarding/bundles/row.rs b/packages/oneclient_app/src/view/onboarding/bundles/row.rs index 705696ae..35d2d9fa 100644 --- a/packages/oneclient_app/src/view/onboarding/bundles/row.rs +++ b/packages/oneclient_app/src/view/onboarding/bundles/row.rs @@ -1,7 +1,7 @@ use freya::prelude::*; use oneclient_content::packages::ProviderId; -use crate::components::{Icon, provider_badge}; +use crate::components::{Icon, IconType, github_badge, provider_badge}; use crate::hooks::{loaded_image, use_cached_image}; use crate::theme::colors; use crate::ui::{ImageFallbackExt, border_all_color}; @@ -16,6 +16,7 @@ pub(super) const GRID_GAP: f32 = 10.; #[derive(PartialEq)] pub(super) struct OnboardingModCard { pub(super) provider: ProviderId, + pub(super) github_hosted: bool, pub(super) name: String, pub(super) author: String, pub(super) description: String, @@ -36,10 +37,10 @@ impl Component for OnboardingModCard { .height(Size::px(44.)) .aspect_ratio(AspectRatio::Min) .corner_radius(CornerRadius::new_all(8.)) - .fallback(icon_box(self.provider)) + .fallback(icon_box(self.placeholder_icon())) .into_element(), - None => icon_box(self.provider).into_element(), + None => icon_box(self.placeholder_icon()).into_element(), }; let (bg, border, alpha) = if self.enabled { @@ -82,7 +83,11 @@ impl Component for OnboardingModCard { .color(colors::fg_secondary().with_a(alpha)), ) }) - .child(provider_badge(self.provider)), + .child(if self.github_hosted { + github_badge() + } else { + provider_badge(self.provider) + }), ); let description = (!self.description.is_empty()).then(|| { @@ -132,14 +137,24 @@ impl Component for OnboardingModCard { } } -fn icon_box(provider: ProviderId) -> impl IntoElement { +impl OnboardingModCard { + fn placeholder_icon(&self) -> IconType { + if self.github_hosted { + IconType::Github + } else { + self.provider.into() + } + } +} + +fn icon_box(icon: IconType) -> impl IntoElement { rect() .center() .width(Size::px(44.)) .height(Size::px(44.)) .corner_radius(CornerRadius::new_all(8.)) .background(colors::component_bg()) - .child(Icon::new(provider).size(20.).color(colors::fg_secondary())) + .child(Icon::new(icon).size(20.).color(colors::fg_secondary())) .into_element() } diff --git a/packages/oneclient_app/src/view/onboarding/test_support.rs b/packages/oneclient_app/src/view/onboarding/test_support.rs index 87f25aa8..118f16a7 100644 --- a/packages/oneclient_app/src/view/onboarding/test_support.rs +++ b/packages/oneclient_app/src/view/onboarding/test_support.rs @@ -10,13 +10,17 @@ pub fn file(package_id: &str, enabled: bool, hidden: bool) -> BundleFile { hidden, path: format!("mods/{package_id}.jar"), size: 1, - kind: BundleFileKind::External(ExternalFile { - name: format!("{package_id}.jar"), - url: format!("https://example.invalid/{package_id}.jar"), - sha1: package_id.to_string(), - size: 1, - content_type: ContentType::Mod, - }), + kind: BundleFileKind::External { + file: ExternalFile { + name: format!("{package_id}.jar"), + url: format!("https://example.invalid/{package_id}.jar"), + sha1: package_id.to_string(), + size: 1, + content_type: ContentType::Mod, + }, + id: None, + meta: None, + }, } } diff --git a/packages/oneclient_content/src/bundles/install.rs b/packages/oneclient_content/src/bundles/install.rs index 9a890ce1..7706ac88 100644 --- a/packages/oneclient_content/src/bundles/install.rs +++ b/packages/oneclient_content/src/bundles/install.rs @@ -120,7 +120,7 @@ pub async fn install_package_from_bundle( .await?; artifact.hash } - BundleFileKind::External(ext) => { + BundleFileKind::External { file: ext, .. } => { install_external(ext, &cluster, skip_compatibility, child, ctx).await? } }; @@ -239,6 +239,74 @@ async fn clear_reconciler_disables( Ok(()) } +pub(crate) fn external_ids_by_sha1( + archives: &[BundleArchive], +) -> std::collections::HashMap { + archives + .iter() + .flat_map(|archive| &archive.manifest.files) + .filter_map(|file| match &file.kind { + BundleFileKind::External { + file: ext, + id: Some(id), + .. + } => Some((ext.sha1.clone(), id.clone())), + _ => None, + }) + .collect() +} + +async fn adopt_external_ids( + cluster_id: i64, + archives: &[BundleArchive], + ctx: &ContentCtx, +) -> ContentResult<()> { + let ids = external_ids_by_sha1(archives); + if ids.is_empty() { + return Ok(()); + } + + for row in bundle_dao::list_bundle_tracked(&ctx.db, cluster_id).await? { + let (Some(bundle_name), Some(version_id), Some(package_id)) = + (&row.bundle_name, &row.bundle_version_id, &row.package_id) + else { + continue; + }; + if *package_id != row.hash { + continue; + } + let Some(id) = ids.get(package_id) else { + continue; + }; + bundle_dao::track_bundle_artifact( + &ctx.db, + cluster_id, + &row.hash, + bundle_name, + version_id, + id, + ) + .await?; + } + + let overrides = bundle_dao::list_overrides(&ctx.db, cluster_id).await?; + for row in &overrides { + let Some(id) = ids.get(&row.package_id) else { + continue; + }; + let superseded = overrides + .iter() + .any(|other| other.bundle_name == row.bundle_name && other.package_id == *id); + if !superseded && let Some(override_type) = OverrideType::parse(&row.override_type) { + bundle_dao::save_override(&ctx.db, cluster_id, &row.bundle_name, id, override_type) + .await?; + } + bundle_dao::remove_override(&ctx.db, cluster_id, &row.bundle_name, &row.package_id).await?; + } + + Ok(()) +} + #[tracing::instrument(level = "debug", skip(archives, ctx))] pub async fn heal_bundle_activity( cluster_id: i64, @@ -246,6 +314,7 @@ pub async fn heal_bundle_activity( ctx: &ContentCtx, ) -> ContentResult<()> { clear_reconciler_disables(cluster_id, archives, ctx).await?; + adopt_external_ids(cluster_id, archives, ctx).await?; let tracked = bundle_dao::list_bundle_tracked(&ctx.db, cluster_id).await?; if tracked.iter().all(|row| row.enabled != 0) { @@ -332,18 +401,10 @@ pub async fn install_enabled_bundle_files( let bundle_name = archive.manifest.name.clone(); let mut installed = Vec::new(); - let linked = PackageStore::list_linked_artifacts(cluster_id, ctx).await?; - let mut linked_projects: std::collections::HashSet<&str> = std::collections::HashSet::new(); - let mut linked_hashes: std::collections::HashSet<&str> = std::collections::HashSet::new(); // "Already installed" means the database not disk // content lives in the cache between sessions so probing the folder would // reinstall everything - for info in &linked { - if let Some(pid) = &info.project_id { - linked_projects.insert(pid.as_str()); - } - linked_hashes.insert(info.hash.as_str()); - } + let present = PresentContent::load(cluster_id, ctx).await?; let to_install: Vec = archive .manifest @@ -351,16 +412,8 @@ pub async fn install_enabled_bundle_files( .iter() .filter(|file| { let package_id = file.kind.package_id(); - if !effective_enabled(file, find_override(&overrides, &bundle_name, &package_id)) { - return false; - } - let already_installed = match &file.kind { - BundleFileKind::Managed { project_id, .. } => { - linked_projects.contains(project_id.as_str()) - } - BundleFileKind::External(ext) => linked_hashes.contains(ext.sha1.as_str()), - }; - !already_installed + effective_enabled(file, find_override(&overrides, &bundle_name, &package_id)) + && !present.contains(file) }) .cloned() .collect(); @@ -425,6 +478,42 @@ pub async fn install_enabled_bundle_files( Ok(installed) } +struct PresentContent { + projects: std::collections::HashSet, + hashes: std::collections::HashSet, + tracked_ids: std::collections::HashSet, +} + +impl PresentContent { + async fn load(cluster_id: i64, ctx: &ContentCtx) -> ContentResult { + let linked = PackageStore::list_linked_artifacts(cluster_id, ctx).await?; + let tracked_ids = bundle_dao::list_bundle_tracked(&ctx.db, cluster_id) + .await? + .into_iter() + .filter_map(|row| row.package_id) + .collect(); + + Ok(Self { + projects: linked + .iter() + .filter_map(|info| info.project_id.clone()) + .collect(), + hashes: linked.into_iter().map(|info| info.hash).collect(), + tracked_ids, + }) + } + + fn contains(&self, file: &BundleFile) -> bool { + match &file.kind { + BundleFileKind::Managed { project_id, .. } => self.projects.contains(project_id), + BundleFileKind::External { file: ext, id, .. } => { + self.hashes.contains(&ext.sha1) + || id.as_ref().is_some_and(|id| self.tracked_ids.contains(id)) + } + } + } +} + /// Kept modest each fetch also costs a provider API call rate limited per-minute pub(crate) const BUNDLE_INSTALL_CONCURRENCY: usize = 6; @@ -445,32 +534,16 @@ pub async fn enabled_bundle_bytes( .archives_for(ctx, &cluster.mc_version, loader) .await?; let overrides = bundle_dao::list_overrides(&ctx.db, cluster_id).await?; - let linked = PackageStore::list_linked_artifacts(cluster_id, ctx).await?; - - let mut linked_projects: std::collections::HashSet<&str> = std::collections::HashSet::new(); - let mut linked_hashes: std::collections::HashSet<&str> = std::collections::HashSet::new(); - for info in &linked { - if let Some(pid) = &info.project_id { - linked_projects.insert(pid.as_str()); - } - linked_hashes.insert(info.hash.as_str()); - } + let present = PresentContent::load(cluster_id, ctx).await?; let mut total = 0u64; for archive in &archives { let bundle_name = &archive.manifest.name; for file in &archive.manifest.files { let package_id = file.kind.package_id(); - if !effective_enabled(file, find_override(&overrides, bundle_name, &package_id)) { - continue; - } - let already_installed = match &file.kind { - BundleFileKind::Managed { project_id, .. } => { - linked_projects.contains(project_id.as_str()) - } - BundleFileKind::External(ext) => linked_hashes.contains(ext.sha1.as_str()), - }; - if already_installed { + if !effective_enabled(file, find_override(&overrides, bundle_name, &package_id)) + || present.contains(file) + { continue; } total += file.size; @@ -832,13 +905,17 @@ mod tests { hidden: false, path: "mods/example.jar".to_string(), size: 1, - kind: BundleFileKind::External(ExternalFile { - name: "example.jar".to_string(), - url: "https://example.invalid/example.jar".to_string(), - sha1: "abc123".to_string(), - size: 1, - content_type: ContentType::Mod, - }), + kind: BundleFileKind::External { + file: ExternalFile { + name: "example.jar".to_string(), + url: "https://example.invalid/example.jar".to_string(), + sha1: "abc123".to_string(), + size: 1, + content_type: ContentType::Mod, + }, + id: None, + meta: None, + }, } } diff --git a/packages/oneclient_content/src/bundles/manager.rs b/packages/oneclient_content/src/bundles/manager.rs index 7b2e1153..24da00c9 100644 --- a/packages/oneclient_content/src/bundles/manager.rs +++ b/packages/oneclient_content/src/bundles/manager.rs @@ -4,16 +4,17 @@ use std::str::FromStr; use chrono::Utc; use oneclient_db::dao::bundle as bundle_dao; +use oneclient_db::dao::package_metadata as meta_dao; use oneclient_db::models::{BundleRow, NewBundle}; use tokio::sync::RwLock; use crate::bundles::error::BundleError; use crate::bundles::manifest::{BundleManifest as RemoteBundleManifest, RemoteBundleRef}; use crate::bundles::polymrpack; -use crate::bundles::types::BundleArchive; +use crate::bundles::types::{BundleArchive, BundleFileKind}; use crate::ctx::ContentCtx; use crate::error::{ContentError, ContentResult}; -use oneclient_common::domain::GameLoader; +use oneclient_common::domain::{GameLoader, ProviderId}; use oneclient_common::paths; use oneclient_net::RequestError; use oneclient_net::{EtagPolicy, fetch_cached}; @@ -149,21 +150,23 @@ impl BundlesManager { ) -> ContentResult> { let mut archives = Vec::new(); for bundle in self.list_for(ctx, mc_version, loader).await? { - let manifest = self.manifest_for_archive(&bundle.path).await?; + let manifest = self.manifest_for_archive(ctx, &bundle.path).await?; archives.push(BundleArchive { bundle, manifest }); } Ok(archives) } - #[tracing::instrument(level = "debug", skip(self))] + #[tracing::instrument(level = "debug", skip(self, ctx))] async fn manifest_for_archive( &self, + ctx: &ContentCtx, path: &Path, ) -> ContentResult { if let Some(manifest) = self.archive_cache.read().await.get(path) { return Ok(manifest.clone()); } let manifest = polymrpack::read_manifest_from_archive(path).await?; + record_external_metadata(&manifest, ctx).await; self.archive_cache .write() .await @@ -270,6 +273,37 @@ impl Default for BundlesManager { } } +#[tracing::instrument(level = "debug", skip_all, fields(bundle = %manifest.name))] +async fn record_external_metadata( + manifest: &crate::bundles::types::BundleManifest, + ctx: &ContentCtx, +) { + for bundle_file in &manifest.files { + let BundleFileKind::External { + file, + meta: Some(meta), + .. + } = &bundle_file.kind + else { + continue; + }; + + if let Err(err) = meta_dao::upsert_package_metadata( + &ctx.db, + ProviderId::Local as i64, + &file.sha1, + meta.name.as_deref().unwrap_or(&file.name), + meta.description.as_deref().unwrap_or_default(), + &meta.authors.join(", "), + meta.icon_url.as_deref(), + ) + .await + { + tracing::warn!(file = %file.name, error = %err, "failed to record bundle file metadata"); + } + } +} + struct FetchedManifest { manifest: RemoteBundleManifest, changed: bool, diff --git a/packages/oneclient_content/src/bundles/polymrpack.rs b/packages/oneclient_content/src/bundles/polymrpack.rs index f5245983..930161d5 100644 --- a/packages/oneclient_content/src/bundles/polymrpack.rs +++ b/packages/oneclient_content/src/bundles/polymrpack.rs @@ -6,7 +6,7 @@ use serde::Deserialize; use crate::bundles::error::BundleError; use crate::bundles::types::{ - BundleFile, BundleFileKind, BundleManifest, content_type_from_bundle_path, + BundleFile, BundleFileKind, BundleManifest, ExternalFileMeta, content_type_from_bundle_path, }; use crate::error::ContentResult; use crate::packages::types::ExternalFile; @@ -49,6 +49,22 @@ struct PolyMrpackFile { pub enabled: bool, #[serde(default)] pub hidden: bool, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub overrides: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct PolyMrpackFileOverrides { + #[serde(default)] + pub name: Option, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub authors: Vec, + #[serde(default)] + pub icon: Option, } #[derive(Debug, Deserialize)] @@ -165,12 +181,45 @@ fn parse_bundle_file(file: &PolyMrpackFile) -> Option { hidden: file.hidden, path: file.path.clone(), size: file.file_size, - kind: BundleFileKind::External(ExternalFile { - name: file_name, - url: download_url, - sha1: file.hashes.sha1.to_ascii_lowercase(), - size: file.file_size, - content_type: content_type_from_bundle_path(&file.path), - }), + kind: BundleFileKind::External { + file: ExternalFile { + name: file_name, + url: download_url, + sha1: file.hashes.sha1.to_ascii_lowercase(), + size: file.file_size, + content_type: content_type_from_bundle_path(&file.path), + }, + id: non_blank(file.id.as_deref()).map(|id| { + if id.starts_with(EXTERNAL_ID_PREFIX) { + id + } else { + format!("{EXTERNAL_ID_PREFIX}{id}") + } + }), + meta: file.overrides.as_ref().and_then(external_meta), + }, }) } + +const EXTERNAL_ID_PREFIX: &str = "ext:"; + +fn non_blank(value: Option<&str>) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +fn external_meta(overrides: &PolyMrpackFileOverrides) -> Option { + let meta = ExternalFileMeta { + name: non_blank(overrides.name.as_deref()), + description: non_blank(overrides.description.as_deref()), + authors: overrides + .authors + .iter() + .filter_map(|author| non_blank(Some(author))) + .collect(), + icon_url: non_blank(overrides.icon.as_deref()), + }; + (meta != ExternalFileMeta::default()).then_some(meta) +} diff --git a/packages/oneclient_content/src/bundles/types.rs b/packages/oneclient_content/src/bundles/types.rs index 42d2e4e3..0c77ed82 100644 --- a/packages/oneclient_content/src/bundles/types.rs +++ b/packages/oneclient_content/src/bundles/types.rs @@ -35,19 +35,39 @@ pub struct BundleFile { impl BundleFile { pub fn content_type(&self) -> ContentType { - if let BundleFileKind::External(ext) = &self.kind { - return ext.content_type; + if let BundleFileKind::External { file, .. } = &self.kind { + return file.content_type; } content_type_from_bundle_path(&self.path) } + pub fn is_github_hosted(&self) -> bool { + let BundleFileKind::External { file, .. } = &self.kind else { + return false; + }; + url::Url::parse(&file.url).is_ok_and(|url| { + url.host_str().is_some_and(|host| { + host == "github.com" + || host.ends_with(".github.com") + || host.ends_with(".githubusercontent.com") + }) + }) + } + pub fn display_name(&self) -> String { + if let BundleFileKind::External { + meta: Some(meta), .. + } = &self.kind + && let Some(name) = &meta.name + { + return name.clone(); + } let from_path = self.path.rsplit('/').next().filter(|s| !s.is_empty()); if let Some(name) = from_path { return name.to_string(); } match &self.kind { - BundleFileKind::External(ext) => ext.name.clone(), + BundleFileKind::External { file, .. } => file.name.clone(), BundleFileKind::Managed { project_id, .. } => project_id.clone(), } } @@ -61,21 +81,58 @@ pub enum BundleFileKind { version_id: String, sha1: String, }, - External(ExternalFile), + External { + file: ExternalFile, + id: Option, + meta: Option, + }, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExternalFileMeta { + pub name: Option, + pub description: Option, + pub authors: Vec, + pub icon_url: Option, } impl BundleFileKind { pub fn package_id(&self) -> String { match self { Self::Managed { project_id, .. } => project_id.clone(), - Self::External(ext) => ext.sha1.clone(), + Self::External { file, id, .. } => id.clone().unwrap_or_else(|| file.sha1.clone()), } } pub fn bundle_version_id(&self) -> String { match self { Self::Managed { version_id, .. } => version_id.clone(), - Self::External(ext) => ext.sha1.clone(), + Self::External { file, .. } => file.sha1.clone(), + } + } + + pub fn metadata_id(&self) -> String { + match self { + Self::Managed { project_id, .. } => project_id.clone(), + Self::External { file, .. } => file.sha1.clone(), + } + } + + pub fn metadata_provider(&self) -> ProviderId { + match self { + Self::Managed { provider, .. } => *provider, + Self::External { .. } => ProviderId::Local, + } + } + + pub fn bundle_key(&self) -> String { + match self { + Self::Managed { + provider, + project_id, + .. + } => managed_bundle_key(*provider, project_id), + Self::External { .. } => external_bundle_key(&self.package_id()), } } } @@ -84,8 +141,8 @@ pub fn managed_bundle_key(provider: ProviderId, package_id: &str) -> String { format!("m:{}:{package_id}", provider.dir_name()) } -pub fn external_bundle_key(sha1: &str) -> String { - format!("e:{sha1}") +pub fn external_bundle_key(package_id: &str) -> String { + format!("e:{package_id}") } pub fn content_type_from_bundle_path(path: &str) -> ContentType { diff --git a/packages/oneclient_content/src/bundles/updates.rs b/packages/oneclient_content/src/bundles/updates.rs index 6cfd50e7..52030d4b 100644 --- a/packages/oneclient_content/src/bundles/updates.rs +++ b/packages/oneclient_content/src/bundles/updates.rs @@ -17,9 +17,9 @@ use tokio::sync::Mutex as AsyncMutex; use futures_util::StreamExt; use crate::bundles::install::{ - BUNDLE_INSTALL_CONCURRENCY, disable_was_deliberate, find_user_suppression, - heal_bundle_activity, install_package_from_bundle, remove_artifact_from_cluster, - set_artifact_enabled_to, + BUNDLE_INSTALL_CONCURRENCY, disable_was_deliberate, external_ids_by_sha1, + find_user_suppression, heal_bundle_activity, install_package_from_bundle, + remove_artifact_from_cluster, set_artifact_enabled_to, }; use crate::bundles::manager::BundlesManager; use crate::bundles::overrides; @@ -144,14 +144,7 @@ async fn check_bundle_updates_inner( let mut shipped_keys = HashSet::new(); for file in &archive.manifest.files { - shipped_keys.insert(match &file.kind { - BundleFileKind::Managed { - provider, - project_id, - .. - } => managed_bundle_key(*provider, project_id), - BundleFileKind::External(ext) => external_bundle_key(&ext.sha1), - }); + shipped_keys.insert(file.kind.bundle_key()); let user_override = overrides_map .get(&(archive.manifest.name.clone(), file.kind.package_id())) @@ -159,26 +152,10 @@ async fn check_bundle_updates_inner( if !crate::bundles::effective_enabled(file, user_override) { continue; } - match &file.kind { - BundleFileKind::Managed { - provider, - project_id, - version_id, - .. - } => { - let key = managed_bundle_key(*provider, project_id); - files_map.insert(key.clone(), (version_id.clone(), file.clone())); - if file.hidden { - hidden_explicit_keys.insert(key); - } - } - BundleFileKind::External(ext) => { - let key = external_bundle_key(&ext.sha1); - files_map.insert(key.clone(), (ext.sha1.clone(), file.clone())); - if file.hidden { - hidden_explicit_keys.insert(key); - } - } + let key = file.kind.bundle_key(); + files_map.insert(key.clone(), (file.kind.bundle_version_id(), file.clone())); + if file.hidden { + hidden_explicit_keys.insert(key); } } @@ -230,6 +207,7 @@ async fn check_bundle_updates_inner( .map(|a| a.manifest.name.as_str()) .collect(); + let external_ids = external_ids_by_sha1(&archives); let mut updates_available = Vec::new(); let mut removals_available = Vec::new(); @@ -241,7 +219,8 @@ async fn check_bundle_updates_inner( continue; }; - let installed_key = bundle_package_key(bundle_pkg, &all_linked_by_hash, pkg_id); + let installed_key = + bundle_package_key(bundle_pkg, &all_linked_by_hash, &external_ids, pkg_id); let Some(bundle_name) = &bundle_pkg.bundle_name else { continue; @@ -343,8 +322,15 @@ async fn check_bundle_updates_inner( let mut additions_available = Vec::new(); let mut optional_available: Vec<(String, BundleOptionalPackage)> = Vec::new(); let mut planned_addition_keys = all_installed_managed_keys.clone(); - for hash in all_installed_external_hashes { - planned_addition_keys.insert(external_bundle_key(&hash)); + let tracked_package_by_hash: HashMap<&str, &str> = bundle_packages + .iter() + .filter_map(|bp| Some((bp.hash.as_str(), bp.package_id.as_deref()?))) + .collect(); + for hash in &all_installed_external_hashes { + planned_addition_keys.insert(external_bundle_key(hash)); + if let Some(package_id) = tracked_package_by_hash.get(hash.as_str()) { + planned_addition_keys.insert(external_bundle_key(package_id)); + } } for archive in &archives { @@ -358,15 +344,13 @@ async fn check_bundle_updates_inner( .get(&(archive.manifest.name.clone(), file_id.clone())) .copied(); - let file_key = match &file.kind { - BundleFileKind::Managed { - provider, - project_id, - .. - } => managed_bundle_key(*provider, project_id), - BundleFileKind::External(ext) => external_bundle_key(&ext.sha1), - }; - if planned_addition_keys.contains(&file_key) { + let file_key = file.kind.bundle_key(); + let exact_file_installed = matches!( + &file.kind, + BundleFileKind::External { file: ext, .. } + if planned_addition_keys.contains(&external_bundle_key(&ext.sha1)) + ); + if planned_addition_keys.contains(&file_key) || exact_file_installed { continue; } @@ -819,12 +803,17 @@ pub async fn get_bundles_with_update_status( let all_linked_by_hash: HashMap = all_linked.iter().map(|a| (a.hash.clone(), a)).collect(); + let archives = bundles + .archives_for(ctx, &cluster.mc_version, loader) + .await?; + let external_ids = external_ids_by_sha1(&archives); + let mut installed_map: HashMap = HashMap::new(); for bundle_pkg in &bundle_packages { let Some(pkg_id) = &bundle_pkg.package_id else { continue; }; - let key = bundle_package_key(bundle_pkg, &all_linked_by_hash, pkg_id); + let key = bundle_package_key(bundle_pkg, &all_linked_by_hash, &external_ids, pkg_id); installed_map.insert(key, bundle_pkg); } @@ -837,9 +826,6 @@ pub async fn get_bundles_with_update_status( }) .collect(); - let archives = bundles - .archives_for(ctx, &cluster.mc_version, loader) - .await?; let mut results = Vec::new(); for archive in archives { @@ -847,48 +833,25 @@ pub async fn get_bundles_with_update_status( let mut has_updates = false; for file in &archive.manifest.files { - let status = match &file.kind { - BundleFileKind::Managed { - provider, - project_id, - version_id, - .. - } => { - let key = managed_bundle_key(*provider, project_id); - if let Some(installed) = installed_map.get(&key) { - let installed_version = - installed.bundle_version_id.as_deref().unwrap_or(""); - if installed_version == version_id.as_str() { - FileUpdateStatus::UpToDate - } else { - has_updates = true; - FileUpdateStatus::UpdateAvailable { - installed_version_id: installed_version.to_string(), - new_version_id: version_id.clone(), - } - } - } else if matches!( - overrides_map.get(&(archive.manifest.name.clone(), project_id.clone())), - Some(OverrideType::Removed) - ) { - FileUpdateStatus::RemovedByUser - } else { - FileUpdateStatus::NotInstalled - } - } - BundleFileKind::External(ext) => { - let key = external_bundle_key(&ext.sha1); - if installed_map.contains_key(&key) { - FileUpdateStatus::UpToDate - } else if matches!( - overrides_map.get(&(archive.manifest.name.clone(), ext.sha1.clone())), - Some(OverrideType::Removed) - ) { - FileUpdateStatus::RemovedByUser - } else { - FileUpdateStatus::NotInstalled + let new_version_id = file.kind.bundle_version_id(); + let status = if let Some(installed) = installed_map.get(&file.kind.bundle_key()) { + let installed_version = installed.bundle_version_id.as_deref().unwrap_or(""); + if installed_version == new_version_id { + FileUpdateStatus::UpToDate + } else { + has_updates = true; + FileUpdateStatus::UpdateAvailable { + installed_version_id: installed_version.to_string(), + new_version_id, } } + } else if matches!( + overrides_map.get(&(archive.manifest.name.clone(), file.kind.package_id())), + Some(OverrideType::Removed) + ) { + FileUpdateStatus::RemovedByUser + } else { + FileUpdateStatus::NotInstalled }; files.push((file.clone(), status)); } @@ -906,14 +869,21 @@ pub async fn get_bundles_with_update_status( fn bundle_package_key( bundle_pkg: &BundleTrackedArtifactRow, linked: &HashMap, + external_ids: &HashMap, package_id: &str, ) -> String { if package_id == bundle_pkg.hash { - external_bundle_key(&bundle_pkg.hash) - } else if let Some(info) = linked.get(&bundle_pkg.hash) { - managed_bundle_key(info.provider.unwrap_or(ProviderId::Modrinth), package_id) + return external_bundle_key( + external_ids + .get(package_id) + .map_or(package_id, String::as_str), + ); + } + let provider = linked.get(&bundle_pkg.hash).and_then(|info| info.provider); + if provider.is_none() && external_ids.values().any(|id| id == package_id) { + external_bundle_key(package_id) } else { - managed_bundle_key(ProviderId::Modrinth, package_id) + managed_bundle_key(provider.unwrap_or(ProviderId::Modrinth), package_id) } }