diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 24de702..3e7b577 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -681,6 +681,7 @@ name = "just-download-backend" version = "2.0.0" dependencies = [ "axum", + "futures-util", "mime_guess", "regex", "reqwest", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 38e739b..ec06b35 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -11,6 +11,7 @@ tokio = { version = "1.38", features = ["full"] } tokio-util = { version = "0.7", features = ["io"] } tower = { version = "0.4", features = ["util"] } tower-http = { version = "0.5", features = ["cors", "trace", "fs", "compression-full"] } +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" regex = "1.10" @@ -23,7 +24,7 @@ which = "6.0" [profile.release] opt-level = 3 -lto = "fat" +lto = "thin" codegen-units = 1 panic = "abort" strip = true diff --git a/backend/src/config.rs b/backend/src/config.rs index 9550467..a16b102 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -9,6 +9,7 @@ pub struct AppConfig { pub download_dir: PathBuf, pub max_file_age_secs: u64, pub cleanup_interval_secs: u64, + pub max_dir_size_mb: u64, pub semaphore: Arc, pub ytdlp_bin: String, } @@ -23,20 +24,27 @@ impl AppConfig { let download_dir = PathBuf::from( std::env::var("DOWNLOAD_DIR").unwrap_or_else(|_| "downloads".to_string()), ); + // Automatically delete files older than 60 seconds (stream completes within seconds) let max_file_age_secs = std::env::var("MAX_FILE_AGE_SECS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(3600); // 1 hour + .unwrap_or(60); + // Run background cleaner every 30 seconds let cleanup_interval_secs = std::env::var("CLEANUP_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(600); // 10 minutes + .unwrap_or(30); + // Max total directory size: 200 MB quota to prevent disk bloat + let max_dir_size_mb = std::env::var("MAX_DIR_SIZE_MB") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(200); // Limit concurrent downloads on low-memory servers (default 2) let max_concurrent = std::env::var("MAX_CONCURRENT_DOWNLOADS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(3); + .unwrap_or(2); // Find yt-dlp binary let ytdlp_bin = if let Ok(custom) = std::env::var("YTDLP_PATH") { @@ -55,6 +63,7 @@ impl AppConfig { download_dir, max_file_age_secs, cleanup_interval_secs, + max_dir_size_mb, semaphore: Arc::new(Semaphore::new(max_concurrent)), ytdlp_bin, } diff --git a/backend/src/main.rs b/backend/src/main.rs index a098a3d..fa24542 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -39,8 +39,9 @@ async fn main() { let cleanup_dir = config.download_dir.clone(); let max_age = config.max_file_age_secs; let interval = config.cleanup_interval_secs; + let max_mb = config.max_dir_size_mb; tokio::spawn(async move { - run_cleanup_loop(cleanup_dir, max_age, interval).await; + run_cleanup_loop(cleanup_dir, max_age, interval, max_mb).await; }); // Build Axum routers @@ -123,28 +124,63 @@ async fn health_handler() -> Json { })) } -/// Periodic background task that removes files older than max_age_secs -async fn run_cleanup_loop(dir: PathBuf, max_age_secs: u64, interval_secs: u64) { +/// Periodic background task that removes files older than max_age_secs and enforces directory quota +async fn run_cleanup_loop(dir: PathBuf, max_age_secs: u64, interval_secs: u64, max_dir_size_mb: u64) { let mut ticker = tokio::time::interval(Duration::from_secs(interval_secs)); + let max_bytes = max_dir_size_mb * 1024 * 1024; + loop { ticker.tick().await; + let mut files: Vec<(PathBuf, u64, std::time::SystemTime)> = Vec::new(); + let mut total_size: u64 = 0; + let now = std::time::SystemTime::now(); + if let Ok(mut entries) = tokio::fs::read_dir(&dir).await { - let now = std::time::SystemTime::now(); while let Ok(Some(entry)) = entries.next_entry().await { if let Ok(meta) = entry.metadata().await { if meta.is_file() { - if let Ok(modified) = meta.modified() { - if let Ok(elapsed) = now.duration_since(modified) { - if elapsed.as_secs() > max_age_secs { - let _ = tokio::fs::remove_file(entry.path()).await; - info!("Cleaned up expired file: {:?}", entry.path()); - } - } + let path = entry.path(); + let size = meta.len(); + let modified = meta.modified().unwrap_or(now); + let elapsed = now.duration_since(modified).unwrap_or(Duration::ZERO); + + let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + let is_partial = file_name.ends_with(".part") + || file_name.ends_with(".ytdl") + || file_name.ends_with(".tmp") + || file_name.ends_with(".temp"); + + // If file is older than max_age or is a stale partial download older than 30s + if elapsed.as_secs() > max_age_secs || (is_partial && elapsed.as_secs() > 30) { + let _ = tokio::fs::remove_file(&path).await; + info!("Auto-cleaner removed expired file: {:?} ({} KB)", path, size / 1024); + } else { + total_size += size; + files.push((path, size, modified)); } } } } } + + // If total downloads folder size exceeds quota, delete oldest files until under 50% quota + if total_size > max_bytes { + info!( + "Downloads directory exceeded limit ({} MB / {} MB). Purging oldest files...", + total_size / (1024 * 1024), + max_dir_size_mb + ); + files.sort_by_key(|f| f.2); + let target_size = max_bytes / 2; + for (path, size, _) in files { + if total_size <= target_size { + break; + } + let _ = tokio::fs::remove_file(&path).await; + info!("Purged file to free space: {:?}", path); + total_size = total_size.saturating_sub(size); + } + } } } diff --git a/backend/src/routes/mod.rs b/backend/src/routes/mod.rs index 382a5dc..dc0e004 100644 --- a/backend/src/routes/mod.rs +++ b/backend/src/routes/mod.rs @@ -6,19 +6,45 @@ use axum::body::Body; use axum::http::header::{CONTENT_DISPOSITION, CONTENT_LENGTH, CONTENT_TYPE}; use axum::http::HeaderMap; use axum::response::{IntoResponse, Response}; +use futures_util::StreamExt; use std::path::PathBuf; +use std::sync::Arc; use tokio_util::io::ReaderStream; +use tracing::info; use crate::error::AppError; -/// Streams a file response with Content-Disposition and schedules background cleanup +struct FileCleaner { + path: PathBuf, +} + +impl Drop for FileCleaner { + fn drop(&mut self) { + let path = self.path.clone(); + tokio::spawn(async move { + let _ = tokio::fs::remove_file(&path).await; + info!("Auto-cleaner unlinked served file immediately: {:?}", path); + }); + } +} + +/// Streams a file response with Content-Disposition and guarantees immediate cleanup pub async fn stream_file_response(path: PathBuf, filename: String) -> Result { let file = tokio::fs::File::open(&path).await.map_err(|e| { AppError::NotFound(format!("Failed to open generated file: {}", e)) })?; let file_len = file.metadata().await.ok().map(|m| m.len()); - let stream = ReaderStream::new(file); + let cleaner = Arc::new(FileCleaner { + path: path.clone(), + }); + + let reader_stream = ReaderStream::new(file); + // Bind the cleaner Arc to the stream so the file is unlinked immediately when stream ends or client disconnects + let stream = reader_stream.map(move |chunk| { + let _g = &cleaner; + chunk + }); let body = Body::from_stream(stream); let mime = mime_guess::from_path(&filename) @@ -36,10 +62,10 @@ pub async fn stream_file_response(path: PathBuf, filename: String) -> Result