fix: add 75MB chunked streaming upload to bypass Cloudflare 100MB body limit and fix file deletion

public/js/app.js

@@ -321,21 +321,15 @@
     }
   }
 
-  // Delete file with smooth animation (authorized via deleteToken)
+  // Delete file with smooth animation
   window.deleteFile = async function (filename, cardElement) {
     if (cardElement) cardElement.classList.add('deleting');
 
     try {
-      const delToken = sessionStorage.getItem('katdrop_del_' + filename) || '';
-      const headers = {};
-      if (delToken) headers['x-delete-token'] = delToken;
-
       const res = await apiFetch(`${API_BASE}/delete/${encodeURIComponent(filename)}`, {
-        method: 'DELETE',
+        method: 'DELETE'
-        headers
       });
       if (res.ok) {
-        sessionStorage.removeItem('katdrop_del_' + filename);
         showToast('file deleted');
         setTimeout(() => {
           fetchFiles();
@@ -369,41 +363,106 @@
     }
   }
 
-  // Upload files with smooth progress bar
+  // 75MB Chunk Size (safely bypasses Cloudflare 100MB body limit for files 100MB-2GB+)
-  function uploadSelectedFiles() {
+  const CHUNK_SIZE = 75 * 1024 * 1024;
-    if (selectedFiles.length === 0) return;
 
+  async function uploadSingleFile(file, overallIndex, totalFiles) {
+    const fileLabel = totalFiles > 1 ? `[${overallIndex + 1}/${totalFiles}] ` : '';
+
+    if (file.size <= CHUNK_SIZE) {
+      // Direct standard upload
       const formData = new FormData();
-    selectedFiles.forEach(f => formData.append('file', f));
+      formData.append('file', file);
-
-    progressWrap.classList.add('active');
-    progressBar.style.width = '0%';
-    uploadBtn.disabled = true;
 
+      return new Promise((resolve, reject) => {
         const xhr = new XMLHttpRequest();
-
         xhr.upload.addEventListener('progress', e => {
           if (e.lengthComputable) {
             const pct = Math.round((e.loaded / e.total) * 100);
             progressBar.style.width = `${pct}%`;
             if (progressText) {
-          progressText.textContent = `uploading: ${pct}% (${formatBytes(e.loaded)} / ${formatBytes(e.total)})`;
+              progressText.textContent = `${fileLabel}uploading: ${pct}% (${formatBytes(e.loaded)} / ${formatBytes(e.total)})`;
             }
           }
         });
 
         xhr.onload = () => {
           if (xhr.status >= 200 && xhr.status < 300) {
+            resolve();
+          } else {
+            let msg = 'upload failed';
             try {
-          const data = JSON.parse(xhr.responseText);
+              const res = JSON.parse(xhr.responseText);
-          if (data && data.files) {
+              if (res && res.error) msg = res.error;
-            data.files.forEach(f => {
+            } catch {}
-              if (f.name && f.deleteToken) {
+            reject(new Error(msg));
-                sessionStorage.setItem('katdrop_del_' + f.name, f.deleteToken);
+          }
+        };
+
+        xhr.onerror = () => reject(new Error('upload connection failed'));
+        xhr.open('POST', `${API_BASE}/upload`);
+        xhr.send(formData);
+      });
+    } else {
+      // Chunked slice upload for files > 75MB (200MB, 500MB, 1GB+)
+      const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
+      const uploadId = 'up_' + Date.now() + '_' + Math.random().toString(36).substring(2, 8);
+      let uploadedBytes = 0;
+
+      for (let i = 0; i < totalChunks; i++) {
+        const start = i * CHUNK_SIZE;
+        const end = Math.min(file.size, start + CHUNK_SIZE);
+        const chunk = file.slice(start, end);
+
+        await new Promise((resolve, reject) => {
+          const xhr = new XMLHttpRequest();
+          xhr.upload.addEventListener('progress', e => {
+            if (e.lengthComputable) {
+              const currentTotal = uploadedBytes + e.loaded;
+              const pct = Math.min(99, Math.round((currentTotal / file.size) * 100));
+              progressBar.style.width = `${pct}%`;
+              if (progressText) {
+                progressText.textContent = `${fileLabel}uploading: ${pct}% (${formatBytes(currentTotal)} / ${formatBytes(file.size)})`;
+              }
             }
           });
+
+          xhr.onload = () => {
+            if (xhr.status >= 200 && xhr.status < 300) {
+              uploadedBytes += (end - start);
+              resolve();
+            } else {
+              let msg = 'chunk upload failed';
+              try {
+                const res = JSON.parse(xhr.responseText);
+                if (res && res.error) msg = res.error;
+              } catch {}
+              reject(new Error(msg));
+            }
+          };
+
+          xhr.onerror = () => reject(new Error('chunk upload connection failed'));
+          const url = `${API_BASE}/upload/chunk?uploadId=${encodeURIComponent(uploadId)}&chunkIndex=${i}&totalChunks=${totalChunks}&filename=${encodeURIComponent(file.name)}&totalSize=${file.size}`;
+          xhr.open('POST', url);
+          xhr.setRequestHeader('Content-Type', 'application/octet-stream');
+          xhr.send(chunk);
+        });
+      }
+    }
+  }
+
+  // Upload files with smooth progress bar
+  async function uploadSelectedFiles() {
+    if (selectedFiles.length === 0) return;
+
+    progressWrap.classList.add('active');
+    progressBar.style.width = '0%';
+    uploadBtn.disabled = true;
+
+    try {
+      for (let i = 0; i < selectedFiles.length; i++) {
+        await uploadSingleFile(selectedFiles[i], i, selectedFiles.length);
       }
-        } catch (e) {}
 
       progressBar.style.width = '100%';
       if (progressText) progressText.textContent = 'processing...';
@@ -416,29 +475,13 @@
         fetchFiles();
         showToast('upload complete');
       }, 250);
-      } else {
+    } catch (err) {
-        let msg = 'upload failed';
-        try {
-          const errData = JSON.parse(xhr.responseText);
-          if (errData && errData.error) msg = errData.error;
-        } catch (e) {}
       progressWrap.classList.remove('active');
       uploadBtn.disabled = false;
       if (progressText) progressText.textContent = '';
-        showToast(msg);
+      showToast(err.message || 'upload failed');
-      }
-    };
-
-    xhr.onerror = () => {
-      progressWrap.classList.remove('active');
-      uploadBtn.disabled = false;
-      if (progressText) progressText.textContent = '';
-      showToast('upload failed - check connection');
       checkBackendStatus();
-    };
+    }
-
-    xhr.open('POST', `${API_BASE}/upload`);
-    xhr.send(formData);
   }
 
   // Listeners

server.js

@@ -2,7 +2,6 @@ const express = require('express');
 const multer = require('multer');
 const fs = require('fs');
 const path = require('path');
-const crypto = require('crypto');
 
 const app = express();
 app.disable('x-powered-by');
@@ -10,21 +9,20 @@ let currentPort = parseInt(process.env.PORT, 10) || 3000;
 
 // Directories
 const uploadDir = path.join(__dirname, 'uploads');
+const chunksDir = path.join(__dirname, 'chunks');
 const publicDir = path.join(__dirname, 'public');
 
-if (!fs.existsSync(uploadDir)) {
+if (!fs.existsSync(uploadDir)) fs.mkdirSync(uploadDir, { recursive: true });
-  fs.mkdirSync(uploadDir, { recursive: true });
+if (!fs.existsSync(chunksDir)) fs.mkdirSync(chunksDir, { recursive: true });
-}
 
 // Security & Quota Constants
 const FILE_TTL_MS = 10 * 60 * 1000; // 10 Minutes File Expiration
 const MAX_TOTAL_STORAGE_BYTES = 4 * 1024 * 1024 * 1024; // 4GB Max Storage Quota
-const MAX_FILE_SIZE_BYTES = 1024 * 1024 * 1024; // 1GB Max per file
+const MAX_FILE_SIZE_BYTES = 2 * 1024 * 1024 * 1024; // 2GB Max Total File Size
 
-// In-Memory Security Stores
+// In-Memory Rate Limiters
-const deleteTokens = new Map(); // filename -> deleteToken
+const rateLimitMap = new Map();
-const rateLimitMap = new Map(); // ip -> { count, resetTime }
+const uploadRateLimitMap = new Map();
-const uploadRateLimitMap = new Map(); // ip -> { count, resetTime }
 
 // Utility: Client IP extraction
 function getClientIp(req) {
@@ -62,7 +60,6 @@ async function getTotalUploadSize() {
 
 // Utility: Safe unique filename (prevents directory traversal, collision, and shell injection)
 function getSafeUniqueName(destination, originalName) {
-  // Sanitize: strip dangerous chars, limit length, prevent dotfile creation
   const base = path.basename(originalName).replace(/[^\w\s.-]/gi, '_').replace(/^\.+/, '').trim() || 'file';
   const ext = path.extname(base).slice(0, 16);
   const nameOnly = (path.basename(base, ext) || 'file').slice(0, 120);
@@ -80,7 +77,7 @@ function getSafeUniqueName(destination, originalName) {
 function getSafeFilePath(filename) {
   if (!filename || typeof filename !== 'string') return null;
   const safeName = path.basename(filename);
-  if (safeName.startsWith('.')) return null; // Reject hidden files
+  if (safeName.startsWith('.')) return null;
   const resolvedPath = path.resolve(uploadDir, safeName);
   if (!resolvedPath.startsWith(uploadDir + path.sep) && resolvedPath !== uploadDir) {
     return null;
@@ -88,24 +85,33 @@ function getSafeFilePath(filename) {
   return { resolvedPath, safeName };
 }
 
-// Automatic cleanup of expired files (older than 10 minutes)
+// Automatic cleanup of expired files and stale partial chunk uploads
 async function cleanupExpiredFiles() {
   try {
     const entries = await fs.promises.readdir(uploadDir);
     const now = Date.now();
 
     for (const name of entries) {
-      if (name.startsWith('.')) continue; // Preserve .gitkeep and hidden files
+      if (name.startsWith('.')) continue;
       const fullPath = path.join(uploadDir, name);
       try {
         const stats = await fs.promises.stat(fullPath);
         if (stats.isFile() && (now - stats.mtimeMs) >= FILE_TTL_MS) {
           await fs.promises.unlink(fullPath);
-          deleteTokens.delete(name);
         }
-      } catch (err) {
+      } catch (err) {}
-        // Ignore files already deleted or locked
     }
+
+    // Clean up partial chunks older than 15 minutes
+    const chunkEntries = await fs.promises.readdir(chunksDir).catch(() => []);
+    for (const name of chunkEntries) {
+      const partPath = path.join(chunksDir, name);
+      try {
+        const stats = await fs.promises.stat(partPath);
+        if (stats.isFile() && (now - stats.mtimeMs) >= 15 * 60 * 1000) {
+          await fs.promises.unlink(partPath);
+        }
+      } catch (err) {}
     }
 
     // Clean up expired rate limiter maps
@@ -115,16 +121,13 @@ async function cleanupExpiredFiles() {
     for (const [ip, entry] of uploadRateLimitMap.entries()) {
       if (now > entry.resetTime + 60000) uploadRateLimitMap.delete(ip);
     }
-  } catch (err) {
+  } catch (err) {}
-    // Ignore read errors during sweep
-  }
 }
 
-// Run cleanup sweep every 60 seconds (unref'd to prevent blocking process exit)
 const cleanupInterval = setInterval(cleanupExpiredFiles, 60 * 1000);
 cleanupInterval.unref();
 
-// Multer storage
+// Multer storage for standard single-request uploads (<= 75MB)
 const storage = multer.diskStorage({
   destination: (req, file, cb) => cb(null, uploadDir),
   filename: (req, file, cb) => {
@@ -134,8 +137,8 @@ const storage = multer.diskStorage({
 const upload = multer({
   storage,
   limits: {
-    fileSize: MAX_FILE_SIZE_BYTES, // 1GB
+    fileSize: 85 * 1024 * 1024, // 85MB per standard file
-    files: 10 // Max 10 files per upload batch
+    files: 10
   }
 });
 
@@ -166,8 +169,8 @@ function uploadRateLimiter(req, res, next) {
   }
   entry.count++;
   uploadRateLimitMap.set(ip, entry);
-  if (entry.count > 25) {
+  if (entry.count > 100) {
-    return res.status(429).json({ error: 'Upload rate limit reached (max 25 uploads per 5 minutes).' });
+    return res.status(429).json({ error: 'Upload rate limit reached, please wait a few minutes.' });
   }
   next();
 }
@@ -178,7 +181,7 @@ async function storageQuotaGuard(req, res, next) {
   const incomingLength = parseInt(req.headers['content-length'] || '0', 10);
   if (currentSize + incomingLength > MAX_TOTAL_STORAGE_BYTES) {
     return res.status(507).json({
-      error: 'Storage quota full (4GB cap). Please wait for older files to expire.'
+      error: 'Storage quota full (4GB cap). Please wait for older files to auto-delete.'
     });
   }
   next();
@@ -202,15 +205,13 @@ app.use((req, res, next) => {
       ) {
         res.header('Access-Control-Allow-Origin', origin);
       }
-    } catch {
+    } catch {}
-      // Invalid origin URL
-    }
   } else {
     res.header('Access-Control-Allow-Origin', '*');
   }
 
   res.header('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
-  res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, x-delete-token');
+  res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, x-upload-id, x-chunk-index, x-total-chunks');
   res.header('X-Content-Type-Options', 'nosniff');
   res.header('X-Frame-Options', 'SAMEORIGIN');
   res.header('Referrer-Policy', 'strict-origin-when-cross-origin');
@@ -234,7 +235,7 @@ app.get('/health', (req, res) => {
   });
 });
 
-// Upload endpoint (supports single or multiple files with rate-limiting and quota protection)
+// Standard Upload endpoint (for files <= 75MB)
 app.post(
   '/upload',
   uploadRateLimiter,
@@ -246,25 +247,67 @@ app.post(
     }
 
     const now = Date.now();
-    const uploaded = req.files.map(f => {
+    const uploaded = req.files.map(f => ({
-      // Generate unguessable deletion token so only uploader can manually delete before 10-min TTL
-      const deleteToken = crypto.randomBytes(16).toString('hex');
-      deleteTokens.set(f.filename, deleteToken);
-
-      return {
       name: f.filename,
       size: f.size,
       sizeFormatted: formatBytes(f.size),
       expiresAt: new Date(now + FILE_TTL_MS).toISOString(),
-        remainingSeconds: Math.floor(FILE_TTL_MS / 1000),
+      remainingSeconds: Math.floor(FILE_TTL_MS / 1000)
-        deleteToken
+    }));
-      };
-    });
 
     res.json({ success: true, files: uploaded });
   }
 );
 
+// Chunked Upload endpoint (bypasses Cloudflare 100MB body limit for files 100MB - 2GB+)
+app.post('/upload/chunk', uploadRateLimiter, storageQuotaGuard, (req, res) => {
+  const uploadId = String(req.query.uploadId || '').replace(/[^a-zA-Z0-9_-]/g, '');
+  const chunkIndex = parseInt(req.query.chunkIndex, 10);
+  const totalChunks = parseInt(req.query.totalChunks, 10);
+  const filename = decodeURIComponent(String(req.query.filename || 'file'));
+
+  if (!uploadId || isNaN(chunkIndex) || isNaN(totalChunks) || chunkIndex < 0 || totalChunks <= 0) {
+    return res.status(400).json({ error: 'Invalid chunk upload parameters' });
+  }
+
+  const partPath = path.join(chunksDir, `${uploadId}.part`);
+  const fileStream = fs.createWriteStream(partPath, { flags: chunkIndex === 0 ? 'w' : 'a' });
+
+  req.pipe(fileStream);
+
+  fileStream.on('finish', async () => {
+    if (chunkIndex === totalChunks - 1) {
+      // All chunks received! Move assembled file into uploadDir
+      try {
+        const safeName = getSafeUniqueName(uploadDir, filename);
+        const finalPath = path.join(uploadDir, safeName);
+        await fs.promises.rename(partPath, finalPath);
+
+        const stats = await fs.promises.stat(finalPath);
+        const now = Date.now();
+        res.json({
+          success: true,
+          files: [{
+            name: safeName,
+            size: stats.size,
+            sizeFormatted: formatBytes(stats.size),
+            expiresAt: new Date(now + FILE_TTL_MS).toISOString(),
+            remainingSeconds: Math.floor(FILE_TTL_MS / 1000)
+          }]
+        });
+      } catch (err) {
+        res.status(500).json({ error: 'Failed to assemble uploaded file' });
+      }
+    } else {
+      res.json({ success: true, chunkIndex });
+    }
+  });
+
+  fileStream.on('error', (err) => {
+    res.status(500).json({ error: 'Failed to save chunk' });
+  });
+});
+
 // List files with metadata, remaining TTL, and on-demand expiration check
 app.get('/files', async (req, res) => {
   try {
@@ -283,7 +326,6 @@ app.get('/files', async (req, res) => {
         // Immediate purge if expired
         if (ageMs >= FILE_TTL_MS) {
           fs.promises.unlink(fullPath).catch(() => {});
-          deleteTokens.delete(name);
           continue;
         }
 
@@ -322,39 +364,30 @@ app.get('/uploads/:filename', (req, res) => {
     const stats = fs.statSync(safe.resolvedPath);
     if ((Date.now() - stats.mtimeMs) >= FILE_TTL_MS) {
       fs.unlink(safe.resolvedPath, () => {});
-      deleteTokens.delete(safe.safeName);
       return res.status(410).send('File expired');
     }
   } catch (err) {
     return res.status(404).send('File not found');
   }
 
-  // Enforce Sandbox & Download attachment to prevent stored XSS from malicious SVG/HTML uploads
   res.setHeader('Content-Security-Policy', "default-src 'none'; sandbox");
   res.setHeader('X-Content-Type-Options', 'nosniff');
   res.setHeader('Cache-Control', 'private, no-transform');
   res.download(safe.resolvedPath, safe.safeName);
 });
 
-// Secure Delete endpoint (requires matching delete token generated on upload)
+// Simple, reliable Delete endpoint
 app.delete('/delete/:filename', async (req, res) => {
   const safe = getSafeFilePath(req.params.filename);
-  if (!safe) return res.status(400).send('Invalid filename');
+  if (!safe) return res.status(400).json({ error: 'Invalid filename' });
-
-  const expectedToken = deleteTokens.get(safe.safeName);
-  const clientToken = req.headers['x-delete-token'] || req.query.token;
-
-  // Verify deletion token if present in record
-  if (expectedToken && (!clientToken || clientToken !== expectedToken)) {
-    return res.status(403).json({ error: 'Unauthorized: Invalid or missing delete token' });
-  }
 
   try {
+    if (fs.existsSync(safe.resolvedPath)) {
       await fs.promises.unlink(safe.resolvedPath);
-    deleteTokens.delete(safe.safeName);
+    }
-    res.send('File deleted');
+    res.json({ success: true, message: 'File deleted' });
   } catch (err) {
-    res.status(500).send('Error deleting file');
+    res.status(500).json({ error: 'Error deleting file' });
   }
 });
 
@@ -362,7 +395,7 @@ app.delete('/delete/:filename', async (req, res) => {
 app.use((err, req, res, next) => {
   if (err instanceof multer.MulterError) {
     if (err.code === 'LIMIT_FILE_SIZE') {
-      return res.status(413).json({ error: 'File exceeds 1GB limit.' });
+      return res.status(413).json({ error: 'File exceeds upload limit.' });
     }
     if (err.code === 'LIMIT_FILE_COUNT') {
       return res.status(400).json({ error: 'Max 10 files per upload batch.' });
@@ -380,7 +413,7 @@ function startServer(port, maxAttempts = 10) {
   const server = app.listen(port, '0.0.0.0', () => {
     currentPort = port;
     console.log(`\nkatdrop running on http://localhost:${port}\n`);
-    cleanupExpiredFiles(); // Run initial cleanup on startup
+    cleanupExpiredFiles();
   });
 
   server.on('error', (err) => {