fix(transfer): 8MB chunk slicing with exponential backoff retry and isolated part assembly to eliminate upload dropouts

This commit is contained in:
katR 2026-09-12 18:46:29 +05:30
parent 2a642bbfa4
commit 49c998852b
2 changed files with 90 additions and 44 deletions

View file

@ -60,29 +60,33 @@
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
} }
// Backend Status Check // Backend Status Check (with resilience against momentary network congestion)
let consecutiveHealthFailures = 0;
async function checkBackendStatus() { async function checkBackendStatus() {
if (!backendStatus || !statusText) return; if (!backendStatus || !statusText) return;
backendStatus.className = 'backend-status checking';
statusText.textContent = 'checking...';
try { try {
const controller = new AbortController(); const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 4000); const timeoutId = setTimeout(() => controller.abort(), 6000);
const res = await apiFetch(`${API_BASE}/health`, { signal: controller.signal }); const res = await apiFetch(`${API_BASE}/health`, { signal: controller.signal });
clearTimeout(timeoutId); clearTimeout(timeoutId);
if (res.ok) { if (res.ok) {
consecutiveHealthFailures = 0;
backendStatus.className = 'backend-status online'; backendStatus.className = 'backend-status online';
statusText.textContent = 'online'; statusText.textContent = 'online';
} else { } else {
throw new Error('Non-200'); throw new Error('Non-200');
} }
} catch (err) { } catch (err) {
consecutiveHealthFailures++;
// Require at least 2 consecutive failures so high-traffic uploads don't trigger false offline badges
if (consecutiveHealthFailures >= 2) {
backendStatus.className = 'backend-status offline'; backendStatus.className = 'backend-status offline';
statusText.textContent = 'offline'; statusText.textContent = 'offline';
} }
} }
}
// Autonomous & Interactive Particle Background // Autonomous & Interactive Particle Background
function initInteractiveParticles() { function initInteractiveParticles() {
@ -389,8 +393,8 @@
} }
} }
// 16MB Chunk Size (safely bypasses Cloudflare 100MB body limit & connection timeouts on Wi-Fi) // 8MB Chunk Size (optimal for mobile Wi-Fi, low latency, avoids proxy timeouts)
const CHUNK_SIZE = 16 * 1024 * 1024; const CHUNK_SIZE = 8 * 1024 * 1024;
async function uploadSingleFile(file, overallIndex, totalFiles) { async function uploadSingleFile(file, overallIndex, totalFiles) {
const fileLabel = totalFiles > 1 ? `[${overallIndex + 1}/${totalFiles}] ` : ''; const fileLabel = totalFiles > 1 ? `[${overallIndex + 1}/${totalFiles}] ` : '';
@ -430,7 +434,7 @@
xhr.send(formData); xhr.send(formData);
}); });
} else { } else {
// Chunked slice upload for files > 75MB (200MB, 500MB, 1GB+) // Chunked slice upload for files > 8MB (200MB, 500MB, 1GB+)
const totalChunks = Math.ceil(file.size / CHUNK_SIZE); const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
const uploadId = 'up_' + Date.now() + '_' + Math.random().toString(36).substring(2, 8); const uploadId = 'up_' + Date.now() + '_' + Math.random().toString(36).substring(2, 8);
let uploadedBytes = 0; let uploadedBytes = 0;
@ -439,9 +443,18 @@
const start = i * CHUNK_SIZE; const start = i * CHUNK_SIZE;
const end = Math.min(file.size, start + CHUNK_SIZE); const end = Math.min(file.size, start + CHUNK_SIZE);
const chunk = file.slice(start, end); const chunk = file.slice(start, end);
const url = `${API_BASE}/upload/chunk?uploadId=${encodeURIComponent(uploadId)}&chunkIndex=${i}&totalChunks=${totalChunks}&filename=${encodeURIComponent(file.name)}&totalSize=${file.size}`;
let attempt = 0;
const maxRetries = 4;
let chunkSuccess = false;
while (attempt < maxRetries && !chunkSuccess) {
try {
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest(); const xhr = new XMLHttpRequest();
xhr.timeout = 60000; // 60s timeout per 8MB chunk
xhr.upload.addEventListener('progress', e => { xhr.upload.addEventListener('progress', e => {
if (e.lengthComputable) { if (e.lengthComputable) {
const currentTotal = uploadedBytes + e.loaded; const currentTotal = uploadedBytes + e.loaded;
@ -456,6 +469,7 @@
xhr.onload = () => { xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) { if (xhr.status >= 200 && xhr.status < 300) {
uploadedBytes += (end - start); uploadedBytes += (end - start);
chunkSuccess = true;
resolve(); resolve();
} else { } else {
let msg = 'chunk upload failed'; let msg = 'chunk upload failed';
@ -467,12 +481,24 @@
} }
}; };
xhr.onerror = () => reject(new Error('chunk upload connection failed')); xhr.onerror = () => reject(new Error('connection drop'));
const url = `${API_BASE}/upload/chunk?uploadId=${encodeURIComponent(uploadId)}&chunkIndex=${i}&totalChunks=${totalChunks}&filename=${encodeURIComponent(file.name)}&totalSize=${file.size}`; xhr.ontimeout = () => reject(new Error('timeout'));
xhr.open('POST', url); xhr.open('POST', url);
xhr.setRequestHeader('Content-Type', 'application/octet-stream'); xhr.setRequestHeader('Content-Type', 'application/octet-stream');
xhr.send(chunk); xhr.send(chunk);
}); });
} catch (err) {
attempt++;
if (attempt >= maxRetries) {
throw new Error(`Upload interrupted on chunk ${i + 1}/${totalChunks}. Check connection.`);
}
if (progressText) {
progressText.textContent = `${fileLabel}reconnecting chunk ${i + 1}... (retry ${attempt}/${maxRetries})`;
}
await new Promise(r => setTimeout(r, attempt * 1200));
}
}
} }
} }
} }

View file

@ -270,18 +270,38 @@ app.post('/upload/chunk', uploadRateLimiter, storageQuotaGuard, (req, res) => {
return res.status(400).json({ error: 'Invalid chunk upload parameters' }); return res.status(400).json({ error: 'Invalid chunk upload parameters' });
} }
const partPath = path.join(chunksDir, `${uploadId}.part`); // Write each chunk to an isolated part file to support safe, idempotent retries
const fileStream = fs.createWriteStream(partPath, { flags: chunkIndex === 0 ? 'w' : 'a' }); const partPath = path.join(chunksDir, `${uploadId}_${chunkIndex}.part`);
const fileStream = fs.createWriteStream(partPath, { flags: 'w' });
req.pipe(fileStream); req.pipe(fileStream);
fileStream.on('finish', async () => { fileStream.on('finish', async () => {
if (chunkIndex === totalChunks - 1) { if (chunkIndex === totalChunks - 1) {
// All chunks received! Move assembled file into uploadDir // Final chunk reached - verify and assemble all parts sequentially
try { try {
const safeName = getSafeUniqueName(uploadDir, filename); const safeName = getSafeUniqueName(uploadDir, filename);
const finalPath = path.join(uploadDir, safeName); const finalPath = path.join(uploadDir, safeName);
await fs.promises.rename(partPath, finalPath); const finalStream = fs.createWriteStream(finalPath);
for (let i = 0; i < totalChunks; i++) {
const p = path.join(chunksDir, `${uploadId}_${i}.part`);
await new Promise((resolve, reject) => {
const r = fs.createReadStream(p);
r.on('error', reject);
r.on('end', () => {
fs.promises.unlink(p).catch(() => {});
resolve();
});
r.pipe(finalStream, { end: false });
});
}
finalStream.end();
await new Promise((resolve, reject) => {
finalStream.on('finish', resolve);
finalStream.on('error', reject);
});
const stats = await fs.promises.stat(finalPath); const stats = await fs.promises.stat(finalPath);
const now = Date.now(); const now = Date.now();