fix(transfer): 8MB chunk slicing with exponential backoff retry and isolated part assembly to eliminate upload dropouts
This commit is contained in:
parent
2a642bbfa4
commit
49c998852b
2 changed files with 90 additions and 44 deletions
|
|
@ -60,29 +60,33 @@
|
||||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
|
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backend Status Check
|
// Backend Status Check (with resilience against momentary network congestion)
|
||||||
|
let consecutiveHealthFailures = 0;
|
||||||
async function checkBackendStatus() {
|
async function checkBackendStatus() {
|
||||||
if (!backendStatus || !statusText) return;
|
if (!backendStatus || !statusText) return;
|
||||||
backendStatus.className = 'backend-status checking';
|
|
||||||
statusText.textContent = 'checking...';
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeoutId = setTimeout(() => controller.abort(), 4000);
|
const timeoutId = setTimeout(() => controller.abort(), 6000);
|
||||||
const res = await apiFetch(`${API_BASE}/health`, { signal: controller.signal });
|
const res = await apiFetch(`${API_BASE}/health`, { signal: controller.signal });
|
||||||
clearTimeout(timeoutId);
|
clearTimeout(timeoutId);
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
|
consecutiveHealthFailures = 0;
|
||||||
backendStatus.className = 'backend-status online';
|
backendStatus.className = 'backend-status online';
|
||||||
statusText.textContent = 'online';
|
statusText.textContent = 'online';
|
||||||
} else {
|
} else {
|
||||||
throw new Error('Non-200');
|
throw new Error('Non-200');
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
consecutiveHealthFailures++;
|
||||||
|
// Require at least 2 consecutive failures so high-traffic uploads don't trigger false offline badges
|
||||||
|
if (consecutiveHealthFailures >= 2) {
|
||||||
backendStatus.className = 'backend-status offline';
|
backendStatus.className = 'backend-status offline';
|
||||||
statusText.textContent = 'offline';
|
statusText.textContent = 'offline';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Autonomous & Interactive Particle Background
|
// Autonomous & Interactive Particle Background
|
||||||
function initInteractiveParticles() {
|
function initInteractiveParticles() {
|
||||||
|
|
@ -389,8 +393,8 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 16MB Chunk Size (safely bypasses Cloudflare 100MB body limit & connection timeouts on Wi-Fi)
|
// 8MB Chunk Size (optimal for mobile Wi-Fi, low latency, avoids proxy timeouts)
|
||||||
const CHUNK_SIZE = 16 * 1024 * 1024;
|
const CHUNK_SIZE = 8 * 1024 * 1024;
|
||||||
|
|
||||||
async function uploadSingleFile(file, overallIndex, totalFiles) {
|
async function uploadSingleFile(file, overallIndex, totalFiles) {
|
||||||
const fileLabel = totalFiles > 1 ? `[${overallIndex + 1}/${totalFiles}] ` : '';
|
const fileLabel = totalFiles > 1 ? `[${overallIndex + 1}/${totalFiles}] ` : '';
|
||||||
|
|
@ -430,7 +434,7 @@
|
||||||
xhr.send(formData);
|
xhr.send(formData);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
// Chunked slice upload for files > 75MB (200MB, 500MB, 1GB+)
|
// Chunked slice upload for files > 8MB (200MB, 500MB, 1GB+)
|
||||||
const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
|
const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
|
||||||
const uploadId = 'up_' + Date.now() + '_' + Math.random().toString(36).substring(2, 8);
|
const uploadId = 'up_' + Date.now() + '_' + Math.random().toString(36).substring(2, 8);
|
||||||
let uploadedBytes = 0;
|
let uploadedBytes = 0;
|
||||||
|
|
@ -439,9 +443,18 @@
|
||||||
const start = i * CHUNK_SIZE;
|
const start = i * CHUNK_SIZE;
|
||||||
const end = Math.min(file.size, start + CHUNK_SIZE);
|
const end = Math.min(file.size, start + CHUNK_SIZE);
|
||||||
const chunk = file.slice(start, end);
|
const chunk = file.slice(start, end);
|
||||||
|
const url = `${API_BASE}/upload/chunk?uploadId=${encodeURIComponent(uploadId)}&chunkIndex=${i}&totalChunks=${totalChunks}&filename=${encodeURIComponent(file.name)}&totalSize=${file.size}`;
|
||||||
|
|
||||||
|
let attempt = 0;
|
||||||
|
const maxRetries = 4;
|
||||||
|
let chunkSuccess = false;
|
||||||
|
|
||||||
|
while (attempt < maxRetries && !chunkSuccess) {
|
||||||
|
try {
|
||||||
await new Promise((resolve, reject) => {
|
await new Promise((resolve, reject) => {
|
||||||
const xhr = new XMLHttpRequest();
|
const xhr = new XMLHttpRequest();
|
||||||
|
xhr.timeout = 60000; // 60s timeout per 8MB chunk
|
||||||
|
|
||||||
xhr.upload.addEventListener('progress', e => {
|
xhr.upload.addEventListener('progress', e => {
|
||||||
if (e.lengthComputable) {
|
if (e.lengthComputable) {
|
||||||
const currentTotal = uploadedBytes + e.loaded;
|
const currentTotal = uploadedBytes + e.loaded;
|
||||||
|
|
@ -456,6 +469,7 @@
|
||||||
xhr.onload = () => {
|
xhr.onload = () => {
|
||||||
if (xhr.status >= 200 && xhr.status < 300) {
|
if (xhr.status >= 200 && xhr.status < 300) {
|
||||||
uploadedBytes += (end - start);
|
uploadedBytes += (end - start);
|
||||||
|
chunkSuccess = true;
|
||||||
resolve();
|
resolve();
|
||||||
} else {
|
} else {
|
||||||
let msg = 'chunk upload failed';
|
let msg = 'chunk upload failed';
|
||||||
|
|
@ -467,12 +481,24 @@
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
xhr.onerror = () => reject(new Error('chunk upload connection failed'));
|
xhr.onerror = () => reject(new Error('connection drop'));
|
||||||
const url = `${API_BASE}/upload/chunk?uploadId=${encodeURIComponent(uploadId)}&chunkIndex=${i}&totalChunks=${totalChunks}&filename=${encodeURIComponent(file.name)}&totalSize=${file.size}`;
|
xhr.ontimeout = () => reject(new Error('timeout'));
|
||||||
|
|
||||||
xhr.open('POST', url);
|
xhr.open('POST', url);
|
||||||
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
|
xhr.setRequestHeader('Content-Type', 'application/octet-stream');
|
||||||
xhr.send(chunk);
|
xhr.send(chunk);
|
||||||
});
|
});
|
||||||
|
} catch (err) {
|
||||||
|
attempt++;
|
||||||
|
if (attempt >= maxRetries) {
|
||||||
|
throw new Error(`Upload interrupted on chunk ${i + 1}/${totalChunks}. Check connection.`);
|
||||||
|
}
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `${fileLabel}reconnecting chunk ${i + 1}... (retry ${attempt}/${maxRetries})`;
|
||||||
|
}
|
||||||
|
await new Promise(r => setTimeout(r, attempt * 1200));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
28
server.js
28
server.js
|
|
@ -270,18 +270,38 @@ app.post('/upload/chunk', uploadRateLimiter, storageQuotaGuard, (req, res) => {
|
||||||
return res.status(400).json({ error: 'Invalid chunk upload parameters' });
|
return res.status(400).json({ error: 'Invalid chunk upload parameters' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const partPath = path.join(chunksDir, `${uploadId}.part`);
|
// Write each chunk to an isolated part file to support safe, idempotent retries
|
||||||
const fileStream = fs.createWriteStream(partPath, { flags: chunkIndex === 0 ? 'w' : 'a' });
|
const partPath = path.join(chunksDir, `${uploadId}_${chunkIndex}.part`);
|
||||||
|
const fileStream = fs.createWriteStream(partPath, { flags: 'w' });
|
||||||
|
|
||||||
req.pipe(fileStream);
|
req.pipe(fileStream);
|
||||||
|
|
||||||
fileStream.on('finish', async () => {
|
fileStream.on('finish', async () => {
|
||||||
if (chunkIndex === totalChunks - 1) {
|
if (chunkIndex === totalChunks - 1) {
|
||||||
// All chunks received! Move assembled file into uploadDir
|
// Final chunk reached - verify and assemble all parts sequentially
|
||||||
try {
|
try {
|
||||||
const safeName = getSafeUniqueName(uploadDir, filename);
|
const safeName = getSafeUniqueName(uploadDir, filename);
|
||||||
const finalPath = path.join(uploadDir, safeName);
|
const finalPath = path.join(uploadDir, safeName);
|
||||||
await fs.promises.rename(partPath, finalPath);
|
const finalStream = fs.createWriteStream(finalPath);
|
||||||
|
|
||||||
|
for (let i = 0; i < totalChunks; i++) {
|
||||||
|
const p = path.join(chunksDir, `${uploadId}_${i}.part`);
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
const r = fs.createReadStream(p);
|
||||||
|
r.on('error', reject);
|
||||||
|
r.on('end', () => {
|
||||||
|
fs.promises.unlink(p).catch(() => {});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
r.pipe(finalStream, { end: false });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
finalStream.end();
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
finalStream.on('finish', resolve);
|
||||||
|
finalStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
const stats = await fs.promises.stat(finalPath);
|
const stats = await fs.promises.stat(finalPath);
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue