Skip to content

Commit

Permalink
limit writes to
Browse files Browse the repository at this point in the history
* chunks of 511MiB and
* total of 2GiB
  • Loading branch information
sokra committed Feb 15, 2022
1 parent 339af7e commit 4a53e9a
Showing 1 changed file with 30 additions and 9 deletions.
39 changes: 30 additions & 9 deletions lib/serialization/FileMiddleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ Section -> Buffer

// "wpc" + 1 in little-endian
const VERSION = 0x01637077;
const _1GiB = 1 * 1024 * 1024 * 1024;
const WRITE_LIMIT_TOTAL = 0x7fff0000;
const WRITE_LIMIT_CHUNK = 511 * 1024 * 1024;

/**
* @param {Buffer[]} buffers buffers
Expand Down Expand Up @@ -454,22 +455,42 @@ class FileMiddleware extends SerializerMiddleware {
stream.on("error", err => reject(err));
stream.on("finish", () => resolve());
}
// use unsafe
if (size <= _1GiB) {
for (const b of content) stream.write(b);
return stream.end();
// split into chunks for WRITE_LIMIT_CHUNK size
const chunks = [];
for (const b of content) {
if (b.length < WRITE_LIMIT_CHUNK) {
chunks.push(b);
} else {
for (let i = 0; i < b.length; i += WRITE_LIMIT_CHUNK) {
chunks.push(b.slice(i, i + WRITE_LIMIT_CHUNK));
}
}
}

const len = content.length;
const len = chunks.length;
let i = 0;
const batchWrite = err => {
// will be handled in "on" error handler
if (err) return;

if (i === len) {
stream.end();
return;
}
// will be handle in "on" handler
if (err) return;
stream.write(content[i++], batchWrite);

// queue up a batch of chunks up to the write limit
// end is exclusive
let end = i;
let sum = chunks[end++].length;
while (end < len) {
sum += chunks[end].length;
if (sum > WRITE_LIMIT_TOTAL) break;
end++;
}
while (i < end - 1) {
stream.write(chunks[i++]);
}
stream.write(chunks[i++], batchWrite);
};
batchWrite();
});
Expand Down

0 comments on commit 4a53e9a

Please sign in to comment.