Skip to content

Commit

Permalink
src: add utilities to help debugging reproducibility of snapshots
Browse files Browse the repository at this point in the history
- Print offsets in blob serializer
- Add a special node:generate_default_snapshot ID to generate
  the built-in snapshot.
- Improve logging
- Add a test to check the reproducibilty of the snapshot

PR-URL: #50983
Refs: nodejs/build#3043
Reviewed-By: Daniel Lemire <[email protected]>
Reviewed-By: James M Snell <[email protected]>
  • Loading branch information
joyeecheung authored and targos committed Jun 20, 2024
1 parent 004b9ea commit f077afa
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 19 deletions.
24 changes: 19 additions & 5 deletions src/blob_serializer_deserializer-inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,8 @@ size_t BlobSerializer<Impl>::WriteVector(const std::vector<T>& data) {
if (is_debug) {
std::string str = std::is_arithmetic_v<T> ? "" : ToStr(data);
std::string name = GetName<T>();
Debug("\nWriteVector<%s>() (%d-byte), count=%d: %s\n",
Debug("\nAt 0x%x: WriteVector<%s>() (%d-byte), count=%d: %s\n",
sink.size(),
name.c_str(),
sizeof(T),
data.size(),
Expand Down Expand Up @@ -270,7 +271,10 @@ size_t BlobSerializer<Impl>::WriteVector(const std::vector<T>& data) {
template <typename Impl>
size_t BlobSerializer<Impl>::WriteStringView(std::string_view data,
StringLogMode mode) {
Debug("WriteStringView(), length=%zu: %p\n", data.size(), data.data());
Debug("At 0x%x: WriteStringView(), length=%zu: %p\n",
sink.size(),
data.size(),
data.data());
size_t written_total = WriteArithmetic<size_t>(data.size());

size_t length = data.size();
Expand All @@ -294,17 +298,27 @@ size_t BlobSerializer<Impl>::WriteString(const std::string& data) {
return WriteStringView(data, StringLogMode::kAddressAndContent);
}

static size_t kPreviewCount = 16;

// Helper for writing an array of numeric types.
template <typename Impl>
template <typename T>
size_t BlobSerializer<Impl>::WriteArithmetic(const T* data, size_t count) {
static_assert(std::is_arithmetic_v<T>, "Arithmetic type");
DCHECK_GT(count, 0); // Should not write contents for vectors of size 0.
if (is_debug) {
std::string str =
"{ " + std::to_string(data[0]) + (count > 1 ? ", ... }" : " }");
size_t preview_count = count < kPreviewCount ? count : kPreviewCount;
std::string str = "{ ";
for (size_t i = 0; i < preview_count; ++i) {
str += (std::to_string(data[i]) + ",");
}
if (count > preview_count) {
str += "...";
}
str += "}";
std::string name = GetName<T>();
Debug("Write<%s>() (%zu-byte), count=%zu: %s",
Debug("At 0x%x: Write<%s>() (%zu-byte), count=%zu: %s",
sink.size(),
name.c_str(),
sizeof(T),
count,
Expand Down
24 changes: 15 additions & 9 deletions src/node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1337,18 +1337,24 @@ ExitCode GenerateAndWriteSnapshotData(const SnapshotData** snapshot_data_ptr,
return exit_code;
}
} else {
std::optional<std::string> builder_script_content;
// Otherwise, load and run the specified builder script.
std::unique_ptr<SnapshotData> generated_data =
std::make_unique<SnapshotData>();
std::string builder_script_content;
int r = ReadFileSync(&builder_script_content, builder_script.c_str());
if (r != 0) {
FPrintF(stderr,
"Cannot read builder script %s for building snapshot. %s: %s",
builder_script,
uv_err_name(r),
uv_strerror(r));
return ExitCode::kGenericUserError;
if (builder_script != "node:generate_default_snapshot") {
builder_script_content = std::string();
int r = ReadFileSync(&(builder_script_content.value()),
builder_script.c_str());
if (r != 0) {
FPrintF(stderr,
"Cannot read builder script %s for building snapshot. %s: %s\n",
builder_script,
uv_err_name(r),
uv_strerror(r));
return ExitCode::kGenericUserError;
}
} else {
snapshot_config.builder_script_path = std::nullopt;
}

exit_code = node::SnapshotBuilder::Generate(generated_data.get(),
Expand Down
11 changes: 6 additions & 5 deletions src/node_snapshotable.cc
Original file line number Diff line number Diff line change
Expand Up @@ -599,16 +599,17 @@ std::vector<char> SnapshotData::ToBlob() const {
size_t written_total = 0;

// Metadata
w.Debug("Write magic %" PRIx32 "\n", kMagic);
w.Debug("0x%x: Write magic %" PRIx32 "\n", w.sink.size(), kMagic);
written_total += w.WriteArithmetic<uint32_t>(kMagic);
w.Debug("Write metadata\n");
w.Debug("0x%x: Write metadata\n", w.sink.size());
written_total += w.Write<SnapshotMetadata>(metadata);

w.Debug("0x%x: Write snapshot blob\n", w.sink.size());
written_total += w.Write<v8::StartupData>(v8_snapshot_blob_data);
w.Debug("Write isolate_data_indices\n");
w.Debug("0x%x: Write IsolateDataSerializeInfo\n", w.sink.size());
written_total += w.Write<IsolateDataSerializeInfo>(isolate_data_info);
w.Debug("0x%x: Write EnvSerializeInfo\n", w.sink.size());
written_total += w.Write<EnvSerializeInfo>(env_info);
w.Debug("Write code_cache\n");
w.Debug("0x%x: Write CodeCacheInfo\n", w.sink.size());
written_total += w.WriteVector<builtins::CodeCacheInfo>(code_cache);
w.Debug("SnapshotData::ToBlob() Wrote %d bytes\n", written_total);

Expand Down
70 changes: 70 additions & 0 deletions test/parallel/test-snapshot-reproducible.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
'use strict';

require('../common');
const { spawnSyncAndAssert } = require('../common/child_process');
const tmpdir = require('../common/tmpdir');
const fs = require('fs');
const assert = require('assert');

// When the test fails this helper can be modified to write outputs
// differently and aid debugging.
function log(line) {
console.log(line);
}

function generateSnapshot() {
tmpdir.refresh();

spawnSyncAndAssert(
process.execPath,
[
'--random_seed=42',
'--predictable',
'--build-snapshot',
'node:generate_default_snapshot',
],
{
env: { ...process.env, NODE_DEBUG_NATIVE: 'SNAPSHOT_SERDES' },
cwd: tmpdir.path
},
{
stderr(output) {
const lines = output.split('\n');
for (const line of lines) {
if (line.startsWith('0x')) {
log(line);
}
}
},
}
);
const blobPath = tmpdir.resolve('snapshot.blob');
return fs.readFileSync(blobPath);
}

const buf1 = generateSnapshot();
const buf2 = generateSnapshot();

const diff = [];
let offset = 0;
const step = 16;
do {
const length = Math.min(buf1.length - offset, step);
const slice1 = buf1.slice(offset, offset + length).toString('hex');
const slice2 = buf2.slice(offset, offset + length).toString('hex');
if (slice1 !== slice2) {
diff.push({ offset: '0x' + (offset).toString(16), slice1, slice2 });
}
offset += length;
} while (offset < buf1.length);

assert.strictEqual(offset, buf1.length);
if (offset < buf2.length) {
const length = Math.min(buf2.length - offset, step);
const slice2 = buf2.slice(offset, offset + length).toString('hex');
diff.push({ offset, slice1: '', slice2 });
offset += length;
} while (offset < buf2.length);

assert.deepStrictEqual(diff, []);
assert.strictEqual(buf1.length, buf2.length);

0 comments on commit f077afa

Please sign in to comment.