From d21be77e46a91c1e1a7c06799f906122a2140282 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 31 Jan 2023 23:25:21 +0100 Subject: [PATCH 1/2] src: back snapshot I/O with a std::vector sink --- src/node_snapshotable.cc | 129 ++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 55 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index f74bbc0bc41ae9..b985f6b52d93fa 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -2,6 +2,7 @@ #include "node_snapshotable.h" #include #include +#include #include "base_object-inl.h" #include "debug_utils-inl.h" #include "env-inl.h" @@ -135,11 +136,10 @@ std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) { return output; } -class FileIO { +class SnapshotSerializerDeserializer { public: - explicit FileIO(FILE* file) - : f(file), - is_debug(per_process::enabled_debug_list.enabled( + SnapshotSerializerDeserializer() + : is_debug(per_process::enabled_debug_list.enabled( DebugCategory::MKSNAPSHOT)) {} template @@ -181,14 +181,14 @@ class FileIO { return name; } - FILE* f = nullptr; bool is_debug = false; }; -class FileReader : public FileIO { +class SnapshotDeserializer : public SnapshotSerializerDeserializer { public: - explicit FileReader(FILE* file) : FileIO(file) {} - ~FileReader() {} + explicit SnapshotDeserializer(const std::vector* s) + : SnapshotSerializerDeserializer(), sink(s) {} + ~SnapshotDeserializer() {} // Helper for reading numeric types. template @@ -233,19 +233,19 @@ class FileReader : public FileIO { CHECK_GT(length, 0); // There should be no empty strings. MallocedBuffer buf(length + 1); - size_t r = fread(buf.data, 1, length + 1, f); - CHECK_EQ(r, length + 1); + memcpy(buf.data, sink->data() + read_total, length + 1); std::string result(buf.data, length); // This creates a copy of buf.data. if (is_debug) { - Debug("\"%s\", read %d bytes\n", result.c_str(), r); + Debug("\"%s\", read %zu bytes\n", result.c_str(), length + 1); } - read_total += r; + read_total += length + 1; return result; } size_t read_total = 0; + const std::vector* sink = nullptr; private: // Helper for reading an array of numeric types. @@ -258,15 +258,15 @@ class FileReader : public FileIO { Debug("Read<%s>()(%d-byte), count=%d: ", name.c_str(), sizeof(T), count); } - size_t r = fread(out, sizeof(T), count, f); - CHECK_EQ(r, count); + size_t size = sizeof(T) * count; + memcpy(out, sink->data() + read_total, size); if (is_debug) { std::string str = "{ " + std::to_string(out[0]) + (count > 1 ? ", ... }" : " }"); - Debug("%s, read %d bytes\n", str.c_str(), r); + Debug("%s, read %zu bytes\n", str.c_str(), size); } - read_total += r; + read_total += size; } // Helper for reading numeric vectors. @@ -300,10 +300,15 @@ class FileReader : public FileIO { } }; -class FileWriter : public FileIO { +class SnapshotSerializer : public SnapshotSerializerDeserializer { public: - explicit FileWriter(FILE* file) : FileIO(file) {} - ~FileWriter() {} + SnapshotSerializer() : SnapshotSerializerDeserializer() { + // Currently the snapshot blob built with an empty script is around 4MB. + // So use that as the default sink size. + sink.reserve(4 * 1024 * 1024); + } + ~SnapshotSerializer() {} + std::vector sink; // Helper for writing numeric types. template @@ -349,15 +354,15 @@ class FileWriter : public FileIO { size_t written_total = Write(data.size()); if (is_debug) { std::string str = ToStr(data); - Debug("WriteString(), length=%d: \"%s\"\n", data.size(), data.c_str()); + Debug("WriteString(), length=%zu: \"%s\"\n", data.size(), data.c_str()); } - size_t r = fwrite(data.c_str(), 1, data.size() + 1, f); - CHECK_EQ(r, data.size() + 1); - written_total += r; + // Write the null-terminated string. + sink.insert(sink.end(), data.c_str(), data.c_str() + data.size() + 1); + written_total += data.size(); if (is_debug) { - Debug("WriteString() wrote %d bytes\n", written_total); + Debug("WriteString() wrote %zu bytes\n", written_total); } return written_total; @@ -372,20 +377,21 @@ class FileWriter : public FileIO { std::string str = "{ " + std::to_string(data[0]) + (count > 1 ? ", ... }" : " }"); std::string name = GetName(); - Debug("Write<%s>() (%d-byte), count=%d: %s", + Debug("Write<%s>() (%zu-byte), count=%zu: %s", name.c_str(), sizeof(T), count, str.c_str()); } - size_t r = fwrite(data, sizeof(T), count, f); - CHECK_EQ(r, count); + size_t size = sizeof(T) * count; + const char* pos = reinterpret_cast(data); + sink.insert(sink.end(), pos, pos + (sizeof(T) * count)); if (is_debug) { - Debug(", wrote %d bytes\n", r); + Debug(", wrote %zu bytes\n", size); } - return r; + return size; } // Helper for writing numeric vectors. @@ -418,11 +424,11 @@ class FileWriter : public FileIO { // [ 4/8 bytes ] length // [ |length| bytes ] contents template <> -std::string FileReader::Read() { +std::string SnapshotDeserializer::Read() { return ReadString(); } template <> -size_t FileWriter::Write(const std::string& data) { +size_t SnapshotSerializer::Write(const std::string& data) { return WriteString(data); } @@ -430,7 +436,7 @@ size_t FileWriter::Write(const std::string& data) { // [ 4/8 bytes ] raw_size // [ |raw_size| bytes ] contents template <> -v8::StartupData FileReader::Read() { +v8::StartupData SnapshotDeserializer::Read() { Debug("Read()\n"); int raw_size = Read(); @@ -445,7 +451,7 @@ v8::StartupData FileReader::Read() { } template <> -size_t FileWriter::Write(const v8::StartupData& data) { +size_t SnapshotSerializer::Write(const v8::StartupData& data) { Debug("\nWrite() size=%d\n", data.raw_size); CHECK_GT(data.raw_size, 0); // There should be no startup data of size 0. @@ -462,7 +468,7 @@ size_t FileWriter::Write(const v8::StartupData& data) { // [ 4/8 bytes ] length of module code cache // [ ... ] |length| bytes of module code cache template <> -builtins::CodeCacheInfo FileReader::Read() { +builtins::CodeCacheInfo SnapshotDeserializer::Read() { Debug("Read()\n"); builtins::CodeCacheInfo result{ReadString(), ReadVector()}; @@ -475,7 +481,7 @@ builtins::CodeCacheInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const builtins::CodeCacheInfo& data) { +size_t SnapshotSerializer::Write(const builtins::CodeCacheInfo& data) { Debug("\nWrite() id = %s" ", size=%d\n", data.id.c_str(), @@ -495,7 +501,7 @@ size_t FileWriter::Write(const builtins::CodeCacheInfo& data) { // [ 4/8 bytes ] index in the snapshot blob, can be used with // GetDataFromSnapshotOnce(). template <> -PropInfo FileReader::Read() { +PropInfo SnapshotDeserializer::Read() { Debug("Read()\n"); PropInfo result; @@ -512,7 +518,7 @@ PropInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const PropInfo& data) { +size_t SnapshotSerializer::Write(const PropInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("Write() %s\n", str.c_str()); @@ -535,7 +541,7 @@ size_t FileWriter::Write(const PropInfo& data) { // [ ... ] snapshot indices of each element in // native_execution_async_resources template <> -AsyncHooks::SerializeInfo FileReader::Read() { +AsyncHooks::SerializeInfo SnapshotDeserializer::Read() { Debug("Read()\n"); AsyncHooks::SerializeInfo result; @@ -553,7 +559,7 @@ AsyncHooks::SerializeInfo FileReader::Read() { return result; } template <> -size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) { +size_t SnapshotSerializer::Write(const AsyncHooks::SerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("Write() %s\n", str.c_str()); @@ -573,7 +579,7 @@ size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) { // Layout of TickInfo::SerializeInfo // [ 4/8 bytes ] snapshot index of fields template <> -TickInfo::SerializeInfo FileReader::Read() { +TickInfo::SerializeInfo SnapshotDeserializer::Read() { Debug("Read()\n"); TickInfo::SerializeInfo result; @@ -588,7 +594,7 @@ TickInfo::SerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const TickInfo::SerializeInfo& data) { +size_t SnapshotSerializer::Write(const TickInfo::SerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("Write() %s\n", str.c_str()); @@ -603,7 +609,7 @@ size_t FileWriter::Write(const TickInfo::SerializeInfo& data) { // Layout of TickInfo::SerializeInfo // [ 4/8 bytes ] snapshot index of fields template <> -ImmediateInfo::SerializeInfo FileReader::Read() { +ImmediateInfo::SerializeInfo SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); @@ -617,7 +623,7 @@ ImmediateInfo::SerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) { +size_t SnapshotSerializer::Write(const ImmediateInfo::SerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("Write() %s\n", str.c_str()); @@ -635,7 +641,7 @@ size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) { // [ 4/8 bytes ] snapshot index of milestones // [ 4/8 bytes ] snapshot index of observers template <> -performance::PerformanceState::SerializeInfo FileReader::Read() { +performance::PerformanceState::SerializeInfo SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); @@ -651,7 +657,7 @@ performance::PerformanceState::SerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write( +size_t SnapshotSerializer::Write( const performance::PerformanceState::SerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); @@ -673,7 +679,7 @@ size_t FileWriter::Write( // [ 4/8 bytes ] length of template_values vector // [ ... ] |length| of PropInfo data template <> -IsolateDataSerializeInfo FileReader::Read() { +IsolateDataSerializeInfo SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); @@ -688,7 +694,7 @@ IsolateDataSerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const IsolateDataSerializeInfo& data) { +size_t SnapshotSerializer::Write(const IsolateDataSerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("Write() %s\n", str.c_str()); @@ -702,7 +708,7 @@ size_t FileWriter::Write(const IsolateDataSerializeInfo& data) { } template <> -RealmSerializeInfo FileReader::Read() { +RealmSerializeInfo SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); RealmSerializeInfo result; result.builtins = ReadVector(); @@ -713,7 +719,7 @@ RealmSerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const RealmSerializeInfo& data) { +size_t SnapshotSerializer::Write(const RealmSerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("\nWrite() %s\n", str.c_str()); @@ -730,7 +736,7 @@ size_t FileWriter::Write(const RealmSerializeInfo& data) { } template <> -EnvSerializeInfo FileReader::Read() { +EnvSerializeInfo SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); EnvSerializeInfo result; result.async_hooks = Read(); @@ -747,7 +753,7 @@ EnvSerializeInfo FileReader::Read() { } template <> -size_t FileWriter::Write(const EnvSerializeInfo& data) { +size_t SnapshotSerializer::Write(const EnvSerializeInfo& data) { if (is_debug) { std::string str = ToStr(data); Debug("\nWrite() %s\n", str.c_str()); @@ -780,7 +786,7 @@ size_t FileWriter::Write(const EnvSerializeInfo& data) { // [ ... ] |length| bytes of node platform // [ 4 bytes ] v8 cache version tag template <> -SnapshotMetadata FileReader::Read() { +SnapshotMetadata SnapshotDeserializer::Read() { per_process::Debug(DebugCategory::MKSNAPSHOT, "Read()\n"); SnapshotMetadata result; @@ -798,7 +804,7 @@ SnapshotMetadata FileReader::Read() { } template <> -size_t FileWriter::Write(const SnapshotMetadata& data) { +size_t SnapshotSerializer::Write(const SnapshotMetadata& data) { if (is_debug) { std::string str = ToStr(data); Debug("\nWrite() %s\n", str.c_str()); @@ -833,7 +839,7 @@ size_t FileWriter::Write(const SnapshotMetadata& data) { // [ ... ] code_cache void SnapshotData::ToBlob(FILE* out) const { - FileWriter w(out); + SnapshotSerializer w; w.Debug("SnapshotData::ToBlob()\n"); size_t written_total = 0; @@ -850,11 +856,24 @@ void SnapshotData::ToBlob(FILE* out) const { written_total += w.Write(env_info); w.Debug("Write code_cache\n"); written_total += w.WriteVector(code_cache); + size_t num_written = fwrite(w.sink.data(), w.sink.size(), 1, out); + CHECK_EQ(num_written, 1); w.Debug("SnapshotData::ToBlob() Wrote %d bytes\n", written_total); } bool SnapshotData::FromBlob(SnapshotData* out, FILE* in) { - FileReader r(in); + CHECK_EQ(ftell(in), 0); + int err = fseek(in, 0, SEEK_END); + CHECK_EQ(err, 0); + size_t size = ftell(in); + err = fseek(in, 0, SEEK_SET); + CHECK_EQ(err, 0); + + std::vector sink(size); + size_t num_read = fread(sink.data(), size, 1, in); + CHECK_EQ(num_read, 1); + + SnapshotDeserializer r(&sink); r.Debug("SnapshotData::FromBlob()\n"); DCHECK_EQ(out->data_ownership, SnapshotData::DataOwnership::kOwned); From 1bc6a144bd2781c6f35c4c6d62d5dd242ffb73b9 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 3 Feb 2023 11:34:46 +0100 Subject: [PATCH 2/2] fixup! src: back snapshot I/O with a std::vector sink --- src/node_snapshotable.cc | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/node_snapshotable.cc b/src/node_snapshotable.cc index b985f6b52d93fa..bb4819b1c22407 100644 --- a/src/node_snapshotable.cc +++ b/src/node_snapshotable.cc @@ -186,7 +186,7 @@ class SnapshotSerializerDeserializer { class SnapshotDeserializer : public SnapshotSerializerDeserializer { public: - explicit SnapshotDeserializer(const std::vector* s) + explicit SnapshotDeserializer(const std::vector& s) : SnapshotSerializerDeserializer(), sink(s) {} ~SnapshotDeserializer() {} @@ -233,7 +233,7 @@ class SnapshotDeserializer : public SnapshotSerializerDeserializer { CHECK_GT(length, 0); // There should be no empty strings. MallocedBuffer buf(length + 1); - memcpy(buf.data, sink->data() + read_total, length + 1); + memcpy(buf.data, sink.data() + read_total, length + 1); std::string result(buf.data, length); // This creates a copy of buf.data. if (is_debug) { @@ -245,7 +245,7 @@ class SnapshotDeserializer : public SnapshotSerializerDeserializer { } size_t read_total = 0; - const std::vector* sink = nullptr; + const std::vector& sink; private: // Helper for reading an array of numeric types. @@ -259,7 +259,7 @@ class SnapshotDeserializer : public SnapshotSerializerDeserializer { } size_t size = sizeof(T) * count; - memcpy(out, sink->data() + read_total, size); + memcpy(out, sink.data() + read_total, size); if (is_debug) { std::string str = @@ -358,8 +358,9 @@ class SnapshotSerializer : public SnapshotSerializerDeserializer { } // Write the null-terminated string. - sink.insert(sink.end(), data.c_str(), data.c_str() + data.size() + 1); - written_total += data.size(); + size_t length = data.size() + 1; + sink.insert(sink.end(), data.c_str(), data.c_str() + length); + written_total += length; if (is_debug) { Debug("WriteString() wrote %zu bytes\n", written_total); @@ -386,7 +387,7 @@ class SnapshotSerializer : public SnapshotSerializerDeserializer { size_t size = sizeof(T) * count; const char* pos = reinterpret_cast(data); - sink.insert(sink.end(), pos, pos + (sizeof(T) * count)); + sink.insert(sink.end(), pos, pos + size); if (is_debug) { Debug(", wrote %zu bytes\n", size); @@ -873,7 +874,7 @@ bool SnapshotData::FromBlob(SnapshotData* out, FILE* in) { size_t num_read = fread(sink.data(), size, 1, in); CHECK_EQ(num_read, 1); - SnapshotDeserializer r(&sink); + SnapshotDeserializer r(sink); r.Debug("SnapshotData::FromBlob()\n"); DCHECK_EQ(out->data_ownership, SnapshotData::DataOwnership::kOwned);