From cde36cffc42c73c06f9d39afb2e90c78a9ab595a Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 16 Dec 2020 22:58:53 +0100 Subject: [PATCH] File system image structure, version 2, part 1 --- include/dwarfs/fstypes.h | 23 +++++++++++++++++- src/dwarfs/filesystem_writer.cpp | 40 ++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/include/dwarfs/fstypes.h b/include/dwarfs/fstypes.h index 0d3400ac2..79a6f7de4 100644 --- a/include/dwarfs/fstypes.h +++ b/include/dwarfs/fstypes.h @@ -32,6 +32,7 @@ #include #include "dwarfs/block_compressor.h" // TODO: or the other way round? +#include "dwarfs/checksum.h" namespace dwarfs { @@ -64,7 +65,7 @@ struct iovec_read_buf { }; constexpr uint8_t MAJOR_VERSION = 2; -constexpr uint8_t MINOR_VERSION = 1; +constexpr uint8_t MINOR_VERSION = 2; enum class section_type : uint16_t { BLOCK = 0, @@ -93,6 +94,26 @@ struct section_header { void dump(std::ostream& os) const; }; +struct section_header_v2 { + char magic[6]; // [0] "DWARFS" / file_header no longer needed + uint8_t major; // [6] major version + uint8_t minor; // [7] minor version + uint8_t sha2_512_256[32]; // [8] SHA2-512/256 starting from next field + uint64_t xxh3_64; // [40] XXH3-64 starting from next field + uint32_t number; // [48] section number + uint16_t type; // [52] section type + uint16_t compression; // [54] compression + uint64_t length; // [56] length of section + + static_assert(checksum::digest_size(checksum::algorithm::XXH3_64) == + sizeof(xxh3_64)); + static_assert(checksum::digest_size(checksum::algorithm::SHA2_512_256) == + sizeof(sha2_512_256)); + + std::string to_string() const; + void dump(std::ostream& os) const; +}; + std::string get_compression_name(compression_type type); std::string get_section_name(section_type type); diff --git a/src/dwarfs/filesystem_writer.cpp b/src/dwarfs/filesystem_writer.cpp index 8fd0cccc8..1be459bb0 100644 --- a/src/dwarfs/filesystem_writer.cpp +++ b/src/dwarfs/filesystem_writer.cpp @@ -31,6 +31,7 @@ #include #include "dwarfs/block_compressor.h" +#include "dwarfs/checksum.h" #include "dwarfs/filesystem_writer.h" #include "dwarfs/fstypes.h" #include "dwarfs/logger.h" @@ -155,7 +156,6 @@ class filesystem_writer_ : public filesystem_writer::impl { template void write(const T& obj); void write(const std::vector& data); - void write_file_header(); void writer_thread(); size_t mem_used() const; @@ -172,6 +172,7 @@ class filesystem_writer_ : public filesystem_writer::impl { std::condition_variable cond_; volatile bool flush_; std::thread writer_thread_; + uint32_t section_number_{0}; }; template @@ -188,9 +189,7 @@ filesystem_writer_::filesystem_writer_( , max_queue_size_(max_queue_size) , log_(lgr) , flush_(false) - , writer_thread_(&filesystem_writer_::writer_thread, this) { - write_file_header(); -} + , writer_thread_(&filesystem_writer_::writer_thread, this) {} template filesystem_writer_::~filesystem_writer_() noexcept { @@ -268,24 +267,31 @@ void filesystem_writer_::write(const std::vector& data) { write(reinterpret_cast(&data[0]), data.size()); } -template -void filesystem_writer_::write_file_header() { - file_header hdr; - ::memcpy(&hdr.magic[0], "DWARFS", 6); - hdr.major = MAJOR_VERSION; - hdr.minor = MINOR_VERSION; - write(hdr); -} - template void filesystem_writer_::write(section_type type, compression_type compression, const std::vector& data) { - section_header sh; - sh.type = type; - sh.compression = compression; - sh.unused = 0; + section_header_v2 sh; + ::memcpy(&sh.magic[0], "DWARFS", 6); + sh.major = MAJOR_VERSION; + sh.minor = MINOR_VERSION; + sh.number = section_number_++; + sh.type = static_cast(type); + sh.compression = static_cast(compression); sh.length = data.size(); + + checksum xxh(checksum::algorithm::XXH3_64); + xxh.update(&sh.number, + sizeof(section_header_v2) - offsetof(section_header_v2, number)); + xxh.update(data.data(), data.size()); + DWARFS_CHECK(xxh.finalize(&sh.xxh3_64), "XXH3-64 checksum failed"); + + checksum sha(checksum::algorithm::SHA2_512_256); + sha.update(&sh.xxh3_64, + sizeof(section_header_v2) - offsetof(section_header_v2, xxh3_64)); + sha.update(data.data(), data.size()); + DWARFS_CHECK(sha.finalize(&sh.sha2_512_256), "SHA512/256 checksum failed"); + write(sh); write(data);