diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 76310246..a349ebaa 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -14,7 +14,7 @@ jobs: # This should be the path to the paper within your repo. paper-path: paper/paper.md - name: Upload - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: paper # This is the output path where Pandoc will write the compiled diff --git a/Project.toml b/Project.toml index 3aab73fe..37cb96a1 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.10.32" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" ArraysOfArrays = "65a8f2f4-9b39-5baf-92e2-a9cc46fdf018" BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1" CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" @@ -32,6 +33,7 @@ XXHashNative = "e5d8e439-e7fa-4681-9c12-1c64bda517be" [compat] AbstractTrees = "^0.4" +Accessors = "0.1.36" Aqua = "^0.8" ArraysOfArrays = "^0.6" Arrow = "~2.7.1" @@ -54,6 +56,7 @@ Parameters = "^0.12" Pkg = "^1.0" PrecompileTools = "^1.2.0" PrettyTables = "^2.1" +Random = "^1.0" SHA = "^0.7, ^1.0" SentinelArrays = "^1.3" StaticArrays = "^1" @@ -72,10 +75,11 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LorentzVectors = "3f54b04b-17fc-5cd4-9758-90c048d965e3" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "SHA", "StaticArrays", "TOML", "Test"] +test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "Random", "SHA", "StaticArrays", "TOML", "Test"] diff --git a/src/RNTuple/Writing/Stubs.jl b/src/RNTuple/Writing/Stubs.jl new file mode 100644 index 00000000..8bd7df97 --- /dev/null +++ b/src/RNTuple/Writing/Stubs.jl @@ -0,0 +1,97 @@ +module Stubs +using ..UnROOT + +const file_preamble = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, +] + +const fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +const dummy_padding1 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] + +const tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") +const tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "") +const tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +const dummy_padding2 = [ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + + +const RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") +const rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), +], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + +const RBlob2 = UnROOT.RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") +const page1 = [ + 0xCE, 0xCE, 0xCE, 0xCE, +] + +const RBlob3 = UnROOT.RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") +const cluster_summary = UnROOT.Write_RNTupleListFrame([UnROOT.ClusterSummary(0, 1)]) +const nested_page_locations = +UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + UnROOT.PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), + ]), + ]), +]) +const pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + +const RBlob4 = UnROOT.RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") +const rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), +], UnROOT.EnvLink[]) +const tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") +# these 6 bytes are between tkey32_anchor and the actual anchor +const magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] + +const rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) +const tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") +# 1 key, and it is the RNTuple Anchor +const n_keys = [ + 0x00, 0x00, 0x00, 0x01, +] + + +const tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") + +const tsreamerinfo_compressed = [ + 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, + 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, + 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, + 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, + 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, + 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, + 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, + 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, + 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, + 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, + 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, + 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, + 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, + 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, + 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, + 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, + 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, + 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, + 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, + 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, + 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, + 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, + 0x3D, 0xC1, 0xCA, +] + +const tfile_end = [ + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, +] +end diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl new file mode 100644 index 00000000..ec40d797 --- /dev/null +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -0,0 +1,565 @@ +using StaticArrays +using UnROOT +using UnROOT: RNTupleFrame, ClusterSummary, PageDescription +using XXHashNative: xxh3_64 +using Accessors +using Tables: istable, columntable, schema + +function color_diff(ary1, ary2) + if length(ary1) != length(ary2) + printstyled("!!! Length mismatch !!!: length(ary1)=$(length(ary1)), length(ary2)=$(length(ary2))\n", color=:red) + end + print("[") + x = 0 + for (i,j) in zip(ary1, ary2) + if x % 8 == 0 + print(" ") + end + if x % 16 == 0 + println() + x = 0 + end + if i != j + printstyled("$(repr(i))/$(repr(j)), ", color=:red) + else + printstyled("$(repr(i)), ", color=:green) + end + x += 1 + end + println() + println("]") +end + +function rnt_write(io::IO, x::AbstractString; legacy=false) + L = length(x) + if legacy + if L > typemax(UInt8) + error("String longer than 255 not implemented") + end + write(io, UInt8(L)) + write(io, codeunits(x)) + else + write(io, UInt32(L)) + write(io, codeunits(x)) + end +end + +function rnt_write(io::IO, x::AbstractVector) + error("Ambiguous type: $(typeof(x)), use Write_RNTupleListFrame or similar wrapper") +end + +function rnt_write(io::IO, x; legacy=false) + if legacy + write(io, bswap(x)) + else + write(io, x) + end +end + +function rnt_write(io::IO, x::AbstractVector{UInt8}; legacy=false) + if legacy + write(io, reverse(x)) + else + write(io, x) + end +end + +function test_io(obj, expected; kw...) + a = IOBuffer() + rnt_write(a, obj; kw...) + ours = take!(a) + if ours != expected + color_diff(ours, expected) + end +end + +function rnt_write(io::IO, x::UnROOT.FileHeader32) + rnt_write(io, x.fBEGIN; legacy=true) + rnt_write(io, x.fEND; legacy=true) + rnt_write(io, x.fSeekFree; legacy=true) + rnt_write(io, x.fNbytesFree; legacy=true) + rnt_write(io, x.nfree; legacy=true) + rnt_write(io, x.fNbytesName; legacy=true) + rnt_write(io, x.fUnits; legacy=true) + rnt_write(io, x.fCompress; legacy=true) + rnt_write(io, x.fSeekInfo; legacy=true) + rnt_write(io, x.fNbytesInfo; legacy=true) + rnt_write(io, x.fUUID; legacy=true) +end + +function rnt_write(io::IO, x::UnROOT.TKey32) + p = position(io) + rnt_write(io, x.fNbytes; legacy=true) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fObjlen; legacy=true) + rnt_write(io, x.fDatime; legacy=true) + rnt_write(io, x.fKeylen; legacy=true) + rnt_write(io, x.fCycle; legacy=true) + rnt_write(io, x.fSeekKey; legacy=true) + rnt_write(io, x.fSeekPdir; legacy=true) + rnt_write(io, x.fClassName; legacy=true) + rnt_write(io, x.fName; legacy=true) + rnt_write(io, x.fTitle; legacy=true) + @assert position(io) - p == x.fKeylen +end + +struct TFile_write + filename::String + unknown::String +end +function rnt_write(io::IO, x::TFile_write) + rnt_write(io, x.filename; legacy=true) + rnt_write(io, x.unknown; legacy=true) +end + +function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fDatimeC; legacy=true) + rnt_write(io, x.fDatimeM; legacy=true) + rnt_write(io, x.fNbytesKeys; legacy=true) + rnt_write(io, x.fNbytesName; legacy=true) + rnt_write(io, x.fSeekDir; legacy=true) + rnt_write(io, x.fSeekParent; legacy=true) + rnt_write(io, x.fSeekKeys; legacy=true) +end + +struct RBlob + fNbytes::Int32 + fVersion::Int16 + fObjLen::Int32 + fDatime::UInt32 + fKeyLen::Int16 + fCycle::Int16 + fSeekKey::Int32 + fSeekPdir::Int32 + fClassName::String + fName::String + fTitle::String +end +function rnt_write(io::IO, x::RBlob) + rnt_write(io, x.fNbytes; legacy=true) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fObjLen; legacy=true) + rnt_write(io, x.fDatime; legacy=true) + rnt_write(io, x.fKeyLen; legacy=true) + rnt_write(io, x.fCycle; legacy=true) + rnt_write(io, x.fSeekKey; legacy=true) + rnt_write(io, x.fSeekPdir; legacy=true) + rnt_write(io, x.fClassName; legacy=true) + rnt_write(io, x.fName; legacy=true) + rnt_write(io, x.fTitle; legacy=true) +end + +function rnt_write(io::IO, x::UnROOT.FieldRecord) + rnt_write(io, x.field_version) + rnt_write(io, x.type_version) + rnt_write(io, x.parent_field_id) + rnt_write(io, x.struct_role) + rnt_write(io, x.flags) + if !iszero(x.repetition) + if x.flags != 0x01 + error("Repetition is set but flags is not 0x01") + end + rnt_write(io, x.repetition) + end + rnt_write(io, x.field_name) + rnt_write(io, x.type_name) + rnt_write(io, x.type_alias) + rnt_write(io, x.field_desc) +end + +function rnt_write(io::IO, x::UnROOT.ColumnRecord) + rnt_write(io, x.type) + rnt_write(io, x.nbits) + rnt_write(io, x.field_id) + rnt_write(io, x.flags) + if !iszero(x.first_ele_idx) + if x.flags != 0x08 + error("First element index is set but flags is not 0x08") + end + rnt_write(io, x.first_ele_idx) + end +end + +function rnt_write(io::IO, x::RNTupleFrame{T}) where T + temp_io = IOBuffer() + rnt_write(temp_io, x.payload) + size = temp_io.size + 8 + write(io, Int64(size)) + seekstart(temp_io) + write(io, temp_io) +end + +struct Write_RNTupleListFrame{T<:AbstractArray} + payload::T +end +function rnt_write(io::IO, x::Write_RNTupleListFrame) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, RNTupleFrame(x)) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end + +function _checksum(x::UnROOT.RNTupleHeader) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.name) + rnt_write(temp_io, x.ntuple_description) + rnt_write(temp_io, x.writer_identifier) + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_columns)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_infos)) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0001 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + + return xxh3_64(payload_ary) +end + +function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.name) + rnt_write(temp_io, x.ntuple_description) + rnt_write(temp_io, x.writer_identifier) + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_columns)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_infos)) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0001 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end +end + +function rnt_write(io::IO, x::ClusterSummary) + rnt_write(io, x.first_entry_number) + rnt_write(io, x.number_of_entries) +end + +function rnt_write(io::IO, x::UnROOT.Locator) + rnt_write(io, x.num_bytes) + rnt_write(io, x.offset) +end + +function rnt_write(io::IO, x::PageDescription) + rnt_write(io, x.num_elements) + rnt_write(io, x.locator) +end + +function rnt_write(io::IO, x::UnROOT.RNTuplePageTopList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end +function rnt_write(io::IO, x::UnROOT.RNTuplePageOuterList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end +function rnt_write(io::IO, x::UnROOT.RNTuplePageInnerList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + offset = zero(UInt64) + compression = zero(UInt32) + write(temp_io, offset, compression) + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end + +function rnt_write(io::IO, x::UnROOT.PageLink; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.header_checksum) + rnt_write(temp_io, Write_RNTupleListFrame(x.cluster_summaries)) + rnt_write(temp_io, x.nested_page_locations) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0003 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end +end + +function rnt_write(io::IO, x::UnROOT.EnvLink) + rnt_write(io, x.uncomp_size) + rnt_write(io, x.locator) +end + +""" +@SimpleStruct struct ClusterGroupRecord + minimum_entry_number::Int64 + entry_span::Int64 + num_clusters::Int32 + page_list_link::EnvLink +end +""" +function rnt_write(io::IO, x::UnROOT.ClusterGroupRecord) + rnt_write(io, x.minimum_entry_number) + rnt_write(io, x.entry_span) + rnt_write(io, x.num_clusters) + rnt_write(io, x.page_list_link) +end + +function rnt_write(io::IO, x::UnROOT.RNTupleSchemaExtension) + temp_io = IOBuffer() + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_info)) + + size = temp_io.size + sizeof(Int64) + write(io, Int64(size)) + seekstart(temp_io) + write(io, temp_io) +end + +function rnt_write(io::IO, x::UnROOT.RNTupleFooter; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.header_checksum) + rnt_write(temp_io, x.extension_header_links) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_group_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.cluster_group_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.meta_data_links)) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0002 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end +end + +function rnt_write(io::IO, x::UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple) + temp_io = IOBuffer() + rnt_write(temp_io, x.fVersionEpoch; legacy=true) + rnt_write(temp_io, x.fVersionMajor; legacy=true) + rnt_write(temp_io, x.fVersionMinor; legacy=true) + rnt_write(temp_io, x.fVersionPatch; legacy=true) + rnt_write(temp_io, x.fSeekHeader; legacy=true) + rnt_write(temp_io, x.fNBytesHeader; legacy=true) + rnt_write(temp_io, x.fLenHeader; legacy=true) + rnt_write(temp_io, x.fSeekFooter; legacy=true) + rnt_write(temp_io, x.fNBytesFooter; legacy=true) + rnt_write(temp_io, x.fLenFooter; legacy=true) + payload_ary = take!(temp_io) + checksum = xxh3_64(payload_ary) + rnt_write(io, payload_ary) + rnt_write(io, checksum; legacy=true) +end + +mutable struct WriteObservable{O, T} + io::O + position::Int64 + len::Int64 + object::T +end + +function Base.setindex!(io::WriteObservable, val, key::Symbol) + new_obj = set(io.object, PropertyLens(key), val) + io.object = new_obj + return io +end + +function Base.setindex!(io::WriteObservable, dict::Dict) + for (key, val) in dict + new_obj = set(io.object, PropertyLens(key), val) + io.object = new_obj + end + return io +end + +function flush!(o::WriteObservable) + io = o.io + old_pos = position(io) + + seek(io, o.position) + rnt_write(io, o.object) + seek(io, old_pos) + + nothing +end + +function rnt_write_observe(io::IO, x::T) where T + pos = position(io) + rnt_write(io, x) + len = position(io) - pos + WriteObservable(io, pos, len, x) +end + +function split4_encode(src::AbstractVector{UInt8}) + @views [src[1:4:end-3]; src[2:4:end-2]; src[3:4:end-1]; src[4:4:end]] +end + +function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rntuple_name="myntuple") + if !istable(table) + error("RNTuple writing accepts object compatible with Tables.jl interface, got type $(typeof(table))") + end + + input_schema = schema(table) + input_Ncols = length(input_schema.names) + if input_Ncols != 1 + error("Currently, RNTuple writing only supports a single, UInt32 column, got $input_Ncols columns") + end + input_T = only(input_schema.types) + if input_T != UInt32 + error("Currently, RNTuple writing only supports a single, UInt32 column, got type $input_T") + end + input_col = only(columntable(table)) + input_length = length(input_col) + if input_length > 65535 + error("Input too long: RNTuple writing currently only supports a single page (65535 elements)") + end + + + rntAnchor_update = Dict{Symbol, Any}() + + file_preamble_obs = rnt_write_observe(file, Stubs.file_preamble) + fileheader_obs = rnt_write_observe(file, Stubs.fileheader) + dummy_padding1_obs = rnt_write_observe(file, Stubs.dummy_padding1) + + fileheader_obs[:fBEGIN] = UInt32(position(file)) + + tkey32_tfile_obs = rnt_write_observe(file, Stubs.tkey32_tfile) + tkey32_tfile_obs[:fName] = file_name + tfile_obs = rnt_write_observe(file, Stubs.tfile) + + tdirectory32_obs = rnt_write_observe(file, Stubs.tdirectory32) + dummy_padding2_obs = rnt_write_observe(file, Stubs.dummy_padding2) + + RBlob1_obs = rnt_write_observe(file, Stubs.RBlob1) + rntAnchor_update[:fSeekHeader] = UInt32(position(file)) + rnt_header = UnROOT.RNTupleHeader(zero(UInt64), rntuple_name, "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, string(only(input_schema.names)), "std::uint32_t", "", ""), + ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + rnt_header_obs = rnt_write_observe(file, rnt_header) + rntAnchor_update[:fNBytesHeader] = rnt_header_obs.len + rntAnchor_update[:fLenHeader] = rnt_header_obs.len + + RBlob2_obs = rnt_write_observe(file, Stubs.RBlob2) + page1 = reinterpret(UInt8, input_col) + page1_bytes = split4_encode(page1) + page1_position = position(file) + page1_obs = rnt_write_observe(file, page1_bytes) + + RBlob3_obs = rnt_write_observe(file, Stubs.RBlob3) + cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, input_length)]) + nested_page_locations = + UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + PageDescription(input_length, UnROOT.Locator(sizeof(input_T) * input_length, page1_position, )), + ]), + ]), + ]) + + # stub checksum 0x3dec59c009c67e28 + pagelink = UnROOT.PageLink(_checksum(rnt_header_obs.object), cluster_summary.payload, nested_page_locations) + pagelink_position = position(file) + pagelink_obs = rnt_write_observe(file, pagelink) + + RBlob4_obs = rnt_write_observe(file, Stubs.RBlob4) + rntAnchor_update[:fSeekFooter] = UInt32(position(file)) + rnt_footer = UnROOT.RNTupleFooter(0, _checksum(rnt_header_obs.object), UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, input_length, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, pagelink_position, ))), + ], UnROOT.EnvLink[]) + rnt_footer_obs = rnt_write_observe(file, rnt_footer) + rntAnchor_update[:fNBytesFooter] = 0xac + rntAnchor_update[:fLenFooter] = 0xac + + tkey32_anchor_position = position(file) + tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, tkey32_anchor_position, 100, "ROOT::Experimental::RNTuple", rntuple_name, "") + tkey32_anchor_obs1 = rnt_write_observe(file, tkey32_anchor) + magic_6bytes_obs = rnt_write_observe(file, Stubs.magic_6bytes) + rnt_anchor_obs = rnt_write_observe(file, Stubs.rnt_anchor) + Base.setindex!(rnt_anchor_obs, rntAnchor_update) + + tdirectory32_obs[:fSeekKeys] = UInt32(position(file)) + tkey32_TDirectory_obs = rnt_write_observe(file, Stubs.tkey32_TDirectory) + n_keys_obs = rnt_write_observe(file, Stubs.n_keys) + tkey32_anchor_obs2 = rnt_write_observe(file, tkey32_anchor) + + fileheader_obs[:fSeekInfo] = UInt32(position(file)) + tkey32_TStreamerInfo_obs = rnt_write_observe(file, Stubs.tkey32_TStreamerInfo) + tsreamerinfo_compressed_obs = rnt_write_observe(file, Stubs.tsreamerinfo_compressed) + fileheader_obs[:fSeekFree] = UInt32(position(file)) + tfile_end_obs = rnt_write_observe(file, Stubs.tfile_end) + fileheader_obs[:fEND] = UInt32(position(file)) + + flush!(tkey32_tfile_obs) + flush!(tdirectory32_obs) + flush!(fileheader_obs) + flush!(rnt_anchor_obs) +end diff --git a/src/RNTuple/bootstrap.jl b/src/RNTuple/bootstrap.jl index b2130e88..9ea9f61c 100644 --- a/src/RNTuple/bootstrap.jl +++ b/src/RNTuple/bootstrap.jl @@ -16,6 +16,9 @@ end function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) local_io = datastream(io, tkey) skip(local_io, 6) + _before_anchor = position(local_io) + anchor_checksum = xxh3_64(read(local_io, 2*4 + 6*8)) + seek(local_io, _before_anchor) anchor = ROOT_3a3a_Experimental_3a3a_RNTuple(; fVersionEpoch = readtype(local_io, UInt16), fVersionMajor = readtype(local_io, UInt16), @@ -30,6 +33,9 @@ function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) fChecksum = readtype(local_io, UInt64), ) + @assert anchor.fChecksum == anchor_checksum "RNtuple anchor checksum doesn't match" + + header_bytes = decompress_bytes(read_seek_nb(io, anchor.fSeekHeader, anchor.fNBytesHeader), anchor.fLenHeader) header_io = IOBuffer(header_bytes) header = _rntuple_read(header_io, RNTupleEnvelope{RNTupleHeader}) diff --git a/src/RNTuple/footer.jl b/src/RNTuple/footer.jl index e0a9a006..8c825bec 100644 --- a/src/RNTuple/footer.jl +++ b/src/RNTuple/footer.jl @@ -101,26 +101,32 @@ end number_of_entries::Int64 end -struct RNTupleListNoFrame{T} <: AbstractVector{T} - payload::Vector{T} -end -Base.size(r::RNTupleListNoFrame) = size(r.payload) -Base.getindex(r::RNTupleListNoFrame, i) = r.payload[i] -Base.setindex!(r::RNTupleListNoFrame, v, i) = (r.payload[i] = v) -# without the inner Frame for each item -function _rntuple_read(io, ::Type{RNTupleListNoFrame{T}}) where T - pos = position(io) - Size = read(io, Int64) - @assert Size < 0 - NumItems = read(io, Int32) - end_pos = pos - Size - res = T[_rntuple_read(io, T) for _=1:NumItems] - seek(io, end_pos) - return RNTupleListNoFrame(res) +for x in (:RNTuplePageTopList, :RNTuplePageOuterList, :RNTuplePageInnerList) + @eval begin + struct ($x){T} <: AbstractVector{T} + payload::Vector{T} + end + + function _rntuple_read(io, ::Type{$x{T}}) where T + pos = position(io) + Size = read(io, Int64) + @assert Size < 0 + NumItems = read(io, Int32) + end_pos = pos - Size + res = T[_rntuple_read(io, T) for _=1:NumItems] + seek(io, end_pos) + return $x(res) + end + + Base.size(r::$x) = size(r.payload) + Base.getindex(r::$x, i) = r.payload[i] + Base.setindex!(r::$x, v, i) = (r.payload[i] = v) + + end end @SimpleStruct struct PageLink header_checksum::UInt64 cluster_summaries::Vector{ClusterSummary} - nested_page_locations::RNTupleListNoFrame{RNTupleListNoFrame{RNTupleListNoFrame{PageDescription}}} + nested_page_locations::RNTuplePageTopList{RNTuplePageOuterList{RNTuplePageInnerList{PageDescription}}} end diff --git a/src/UnROOT.jl b/src/UnROOT.jl index 6126c9cf..cddbb185 100644 --- a/src/UnROOT.jl +++ b/src/UnROOT.jl @@ -65,6 +65,9 @@ include("RNTuple/highlevel.jl") include("RNTuple/fieldcolumn_reading.jl") include("RNTuple/displays.jl") +include("RNTuple/Writing/TFileWriter.jl") +include("RNTuple/Writing/Stubs.jl") + _maxthreadid() = @static if VERSION < v"1.9" Threads.nthreads() else diff --git a/src/types.jl b/src/types.jl index e3cb39f1..fc5e08b9 100644 --- a/src/types.jl +++ b/src/types.jl @@ -110,7 +110,7 @@ end iscompressed(t::T) where T<:Union{TKey, TBasketKey} = t.fObjlen != t.fNbytes - t.fKeylen origin(t::T) where T<:Union{TKey, TBasketKey} = iscompressed(t) ? -t.fKeylen : t.fSeekKey -seekstart(io, t::T) where T<:Union{TKey, TBasketKey} = seek(io, t.fSeekKey + t.fKeylen) +Base.seekstart(io, t::T) where T<:Union{TKey, TBasketKey} = seek(io, t.fSeekKey + t.fKeylen) datastream(io, tkey::TKey) = IOBuffer(decompress_datastreambytes(compressed_datastream(io, tkey), tkey)) diff --git a/test/RNTupleWriting/lowlevel.jl b/test/RNTupleWriting/lowlevel.jl new file mode 100644 index 00000000..d01f3f03 --- /dev/null +++ b/test/RNTupleWriting/lowlevel.jl @@ -0,0 +1,365 @@ +using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob +using XXHashNative: xxh3_64 +using Tables: columntable +using Random + +const REFERENCE_BYTES = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, + 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, + 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x18, 0x74, + 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, + 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, + 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, + 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, + 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, + 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, + 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, 0xCE, 0xCE, 0xCE, 0xCE, 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, + 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, + 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, + 0x25, 0x1E, 0x55, 0x4C, 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, + 0xFE, 0x3C, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, + 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, + 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, + 0x3A, 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, + 0x65, 0x00, 0x40, 0x00, 0x00, 0x42, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, + 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, + 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, + 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, + 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x00, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, + 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, + 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, + 0x74, 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, + 0xC2, 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, + 0x15, 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, + 0xDA, 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, + 0x04, 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, + 0xDE, 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, + 0x2A, 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, + 0x37, 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, + 0x64, 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, + 0xD6, 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, + 0xC5, 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, + 0x0C, 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, + 0x8A, 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, + 0x17, 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, + 0x92, 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, + 0x0F, 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, + 0x7C, 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, + 0x58, 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, + 0xF2, 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, + 0xC7, 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, + 0x90, 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, + 0x2E, 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, + 0x15, 0x3D, 0xC1, 0xCA, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, + 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, + 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, + 0x35, 0x94, 0x00, +] + +function test_io(obj, expected; kw...) + a = IOBuffer() + rnt_write(a, obj; kw...) + ours = take!(a) + @test ours == expected +end + +@testset "RNTuple Writing - Internal" begin + +dummy_FileHeader = [ + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(UnROOT.Stubs.fileheader, dummy_FileHeader) + +dummy_tkey32_tfile = [ + 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(UnROOT.Stubs.tkey32_tfile, dummy_tkey32_tfile) + +dummy_tfile = [ + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(UnROOT.Stubs.tfile, dummy_tfile) + +dummy_tdirectory32 = [ + 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, + 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, +] +test_io(UnROOT.Stubs.tdirectory32, dummy_tdirectory32) + +dummy_padding2 = [ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +dummy_RBlob1 = [ + 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(UnROOT.Stubs.RBlob1, dummy_RBlob1) + +# ==================================== side tests begin ==================================== + +field_record = UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", "") +dummy_field_record = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(field_record, dummy_field_record) + +column_record = UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0) +dummy_column_record = [ + 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(column_record, dummy_column_record) + +envelope_frame_field_record = Write_RNTupleListFrame([field_record]) +dummy_envelope_frame_field_record = [ + 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, + 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(envelope_frame_field_record, dummy_envelope_frame_field_record) + +dummy_rnt_header_payload = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, + 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, + 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, + 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, +] +test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header_payload; envelope=false) + +# ==================================== side tests end ==================================== + +dummy_rnt_header = [ + 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, +] +test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header; envelope=true) + +dummy_RBlob2 = [ + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(UnROOT.Stubs.RBlob2, dummy_RBlob2) + +dummy_RBlob3 = [ + 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(UnROOT.Stubs.RBlob3, dummy_RBlob3) + +# ================= side tests begin ================= +dummy_cluster_summary = [ + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +] +test_io(UnROOT.Stubs.cluster_summary, dummy_cluster_summary) + +# > https://github.com/root-project/root/blob/1a854602e42d4493f56a26e35e19bdf23b7d0933/tree/ntuple/v7/doc/specifications.md?plain=1#L672 +# > The inner list is followed by a 64bit unsigned integer element offset and the 32bit compression settings +dummy_inner_list_frame = [ + 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +inner_list_frame = UnROOT.RNTuplePageInnerList([ + PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), +]) +test_io(inner_list_frame, dummy_inner_list_frame) + +dummy_pagelink_noenvelope = [ + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +] +test_io(UnROOT.Stubs.pagelink, dummy_pagelink_noenvelope; envelope=false) +# ================= side tests end ================= + + +dummy_pagelink = [ + 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, +] +test_io(UnROOT.Stubs.pagelink, dummy_pagelink) + +dummy_RBlob4 = [ + 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(UnROOT.Stubs.RBlob4, dummy_RBlob4) + +dummy_rnt_footer = [ + 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, + 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, +] +test_io(UnROOT.Stubs.rnt_footer, dummy_rnt_footer) + +dummy_tkey32_anchor = [ + 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, + 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, + 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, +] + +dummy_rnt_anchor = [ + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, +] +test_io(UnROOT.Stubs.rnt_anchor, dummy_rnt_anchor) + +dummy_tkey32_TDirectory = [ + 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(UnROOT.Stubs.tkey32_TDirectory, dummy_tkey32_TDirectory) + +dummy_tkey32_TStreamerInfo = [ + 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, + 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, + 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, +] +test_io(UnROOT.Stubs.tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) + + +MINE = [ + UnROOT.Stubs.file_preamble; + dummy_FileHeader; UnROOT.Stubs.dummy_padding1; + dummy_tkey32_tfile; dummy_tfile; + dummy_tdirectory32; UnROOT.Stubs.dummy_padding2; + dummy_RBlob1; dummy_rnt_header; + dummy_RBlob2; UnROOT.Stubs.page1; + dummy_RBlob3; dummy_pagelink; + dummy_RBlob4; dummy_rnt_footer; + dummy_tkey32_anchor; UnROOT.Stubs.magic_6bytes; dummy_rnt_anchor; + dummy_tkey32_TDirectory; UnROOT.Stubs.n_keys; dummy_tkey32_anchor; + dummy_tkey32_TStreamerInfo; UnROOT.Stubs.tsreamerinfo_compressed; + UnROOT.Stubs.tfile_end +] + +mytable = Dict("one_uint" => UInt32[0xcececece]) +myio = IOBuffer() +UnROOT.write_rntuple(myio, mytable; rntuple_name="myntuple") +@test MINE == REFERENCE_BYTES +mio = take!(myio) +@test MINE == mio + +for _ = 1:100 + newtable = Dict(randstring(rand(2:10)) => rand(UInt32, rand(1:1000))) + newio = IOBuffer() + UnROOT.write_rntuple(newio, newtable) + nio = take!(newio) + + if isfile("a.root") + rm("a.root") + end + + open("a.root", "w") do f + write(f, nio) + end + + rntuple_name = "myntuple" + t = LazyTree("a.root", rntuple_name) + @test sort(names(t)) == sort(collect(keys(newtable))) + @test only(columntable(t)) == only(columntable(newtable)) + +end +end diff --git a/test/runtests.jl b/test/runtests.jl index 0221f962..d4be502c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,5 +28,6 @@ nthreads == 1 && @warn "Running on a single thread. Please re-run the test suite if VERSION >= v"1.9" include("rntuple.jl") + include("./RNTupleWriting/lowlevel.jl") end end