Merge pull request #343 from JuliaHEP/rntuple_writing

[RNTuple] Writing Phase 1
JuliaHEP · Sep 13, 2024 · fa0a5e3 · fa0a5e3
2 parents ccd8366 + 23e3b93
commit fa0a5e3
Show file tree

Hide file tree

Showing 10 changed files with 1,067 additions and 20 deletions.
diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml
@@ -14,7 +14,7 @@ jobs:
           # This should be the path to the paper within your repo.
           paper-path: paper/paper.md
       - name: Upload
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: paper
           # This is the output path where Pandoc will write the compiled

diff --git a/Project.toml b/Project.toml
@@ -5,6 +5,7 @@ version = "0.10.32"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
 ArraysOfArrays = "65a8f2f4-9b39-5baf-92e2-a9cc46fdf018"
 BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1"
 CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
@@ -32,6 +33,7 @@ XXHashNative = "e5d8e439-e7fa-4681-9c12-1c64bda517be"
 
 [compat]
 AbstractTrees = "^0.4"
+Accessors = "0.1.36"
 Aqua = "^0.8"
 ArraysOfArrays = "^0.6"
 Arrow = "~2.7.1"
@@ -54,6 +56,7 @@ Parameters = "^0.12"
 Pkg = "^1.0"
 PrecompileTools = "^1.2.0"
 PrettyTables = "^2.1"
+Random = "^1.0"
 SHA = "^0.7, ^1.0"
 SentinelArrays = "^1.3"
 StaticArrays = "^1"
@@ -72,10 +75,11 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LorentzVectors = "3f54b04b-17fc-5cd4-9758-90c048d965e3"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "SHA", "StaticArrays", "TOML", "Test"]
+test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "Random", "SHA", "StaticArrays", "TOML", "Test"]
diff --git a/src/RNTuple/Writing/Stubs.jl b/src/RNTuple/Writing/Stubs.jl
@@ -0,0 +1,97 @@
+module Stubs
+using ..UnROOT
+
+const file_preamble = [
+    0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45,
+]
+
+const fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
+const dummy_padding1 = [
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00,
+]
+
+const tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "")
+const tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "")
+const tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000)
+const dummy_padding2 = [
+    0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+]
+
+
+const RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "")
+const rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [
+    UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""),
+], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[])
+
+
+const RBlob2 = UnROOT.RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "")
+const page1 = [
+    0xCE, 0xCE, 0xCE, 0xCE,
+]
+
+const RBlob3 = UnROOT.RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "")
+const cluster_summary = UnROOT.Write_RNTupleListFrame([UnROOT.ClusterSummary(0, 1)])
+const nested_page_locations = 
+UnROOT.RNTuplePageTopList([
+    UnROOT.RNTuplePageOuterList([
+        UnROOT.RNTuplePageInnerList([
+            UnROOT.PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )),
+        ]),
+    ]),
+])
+const pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations)
+
+const RBlob4 = UnROOT.RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "")
+const rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [
+    UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))),
+], UnROOT.EnvLink[])
+const tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "")
+# these 6 bytes are between tkey32_anchor and the actual anchor
+const magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04]
+
+const rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b)
+const tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "")
+# 1 key, and it is the RNTuple Anchor
+const n_keys = [
+    0x00, 0x00, 0x00, 0x01,
+]
+
+
+const tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list")
+
+const tsreamerinfo_compressed = [
+    0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2,
+    0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15,
+    0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA,
+    0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04,
+    0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE,
+    0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A,
+    0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37,
+    0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64,
+    0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6,
+    0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5,
+    0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C,
+    0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A,
+    0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17,
+    0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92,
+    0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F,
+    0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C,
+    0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58,
+    0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2,
+    0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7,
+    0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90,
+    0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E,
+    0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15,
+    0x3D, 0xC1, 0xCA,
+]
+
+const tfile_end = [
+    0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35,
+    0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74,
+    0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E,
+    0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00,
+]
+end