Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RNTuple] Writing Phase 1 #343

Merged
merged 15 commits into from
Sep 13, 2024
+1,067 −20
Merged

[RNTuple] Writing Phase 1 #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
24ece08
tests
Moelf Jun 1, 2024
e381458
Replacing xrootdgo by XRootD.jl (#340)
peremato Jun 3, 2024
40f3520
bump to 0.10.31 (#341)
Moelf Jun 4, 2024
a2864bf
clean up
Moelf Jun 18, 2024
05ab59a
gradually replace dummy with function write
Moelf Jun 19, 2024
71ca815
RNTupleHeader with Envelope
Moelf Jun 19, 2024
14083e6
RNTupleFooter
Moelf Jun 19, 2024
4cfe0c1
track locations that we will need to update later
Moelf Jun 19, 2024
4dd09bc
add tests
Moelf Jun 21, 2024
aa9a51a
finish phase 1
Moelf Jun 30, 2024
cb5e1bc
fix RNTuple Anchor Checksum
Moelf Jul 1, 2024
202db37
allow column name change
Moelf Jul 1, 2024
335e9f1
add Random compat
Moelf Jul 1, 2024
1d4d12c
Merge branch 'main' into rntuple_writing
Moelf Sep 13, 2024
23e3b93
fix upload artifact
Moelf Sep 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/draft-pdf.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
# This should be the path to the paper within your repo.
paper-path: paper/paper.md
- name: Upload
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v4
with:
name: paper
# This is the output path where Pandoc will write the compiled
Expand Down
6 changes: 5 additions & 1 deletion Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ version = "0.10.32"

[deps]
AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
ArraysOfArrays = "65a8f2f4-9b39-5baf-92e2-a9cc46fdf018"
BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1"
CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
Expand Down Expand Up @@ -32,6 +33,7 @@ XXHashNative = "e5d8e439-e7fa-4681-9c12-1c64bda517be"

[compat]
AbstractTrees = "^0.4"
Accessors = "0.1.36"
Aqua = "^0.8"
ArraysOfArrays = "^0.6"
Arrow = "~2.7.1"
Expand All @@ -54,6 +56,7 @@ Parameters = "^0.12"
Pkg = "^1.0"
PrecompileTools = "^1.2.0"
PrettyTables = "^2.1"
Random = "^1.0"
SHA = "^0.7, ^1.0"
SentinelArrays = "^1.3"
StaticArrays = "^1"
Expand All @@ -72,10 +75,11 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
LorentzVectors = "3f54b04b-17fc-5cd4-9758-90c048d965e3"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[targets]
test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "SHA", "StaticArrays", "TOML", "Test"]
test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "Random", "SHA", "StaticArrays", "TOML", "Test"]
97 changes: 97 additions & 0 deletions src/RNTuple/Writing/Stubs.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
module Stubs
using ..UnROOT

const file_preamble = [
0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45,
]

const fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
const dummy_padding1 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00,
]

const tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "")
const tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "")
const tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000)
const dummy_padding2 = [
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
]


const RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "")
const rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [
UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""),
], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[])


const RBlob2 = UnROOT.RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "")
const page1 = [
0xCE, 0xCE, 0xCE, 0xCE,
]

const RBlob3 = UnROOT.RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "")
const cluster_summary = UnROOT.Write_RNTupleListFrame([UnROOT.ClusterSummary(0, 1)])
const nested_page_locations =
UnROOT.RNTuplePageTopList([
UnROOT.RNTuplePageOuterList([
UnROOT.RNTuplePageInnerList([
UnROOT.PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )),
]),
]),
])
const pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations)

const RBlob4 = UnROOT.RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "")
const rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [
UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))),
], UnROOT.EnvLink[])
const tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "")
# these 6 bytes are between tkey32_anchor and the actual anchor
const magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04]

const rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b)
const tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "")
# 1 key, and it is the RNTuple Anchor
const n_keys = [
0x00, 0x00, 0x00, 0x01,
]


const tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list")

const tsreamerinfo_compressed = [
0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2,
0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15,
0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA,
0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04,
0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE,
0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A,
0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37,
0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64,
0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6,
0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5,
0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C,
0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A,
0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17,
0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92,
0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F,
0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C,
0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58,
0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2,
0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7,
0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90,
0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E,
0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15,
0x3D, 0xC1, 0xCA,
]

const tfile_end = [
0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35,
0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74,
0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E,
0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00,
]
end
Loading
Loading