Implement compression and splitting option in TRestProcessRunner

.gitlab-ci.yml

@@ -111,7 +111,7 @@ Build and Install:
     paths:
       - ${CI_PROJECT_DIR}/install
     expire_in: 1 day
-
+    
 Load REST Libraries:
   stage: install
   script:
@@ -301,8 +301,12 @@ Event Selection:
     - source ${CI_PROJECT_DIR}/install/thisREST.sh
     - cd ${CI_PROJECT_DIR}/pipeline/selection
     - restManager --c g4Analysis.rml --f "${CI_PROJECT_DIR}/install/examples/restG4/01.NLDBD/Run00001_NLDBD_Test.root"
+
     - restManager --c g4OnSelection.rml --f Run00001_NLDBD_Test_g4Analysis.root
     - restRoot -b -q Validate.C'("Run00001_NLDBD_Test_EvSelection.root")'
+
+    - restManager --c g4EvSelectionIDsFromFile.rml --f Run00001_NLDBD_Test_g4Analysis.root
+    - restRoot -b -q ValidateIDsFromFile.C'("Run00001_NLDBD_Test_EvSelectionIDsFromFile.root")'
 
   artifacts:
     paths:

pipeline/selection/Validate.C

@@ -7,14 +7,16 @@ Int_t Validate(TString fname) {
     TFile* f = new TFile(fname);
     TTree* tr = (TTree*)f->Get("AnalysisTree");
 
-    if (tr->GetEntries() != 3) {
+    tr->Scan("eventID:g4Ana_totalEdep:g4Ana_energyPrimary");
+
+    if (tr->GetEntries() != 1) {
         cout << "Number of entries is not the same!" << endl;
-        cout << "Expected: 3. Obtained: " << tr->GetEntries() << endl;
+        cout << "Expected: 1. Obtained: " << tr->GetEntries() << endl;
         return 1;
     }
 
     // Check IDs.
-    std::vector<Int_t> ids = {0, 3, 6};
+    std::vector<Int_t> ids = {1};
     for (Int_t i = 0; i < tr->GetEntries(); i++) {
         tr->GetEntry(i);
         if (tr->GetLeaf("eventID")->GetValue(0) != ids[i]) {

pipeline/selection/ValidateIDsFromFile.C

@@ -0,0 +1,30 @@
+#include <TFile.h>
+#include <TTree.h>
+
+#include <iostream>
+#include <vector>
+Int_t Validate(TString fname) {
+    TFile* f = new TFile(fname);
+    TTree* tr = (TTree*)f->Get("AnalysisTree");
+
+    if (tr->GetEntries() != 3) {
+        cout << "Number of entries is not the same!" << endl;
+        cout << "Expected: 3. Obtained: " << tr->GetEntries() << endl;
+        return 1;
+    }
+
+    // Check IDs.
+    std::vector<Int_t> ids = {0, 3, 6};
+    for (Int_t i = 0; i < tr->GetEntries(); i++) {
+        tr->GetEntry(i);
+        if (tr->GetLeaf("eventID")->GetValue(0) != ids[i]) {
+            cout << "Same number of entries but different IDs" << endl;
+            cout << "Found entry " << i << " with ID: " << tr->GetLeaf("eventID")->GetValue(0) << endl;
+            cout << "Expected entry " << i << " with ID: " << ids[i] << endl;
+            return 2;
+        }
+    }
+
+    cout << "Tree validation sucess!" << endl;
+    return 0;
+}

pipeline/selection/g4EvSelectionIDsFromFile.rml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+
+<TRestManager name="RESTManagerSim" title="Template manager to select events from file.">
+
+  <globals>
+    <parameter name="verboseLevel" value="essential" /> <!--options are : silent(0), essential(1), info(2), debug(3), extreme(4)-->
+    <parameter name="inputFileName" value=""/>
+    <searchPath value="$ENV{REST_INPUTDATA}/definitions/"/>
+  </globals>
+
+  <TRestRun name="Run metadata" title="REST Metadata run info (template)" verboseLevel="info">
+    <parameter name="experimentName" value="Test"/>
+    <parameter name="runType" value="simulation"/>
+    <parameter name="runNumber" value="-1"/> 
+    <parameter name="runTag" value="preserve"/>
+    <parameter name="runDescription" value=""/>
+    <parameter name="user" value="${USER}"/>
+    <parameter name="verboseLevel" value="2"/>
+    <parameter name="outputFileName" value="Run00001_NLDBD_Test_EvSelectionIDsFromFile.root" />
+  </TRestRun>
+
+
+  <TRestProcessRunner name="TemplateEventProcess" verboseLevel="info">
+    <parameter name="eventsToProcess" value="0" />
+
+
+    <addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="IDs.txt" value="ON"  verboseLevel="info"/>
+
+  </TRestProcessRunner>
+
+  <addTask type="processEvents" value="ON" />
+
+</TRestManager>

pipeline/selection/g4OnSelection.rml

@@ -26,7 +26,7 @@
 
     <addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="Run00001_NLDBD_Test_g4Analysis.root" conditions="g4Ana_totalEdep<2400 AND g4Ana_energyPrimary>1000" value="ON"  verboseLevel="info"/>
 
-    <addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="IDs.txt" value="ON"  verboseLevel="info"/>
+    <!--<addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="IDs.txt" value="ON"  verboseLevel="info"/>-->
 
   </TRestProcessRunner>
 

source/bin/restRoot.cxx

@@ -103,6 +103,36 @@ int main(int argc, char* argv[]) {
                 runTmp->GetEntry(0);
             }
 
+            // command line AnalysisTree object
+            if (runTmp->GetAnalysisTree() != nullptr) {
+                // if (runTmp->GetAnalysisTree()->GetChain() != nullptr) {
+                //    printf("Attaching ana_tree%i...\n", Nfile);
+                //    string evcmd = Form("TChain* ana_tree%i = (TChain*)%s;", Nfile,
+                //        ToString(runTmp->GetAnalysisTree()->GetChain()).c_str());
+                //    if (debug) printf("%s\n", evcmd.c_str());
+                //    gROOT->ProcessLine(evcmd.c_str());
+                //}
+                // else
+                //{
+                printf("Attaching ana_tree%i...\n", Nfile);
+                string evcmd = Form("TRestAnalysisTree* ana_tree%i = (TRestAnalysisTree*)%s;", Nfile,
+                                    ToString(runTmp->GetAnalysisTree()).c_str());
+                if (debug) printf("%s\n", evcmd.c_str());
+                gROOT->ProcessLine(evcmd.c_str());
+                // runTmp->GetEntry(0);
+                //}
+            }
+
+            // command line EventTree object
+            if (runTmp->GetEventTree() != nullptr) {
+                printf("Attaching ev_tree%i...\n", Nfile);
+                string evcmd =
+                    Form("TTree* ev_tree%i = (TTree*)%s;", Nfile, ToString(runTmp->GetEventTree()).c_str());
+                if (debug) printf("%s\n", evcmd.c_str());
+                gROOT->ProcessLine(evcmd.c_str());
+            }
+
+
             printf("\n%s\n", "Attaching metadata structures...");
             Int_t Nmetadata = runTmp->GetNumberOfMetadataStructures();
             map<string, int> metanames;

source/framework/core/inc/TRestAnalysisTree.h

@@ -23,6 +23,7 @@
 #ifndef RestCore_TRestAnalysisTree
 #define RestCore_TRestAnalysisTree
 
+#include "TChain.h"
 #include "TRestEvent.h"
 #include "TRestReflector.h"
 #include "TTree.h"
@@ -44,7 +45,7 @@ class TRestAnalysisTree : public TTree {
     Bool_t fQuickSetObservableValue = true;       //!
     std::vector<RESTValue> fObservables;          //!
     std::map<std::string, int> fObservableIdMap;  //!
-    TTree* fROOTTree;                             //!
+    TChain* fChain = nullptr;                        //! in case multiple files for reading
 
     // for storage
     Int_t fNObservables;
@@ -87,8 +88,6 @@ class TRestAnalysisTree : public TTree {
         //!< There are branches in the tree with data. There are observables in the list.
         //! Once filled, it is forbidden to add new observable to the tree.
         Filled = 5,
-        //!< first status when constructed from ROOT tree
-        ROOTTree = 6
     };
 
     TRestAnalysisTree(TTree* tree);
@@ -101,13 +100,22 @@ class TRestAnalysisTree : public TTree {
     inline int GetStatus() const { return fStatus; }
     Int_t GetObservableID(const std::string& obsName);
     Bool_t ObservableExists(const std::string& obsName);
-    inline Int_t GetEventID() const { return fEventID; }
-    inline Int_t GetSubEventID() const { return fSubEventID; }
-    inline Double_t GetTimeStamp() const { return fTimeStamp; }
-    inline TString GetSubEventTag() const { return *fSubEventTag; }
-    inline Int_t GetRunOrigin() const { return fRunOrigin; }
-    inline Int_t GetSubRunOrigin() const { return fSubRunOrigin; }
-    inline Int_t GetNumberOfObservables() const { return fNObservables; }
+    // six basic event prameters
+    Int_t GetEventID() { return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetEventID() : fEventID; }
+    Int_t GetSubEventID() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetSubEventID() : fSubEventID;
+    }
+    Double_t GetTimeStamp() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetTimeStamp() : fTimeStamp;
+    }
+    TString GetSubEventTag() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetSubEventTag() : *fSubEventTag;
+    }
+    // we suppose all the chained trees have same run and sub run id.
+    // so there is no need to call fChain->GetTree()
+    Int_t GetRunOrigin() { return fRunOrigin; }
+    Int_t GetSubRunOrigin() { return fSubRunOrigin; }
+    Int_t GetNumberOfObservables() { return fNObservables; }
 
     // observable method
     RESTValue GetObservable(std::string obsName);
@@ -128,6 +136,9 @@ class TRestAnalysisTree : public TTree {
             std::cout << "Error! TRestAnalysisTree::GetObservableValue(): index outside limits!" << endl;
             return T();
         }
+        if (fChain != nullptr) {
+            return ((TRestAnalysisTree*)fChain->GetTree())->GetObservableValue<T>(n);
+        }
         return fObservables[n].GetValue<T>();
     }
     ///////////////////////////////////////////////
@@ -156,6 +167,10 @@ class TRestAnalysisTree : public TTree {
             std::cout << "Error! TRestAnalysisTree::SetObservableValue(): index outside limits!" << endl;
             return;
         }
+        if (fChain != nullptr) {
+            std::cout << "Error! cannot set observable! AnalysisTree is in chain state" << endl;
+            return;
+        }
         fObservables[id].SetValue(value);
     }
     ///////////////////////////////////////////////
@@ -226,6 +241,7 @@ class TRestAnalysisTree : public TTree {
     void SetRunOrigin(Int_t run_origin) { fRunOrigin = run_origin; }
     void SetSubRunOrigin(Int_t sub_run_origin) { fSubRunOrigin = sub_run_origin; }
 
+    void SetEventInfo(TRestAnalysisTree* tree);
     void SetEventInfo(TRestEvent* evt);
     Int_t Fill();
 
@@ -266,6 +282,93 @@ class TRestAnalysisTree : public TTree {
 
     Int_t WriteAsTTree(const char* name = 0, Int_t option = 0, Int_t bufsize = 0);
 
+    Bool_t AddChainFile(std::string file);
+
+    TTree* GetTree() const;
+
+    TChain* GetChain() { return fChain; }
+
+    Long64_t LoadTree(Long64_t entry);
+
+    Long64_t GetEntries() const;
+
+    Long64_t GetEntries(const char* sel);
+
+    void Browse(TBrowser* b) { fChain ? fChain->Browse(b) : TTree::Browse(b); }
+    Long64_t Draw(const char* varexp, const TCut& selection, Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Draw(varexp, selection, option, nentries, firstentry)
+                      : TTree::Draw(varexp, selection, option, nentries, firstentry);
+    }
+    Long64_t Draw(const char* varexp, const char* selection, Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Draw(varexp, selection, option, nentries, firstentry)
+                      : TTree::Draw(varexp, selection, option, nentries, firstentry);
+    }
+    void Draw(Option_t* opt) { fChain ? fChain->Draw(opt) : TTree::Draw(opt); }
+    TBranch* FindBranch(const char* name) {
+        return fChain ? fChain->FindBranch(name) : TTree::FindBranch(name);
+    }
+    TLeaf* FindLeaf(const char* name) { return fChain ? fChain->FindLeaf(name) : TTree::FindLeaf(name); }
+    TBranch* GetBranch(const char* name) { return fChain ? fChain->GetBranch(name) : TTree::GetBranch(name); }
+    Bool_t GetBranchStatus(const char* branchname) const {
+        return fChain ? fChain->GetBranchStatus(branchname) : TTree::GetBranchStatus(branchname);
+    }
+    Long64_t GetCacheSize() const { return fChain ? fChain->GetCacheSize() : TTree::GetCacheSize(); }
+    Long64_t GetChainEntryNumber(Long64_t entry) const {
+        return fChain ? fChain->GetChainEntryNumber(entry) : TTree::GetChainEntryNumber(entry);
+    }
+    Long64_t GetEntryNumber(Long64_t entry) const {
+        return fChain ? fChain->GetEntryNumber(entry) : TTree::GetEntryNumber(entry);
+    }
+    Long64_t GetReadEntry() const { return fChain ? fChain->GetReadEntry() : TTree::GetReadEntry(); }
+
+    TLeaf* GetLeaf(const char* branchname, const char* leafname) {
+        return fChain ? fChain->GetLeaf(branchname, leafname) : TTree::GetLeaf(branchname, leafname);
+    }
+    TLeaf* GetLeaf(const char* name) { return fChain ? fChain->GetLeaf(name) : TTree::GetLeaf(name); }
+    TObjArray* GetListOfBranches() {
+        return fChain ? fChain->GetListOfBranches() : TTree::GetListOfBranches();
+    }
+    TObjArray* GetListOfLeaves() { return fChain ? fChain->GetListOfLeaves() : TTree::GetListOfLeaves(); }
+    Long64_t Process(const char* filename, Option_t* option = "", Long64_t nentries = kMaxEntries,
+                     Long64_t firstentry = 0) {
+        return fChain ? fChain->Process(filename, option, nentries, firstentry)
+                      : TTree::Process(filename, option, nentries, firstentry);
+    }
+    Long64_t Process(TSelector* selector, Option_t* option = "", Long64_t nentries = kMaxEntries,
+                     Long64_t firstentry = 0) {
+        return fChain ? fChain->Process(selector, option, nentries, firstentry)
+                      : TTree::Process(selector, option, nentries, firstentry);
+    }
+    Long64_t Scan(const char* varexp = "", const char* selection = "", Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Scan(varexp, selection, option, nentries, firstentry)
+                      : TTree::Scan(varexp, selection, option, nentries, firstentry);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TBranch** ptr = 0) {
+        return fChain ? fChain->SetBranchAddress(bname, add, ptr) : TTree::SetBranchAddress(bname, add, ptr);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TBranch** ptr, TClass* realClass, EDataType datatype,
+                           Bool_t isptr) {
+        return fChain ? fChain->SetBranchAddress(bname, add, ptr, realClass, datatype, isptr)
+                      : TTree::SetBranchAddress(bname, add, ptr, realClass, datatype, isptr);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TClass* realClass, EDataType datatype,
+                           Bool_t isptr) {
+        return fChain ? fChain->SetBranchAddress(bname, add, realClass, datatype, isptr)
+                      : TTree::SetBranchAddress(bname, add, realClass, datatype, isptr);
+    }
+    void SetBranchStatus(const char* bname, Bool_t status = 1, UInt_t* found = 0) {
+        fChain ? fChain->SetBranchStatus(bname, status, found) : TTree::SetBranchStatus(bname, status, found);
+    }
+    void SetDirectory(TDirectory* dir) { fChain ? fChain->SetDirectory(dir) : TTree::SetDirectory(dir); }
+
+    void ResetBranchAddress(TBranch* br) {
+        fChain ? fChain->ResetBranchAddress(br) : TTree::ResetBranchAddress(br);
+    }
+    void ResetBranchAddresses() { fChain ? fChain->ResetBranchAddresses() : TTree::ResetBranchAddresses(); }
+
     // Constructor
     TRestAnalysisTree();
     TRestAnalysisTree(TString name, TString title);

source/framework/core/inc/TRestEventProcess.h

@@ -196,6 +196,8 @@ class TRestEventProcess : public TRestMetadata {
     void SetFriendProcess(TRestEventProcess* p);
     /// Add parallel process to this process
     void SetParallelProcess(TRestEventProcess* p);
+    /// In case the analysis tree is reset(switched to new file), some process needs to have action
+    virtual void NotifyAnalysisTreeReset() {}
 
     // getters
     /// Get pointer to input event. Must be implemented in the derived class