Implement compression and splitting option in TRestProcessRunner

.gitlab-ci.yml

@@ -7,12 +7,12 @@ stages:
   # Basic checks to verify correctness of code
   - pre-build
 
-  # Build and run tests
-  - test
-
   # Project compilation validation
   - build
 
+  # Build and run tests
+  - test
+
   # REST libraries installed as submodules
   - libraries
 
@@ -78,23 +78,6 @@ Validate Code:
     - python3 pull-submodules.py --force --dontask --latest:${CI_COMMIT_BRANCH}
     - python3 pipeline/validateProcesses.py source/libraries/
 
-Build and Test:
-  stage: test
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - python3 pull-submodules.py --force --dontask --latest:${CI_COMMIT_BRANCH}
-    - mkdir ${CI_PROJECT_DIR}/build && cd ${CI_PROJECT_DIR}/build
-    - cmake ${CI_PROJECT_DIR}
-      -DTEST=ON -DREST_GARFIELD=OFF -DREST_G4=ON -DRESTLIB_DETECTOR=ON -DRESTLIB_RAW=ON -DRESTLIB_TRACK=ON
-    - make -j2
-    - ctest --verbose -O ${CI_PROJECT_DIR}/build/Testing/summary.txt
-
-  artifacts:
-    name: "Testing"
-    paths:
-      - ${CI_PROJECT_DIR}/build/Testing
-    expire_in: 1 day
-
 Build and Install:
   stage: build
   script:
@@ -112,6 +95,24 @@ Build and Install:
       - ${CI_PROJECT_DIR}/install
     expire_in: 1 day
 
+Build and Test:
+  stage: test
+  script:
+    - source ${CI_PROJECT_DIR}/install/thisREST.sh
+    - cd ${CI_PROJECT_DIR}
+    - python3 pull-submodules.py --force --dontask --latest:${CI_COMMIT_BRANCH}
+    - mkdir ${CI_PROJECT_DIR}/build && cd ${CI_PROJECT_DIR}/build
+    - cmake ${CI_PROJECT_DIR}
+      -DTEST=ON -DREST_GARFIELD=OFF -DREST_G4=ON -DRESTLIB_DETECTOR=ON -DRESTLIB_RAW=ON -DRESTLIB_TRACK=ON
+    - make -j2
+    - ctest --verbose -O ${CI_PROJECT_DIR}/build/Testing/summary.txt
+
+  artifacts:
+    name: "Testing"
+    paths:
+      - ${CI_PROJECT_DIR}/build/Testing
+    expire_in: 1 day
+
 Load REST Libraries:
   stage: install
   script:

pipeline/selection/Validate.C

@@ -7,14 +7,16 @@ Int_t Validate(TString fname) {
     TFile* f = new TFile(fname);
     TTree* tr = (TTree*)f->Get("AnalysisTree");
 
-    if (tr->GetEntries() != 3) {
+    tr->Scan("eventID:g4Ana_totalEdep:g4Ana_energyPrimary");
+
+    if (tr->GetEntries() != 1) {
         cout << "Number of entries is not the same!" << endl;
-        cout << "Expected: 3. Obtained: " << tr->GetEntries() << endl;
+        cout << "Expected: 1. Obtained: " << tr->GetEntries() << endl;
         return 1;
     }
 
     // Check IDs.
-    std::vector<Int_t> ids = {0, 3, 6};
+    std::vector<Int_t> ids = {1};
     for (Int_t i = 0; i < tr->GetEntries(); i++) {
         tr->GetEntry(i);
         if (tr->GetLeaf("eventID")->GetValue(0) != ids[i]) {

pipeline/selection/g4OnSelection.rml

@@ -26,7 +26,7 @@
 
     <addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="Run00001_NLDBD_Test_g4Analysis.root" conditions="g4Ana_totalEdep<2400 AND g4Ana_energyPrimary>1000" value="ON"  verboseLevel="info"/>
 
-    <addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="IDs.txt" value="ON"  verboseLevel="info"/>
+    <!--<addProcess type="TRestEventSelectionProcess" name="evSelection" fileWithIDs="IDs.txt" value="ON"  verboseLevel="info"/>-->
 
   </TRestProcessRunner>
 

source/bin/restRoot.cxx

@@ -103,6 +103,36 @@ int main(int argc, char* argv[]) {
                 runTmp->GetEntry(0);
             }
 
+            // command line AnalysisTree object
+            if (runTmp->GetAnalysisTree() != NULL) {
+                // if (runTmp->GetAnalysisTree()->GetChain() != NULL) {
+                //    printf("Attaching ana_tree%i...\n", Nfile);
+                //    string evcmd = Form("TChain* ana_tree%i = (TChain*)%s;", Nfile,
+                //        ToString(runTmp->GetAnalysisTree()->GetChain()).c_str());
+                //    if (debug) printf("%s\n", evcmd.c_str());
+                //    gROOT->ProcessLine(evcmd.c_str());
+                //}
+                // else
+                //{
+                printf("Attaching ana_tree%i...\n", Nfile);
+                string evcmd = Form("TRestAnalysisTree* ana_tree%i = (TRestAnalysisTree*)%s;", Nfile,
+                                    ToString(runTmp->GetAnalysisTree()).c_str());
+                if (debug) printf("%s\n", evcmd.c_str());
+                gROOT->ProcessLine(evcmd.c_str());
+                // runTmp->GetEntry(0);
+                //}
+            }
+
+            // command line EventTree object
+            if (runTmp->GetEventTree() != NULL) {
+                printf("Attaching ev_tree%i...\n", Nfile);
+                string evcmd =
+                    Form("TTree* ev_tree%i = (TTree*)%s;", Nfile, ToString(runTmp->GetEventTree()).c_str());
+                if (debug) printf("%s\n", evcmd.c_str());
+                gROOT->ProcessLine(evcmd.c_str());
+            }
+
+
             printf("\n%s\n", "Attaching metadata structures...");
             Int_t Nmetadata = runTmp->GetNumberOfMetadataStructures();
             map<string, int> metanames;

source/framework/core/inc/TRestAnalysisTree.h

@@ -23,6 +23,7 @@
 #ifndef RestCore_TRestAnalysisTree
 #define RestCore_TRestAnalysisTree
 
+#include "TChain.h"
 #include "TRestEvent.h"
 #include "TRestReflector.h"
 #include "TTree.h"
@@ -44,7 +45,7 @@ class TRestAnalysisTree : public TTree {
     Bool_t fQuickSetObservableValue = true;       //!
     std::vector<RESTValue> fObservables;          //!
     std::map<std::string, int> fObservableIdMap;  //!
-    TTree* fROOTTree;                             //!
+    TChain* fChain = NULL;                        //! in case multiple files for reading
 
     // for storage
     Int_t fNObservables;
@@ -87,8 +88,6 @@ class TRestAnalysisTree : public TTree {
         //!< There are branches in the tree with data. There are observables in the list.
         //! Once filled, it is forbidden to add new observable to the tree.
         Filled = 5,
-        //!< first status when constructed from ROOT tree
-        ROOTTree = 6
     };
 
     TRestAnalysisTree(TTree* tree);
@@ -101,10 +100,19 @@ class TRestAnalysisTree : public TTree {
     int GetStatus() { return fStatus; }
     Int_t GetObservableID(const std::string& obsName);
     Bool_t ObservableExists(const std::string& obsName);
-    Int_t GetEventID() { return fEventID; }
-    Int_t GetSubEventID() { return fSubEventID; }
-    Double_t GetTimeStamp() { return fTimeStamp; }
-    TString GetSubEventTag() { return *fSubEventTag; }
+    // six basic event prameters
+    Int_t GetEventID() { return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetEventID() : fEventID; }
+    Int_t GetSubEventID() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetEventID() : fSubEventID;
+    }
+    Double_t GetTimeStamp() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetEventID() : fTimeStamp;
+    }
+    TString GetSubEventTag() {
+        return fChain ? ((TRestAnalysisTree*)fChain->GetTree())->GetEventID() : *fSubEventTag;
+    }
+    // we suppose all the chained trees have same run and sub run id.
+    // so there is no need to call fChain->GetTree()
     Int_t GetRunOrigin() { return fRunOrigin; }
     Int_t GetSubRunOrigin() { return fSubRunOrigin; }
     Int_t GetNumberOfObservables() { return fNObservables; }
@@ -128,6 +136,9 @@ class TRestAnalysisTree : public TTree {
             std::cout << "Error! TRestAnalysisTree::GetObservableValue(): index outside limits!" << endl;
             return T();
         }
+        if (fChain != NULL) {
+            return ((TRestAnalysisTree*)fChain->GetTree())->GetObservableValue<T>(n);
+        }
         return fObservables[n].GetValue<T>();
     }
     ///////////////////////////////////////////////
@@ -156,6 +167,10 @@ class TRestAnalysisTree : public TTree {
             std::cout << "Error! TRestAnalysisTree::SetObservableValue(): index outside limits!" << endl;
             return;
         }
+        if (fChain != NULL) {
+            std::cout << "Error! cannot set observable! AnalysisTree is in chain state" << endl;
+            return;
+        }
         fObservables[id].SetValue(value);
     }
     ///////////////////////////////////////////////
@@ -266,6 +281,93 @@ class TRestAnalysisTree : public TTree {
 
     Int_t WriteAsTTree(const char* name = 0, Int_t option = 0, Int_t bufsize = 0);
 
+    Bool_t AddChainFile(std::string file);
+
+    TTree* GetTree() const;
+
+    TChain* GetChain() { return fChain; }
+
+    Long64_t LoadTree(Long64_t entry);
+
+    Long64_t GetEntries() const;
+
+    Long64_t GetEntries(const char* sel);
+
+    void Browse(TBrowser* b) { fChain ? fChain->Browse(b) : TTree::Browse(b); }
+    Long64_t Draw(const char* varexp, const TCut& selection, Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Draw(varexp, selection, option, nentries, firstentry)
+                      : TTree::Draw(varexp, selection, option, nentries, firstentry);
+    }
+    Long64_t Draw(const char* varexp, const char* selection, Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Draw(varexp, selection, option, nentries, firstentry)
+                      : TTree::Draw(varexp, selection, option, nentries, firstentry);
+    }
+    void Draw(Option_t* opt) { fChain ? fChain->Draw(opt) : TTree::Draw(opt); }
+    TBranch* FindBranch(const char* name) {
+        return fChain ? fChain->FindBranch(name) : TTree::FindBranch(name);
+    }
+    TLeaf* FindLeaf(const char* name) { return fChain ? fChain->FindLeaf(name) : TTree::FindLeaf(name); }
+    TBranch* GetBranch(const char* name) { return fChain ? fChain->GetBranch(name) : TTree::GetBranch(name); }
+    Bool_t GetBranchStatus(const char* branchname) const {
+        return fChain ? fChain->GetBranchStatus(branchname) : TTree::GetBranchStatus(branchname);
+    }
+    Long64_t GetCacheSize() const { return fChain ? fChain->GetCacheSize() : TTree::GetCacheSize(); }
+    Long64_t GetChainEntryNumber(Long64_t entry) const {
+        return fChain ? fChain->GetChainEntryNumber(entry) : TTree::GetChainEntryNumber(entry);
+    }
+    Long64_t GetEntryNumber(Long64_t entry) const {
+        return fChain ? fChain->GetEntryNumber(entry) : TTree::GetEntryNumber(entry);
+    }
+    Long64_t GetReadEntry() const { return fChain ? fChain->GetReadEntry() : TTree::GetReadEntry(); }
+
+    TLeaf* GetLeaf(const char* branchname, const char* leafname) {
+        return fChain ? fChain->GetLeaf(branchname, leafname) : TTree::GetLeaf(branchname, leafname);
+    }
+    TLeaf* GetLeaf(const char* name) { return fChain ? fChain->GetLeaf(name) : TTree::GetLeaf(name); }
+    TObjArray* GetListOfBranches() {
+        return fChain ? fChain->GetListOfBranches() : TTree::GetListOfBranches();
+    }
+    TObjArray* GetListOfLeaves() { return fChain ? fChain->GetListOfLeaves() : TTree::GetListOfLeaves(); }
+    Long64_t Process(const char* filename, Option_t* option = "", Long64_t nentries = kMaxEntries,
+                     Long64_t firstentry = 0) {
+        return fChain ? fChain->Process(filename, option, nentries, firstentry)
+                      : TTree::Process(filename, option, nentries, firstentry);
+    }
+    Long64_t Process(TSelector* selector, Option_t* option = "", Long64_t nentries = kMaxEntries,
+                     Long64_t firstentry = 0) {
+        return fChain ? fChain->Process(selector, option, nentries, firstentry)
+                      : TTree::Process(selector, option, nentries, firstentry);
+    }
+    Long64_t Scan(const char* varexp = "", const char* selection = "", Option_t* option = "",
+                  Long64_t nentries = kMaxEntries, Long64_t firstentry = 0) {
+        return fChain ? fChain->Scan(varexp, selection, option, nentries, firstentry)
+                      : TTree::Scan(varexp, selection, option, nentries, firstentry);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TBranch** ptr = 0) {
+        return fChain ? fChain->SetBranchAddress(bname, add, ptr) : TTree::SetBranchAddress(bname, add, ptr);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TBranch** ptr, TClass* realClass, EDataType datatype,
+                           Bool_t isptr) {
+        return fChain ? fChain->SetBranchAddress(bname, add, ptr, realClass, datatype, isptr)
+                      : TTree::SetBranchAddress(bname, add, ptr, realClass, datatype, isptr);
+    }
+    Int_t SetBranchAddress(const char* bname, void* add, TClass* realClass, EDataType datatype,
+                           Bool_t isptr) {
+        return fChain ? fChain->SetBranchAddress(bname, add, realClass, datatype, isptr)
+                      : TTree::SetBranchAddress(bname, add, realClass, datatype, isptr);
+    }
+    void SetBranchStatus(const char* bname, Bool_t status = 1, UInt_t* found = 0) {
+        fChain ? fChain->SetBranchStatus(bname, status, found) : TTree::SetBranchStatus(bname, status, found);
+    }
+    void SetDirectory(TDirectory* dir) { fChain ? fChain->SetDirectory(dir) : TTree::SetDirectory(dir); }
+
+    void ResetBranchAddress(TBranch* br) {
+        fChain ? fChain->ResetBranchAddress(br) : TTree::ResetBranchAddress(br);
+    }
+    void ResetBranchAddresses() { fChain ? fChain->ResetBranchAddresses() : TTree::ResetBranchAddresses(); }
+
     // Construtor
     TRestAnalysisTree();
     TRestAnalysisTree(TString name, TString title);

source/framework/core/inc/TRestEventProcess.h

@@ -208,6 +208,8 @@ class TRestEventProcess : public TRestMetadata {
     void SetFriendProcess(TRestEventProcess* p);
     /// Add parallel process to this process
     void SetParallelProcess(TRestEventProcess* p);
+    /// In case the analysis tree is reset(switched to new file), some process needs to have action
+    virtual void NotifyAnalysisTreeReset() {}
 
     // getters
     /// Get pointer to input event. Must be implemented in the derived class

source/framework/core/inc/TRestProcessRunner.h

@@ -36,11 +36,14 @@ class TRestProcessRunner : public TRestMetadata {
 
     // self variables for processing
     std::vector<TRestThread*> fThreads;  //!
-    TFile* fTempOutputDataFile;          //!
-    TTree* fEventTree;                   //!
-    TRestAnalysisTree* fAnalysisTree;    //!
-    ProcStatus fProcStatus;              //!
-    Int_t fNBranches;                    //!
+    TFile* fOutputDataFile;       //! the TFile pointer being used
+    TString fOutputDataFileName;  //! indicates the name of the first file created as output data file. The actual
+                                  //! output file maybe changed if tree is too large
+    TTree* fEventTree;            //!
+    TRestAnalysisTree* fAnalysisTree;  //!
+    ProcStatus fProcStatus;            //!
+    Int_t fNBranches;                  //!
+    Int_t fNFilesSplit;                //! Number of files being split.
 
     // metadata
     Bool_t fUseTestRun;
@@ -56,8 +59,14 @@ class TRestProcessRunner : public TRestMetadata {
     Int_t fFirstEntry;
     Int_t fEventsToProcess;
     Int_t fProcessedEvents;
+
+    Long64_t fFileSplitSize;  // in bytes
+    Int_t fFileCompression;   // 1~9
     std::map<std::string, std::string> fProcessInfo;
 
+    // bool fOutputItem[4] = {
+    //    false};  // the on/off status for item: inputAnalysis, inputEvent, outputEvent, outputAnalysis
+
    public:
     /// REST run class
     void Initialize();
@@ -89,6 +98,8 @@ class TRestProcessRunner : public TRestMetadata {
     Int_t GetNextevtFunc(TRestEvent* targetevt, TRestAnalysisTree* targettree);
     void FillThreadEventFunc(TRestThread* t);
     void ConfigOutputFile();
+    void MergeOutputFile();
+    void WriteMetadata();
 
     // tools
     void ResetRunTimes();
@@ -100,7 +111,7 @@ class TRestProcessRunner : public TRestMetadata {
     TRestEvent* GetInputEvent();
     TRestAnalysisTree* GetInputAnalysisTree();
     TRestAnalysisTree* GetOutputAnalysisTree() { return fAnalysisTree; }
-    TFile* GetTempOutputDataFile() { return fTempOutputDataFile; }
+    TFile* GetOutputDataFile() { return fOutputDataFile; }
     std::string GetProcInfo(std::string infoname) {
         return fProcessInfo[infoname] == "" ? infoname : fProcessInfo[infoname];
     }
@@ -110,12 +121,13 @@ class TRestProcessRunner : public TRestMetadata {
     double GetReadingSpeed();
     bool UseTestRun() const { return fUseTestRun; }
     ProcStatus GetStatus() { return fProcStatus; }
+    Long64_t GetFileSplitSize() { return fFileSplitSize; }
 
     // Constructor & Destructor
     TRestProcessRunner();
     ~TRestProcessRunner();
 
-    ClassDef(TRestProcessRunner, 6);
+    ClassDef(TRestProcessRunner, 7);
 };
 
 #endif