Adjust metadata writing sequence to avoid StreamerInfo missing

pipeline/trex/validateStreamer.py

@@ -1,5 +1,9 @@
 import os, sys
 
+os.system(
+    "restRoot -b -q StreamerOutput.C'(\"Hits_01928.root\")' | grep Process | grep TRest | grep version"
+)
+
 os.system(
     "restRoot -b -q StreamerOutput.C'(\"Hits_01928.root\")' | grep Process | grep TRest | grep version | wc -l > output.log 2>&1"
 )
@@ -8,11 +12,12 @@
     lines = f.readlines()
 
 for line in lines:
-    if line.find("9") == 0:
-        print("The number of processes inside the event data chain is 9. Succeed!")
+    if line.find("11") == 0:
+        print("The number of processes inside the event data chain is 11. Succeed!")
         sys.exit(0)
     else:
-        print("The number of processes inside the event data chain is NOT 6! Fail!")
+        print("The number of processes inside the event data chain is NOT 11! Fail!")
+        print("The number of processes is: " + line + "\n")
         sys.exit(1)
 
 sys.exit(0)

source/framework/core/src/TRestManager.cxx

@@ -37,16 +37,28 @@ ClassImp(TRestManager);
 
 TRestManager::TRestManager() { Initialize(); }
 
-TRestManager::~TRestManager() {}
+TRestManager::~TRestManager() {
+    // delete all the added metadata objects(besides self)
+    if (fMetaObjects.size() > 1) {
+        for (unsigned int i = fMetaObjects.size() - 1; i >= 1; i--) {
+            delete fMetaObjects[i];
+        }
+    }
+}
 
 ///////////////////////////////////////////////
 /// \brief Set the class name as section name during initialization.
 ///
 void TRestManager::Initialize() {
     SetSectionName(this->ClassName());
-
+    // delete all the added metadata objects(besides self)
+    if (fMetaObjects.size() > 1) {
+        for (unsigned int i = fMetaObjects.size() - 1; i >= 1; i--) {
+            delete fMetaObjects[i];
+        }
+    }
+    // add self to the metadata objects list
     fMetaObjects.clear();
-
     fMetaObjects.push_back(this);
 }
 

source/framework/core/src/TRestProcessRunner.cxx

@@ -330,7 +330,7 @@ void TRestProcessRunner::ReadProcInfo() {
 /// detatched after this calling.
 /// 6. The main thread loops for progress bar while waiting the child threads to
 /// finish.
-/// 7. After finished, print some information, reset ROOT mutes.
+/// 7. After finished, print some information, reset ROOT mutexs.
 /// 8. Call ConfigOutputFile() method to save the output file
 ///
 void TRestProcessRunner::RunProcess() {
@@ -408,7 +408,9 @@ void TRestProcessRunner::RunProcess() {
         exit(1);
     }
 
-    ConfigOutputFile();
+    fOutputDataFile->cd();
+    if (fEventTree != nullptr) fEventTree->Write(nullptr, kOverwrite);
+    if (fAnalysisTree != nullptr) fAnalysisTree->Write(nullptr, kOverwrite);
 
     // reset runner
     this->ResetRunTimes();
@@ -526,7 +528,6 @@ void TRestProcessRunner::RunProcess() {
 
     if (fRunInfo->GetOutputFileName() != "/dev/null") {
         ConfigOutputFile();
-        MergeOutputFile();
     }
 }
 
@@ -937,39 +938,48 @@ void TRestProcessRunner::FillThreadEventFunc(TRestThread* t) {
 ///////////////////////////////////////////////
 /// \brief Forming an output file
 ///
-/// It first saves process metadata in to the main output file, then calls
-/// TRestRun::FormOutputFile() to merge the main file with process's tmp file.
+/// Merge all the process's file together, write TRestRun metadata,
+/// and then save process metadata to that file
 void TRestProcessRunner::ConfigOutputFile() {
     RESTEssential << "Configuring output file, writing metadata and tree objects" << RESTendl;
 #ifdef TIME_MEASUREMENT
     fProcessInfo["ProcessTime"] = ToString(deltaTime) + "ms";
 #endif
-    // write tree
+    // write the last tree
     fOutputDataFile->cd();
     if (fEventTree != nullptr) fEventTree->Write(nullptr, kOverwrite);
     if (fAnalysisTree != nullptr) fAnalysisTree->Write(nullptr, kOverwrite);
 
-    // go back to the first file
-    if (fOutputDataFile->GetName() != fOutputDataFileName) {
-        delete fOutputDataFile;
-        fOutputDataFile = new TFile(fOutputDataFileName, "update");
-    }
+    // close file
+    fOutputDataFile->Write();
+    fOutputDataFile->Close();
+    delete fOutputDataFile;
+
+    // merge process's data file to the main file
+    // we must call this method before writing process metadata,
+    // otherwise there would be problems of streamer missing
+    // https://github.com/rest-for-physics/framework/issues/348
+    fRunInfo->SetNFilesSplit(fNFilesSplit);
+    MergeOutputFile();
+
     // write metadata
     WriteProcessesMetadata();
 }
 
+///////////////////////////////////////////////
+/// \brief Write process metadata to fOutputDataFile
+///
 void TRestProcessRunner::WriteProcessesMetadata() {
-    if (fRunInfo->GetInputFile() == nullptr) {
-        if (!fRunInfo->GetOutputFile()) {
-            // We are not ready yet to write
-            return;
-        }
-        fRunInfo->cd();
-    } else
-        fOutputDataFile->cd();
+    // if (fRunInfo->GetInputFile() == nullptr) {
+    //     if (!fRunInfo->GetOutputFile()) {
+    //         // We are not ready yet to write
+    //         return;
+    //     }
+    //     fRunInfo->cd();
+    // } else
+    //
+    fOutputDataFile->cd();
 
-    fRunInfo->SetNFilesSplit(fNFilesSplit);
-    fRunInfo->Write(nullptr, TObject::kOverwrite);
     this->Write(nullptr, TObject::kWriteDelete);
 
     if (fRunInfo->GetFileProcess() != nullptr) {
@@ -983,10 +993,9 @@ void TRestProcessRunner::WriteProcessesMetadata() {
 }
 
 ///////////////////////////////////////////////
-/// \brief Forming an output file
+/// \brief Calls TRestRun::MergeOutputFile() to merge the main file with process's tmp file.
 ///
-/// It first saves process metadata in to the main output file, then calls
-/// TRestRun::FormOutputFile() to merge the main file with process's tmp file.
+/// After this operation, fOutputDataFile will be set to TRestRun's output file
 void TRestProcessRunner::MergeOutputFile() {
     RESTEssential << "Merging thread files together" << RESTendl;
     // add threads file
@@ -1003,11 +1012,13 @@ void TRestProcessRunner::MergeOutputFile() {
         files_to_merge.push_back(f->GetName());
     }
 
-    fOutputDataFile->cd();
-    fOutputDataFile->Write(nullptr, TObject::kOverwrite);
-    fOutputDataFile->Close();
-    fRunInfo->MergeToOutputFile(files_to_merge, fOutputDataFile->GetName());
-    if (fRunInfo->GetInputFile() == nullptr) WriteProcessesMetadata();
+    if (TRestTools::fileExists((string)fOutputDataFileName)) {
+        fOutputDataFile = fRunInfo->MergeToOutputFile(files_to_merge, (string)fOutputDataFileName);
+    } else {
+        RESTError << "Output file: " << fOutputDataFileName << " is lost?" << RESTendl;
+    }
+    // if (fRunInfo->GetInputFile() == nullptr)
+    // WriteProcessesMetadata();
 }
 
 // tools

source/framework/core/src/TRestRun.cxx

@@ -991,7 +991,7 @@ TFile* TRestRun::MergeToOutputFile(vector<string> filenames, string outputfilena
         m->OutputFile(filename.c_str(), "RECREATE");
     } else {
         filename = outputfilename;
-        RESTInfo << "Creating file : " << filename << RESTendl;
+        RESTInfo << "Updating file : " << filename << RESTendl;
         m->OutputFile(filename.c_str(), "UPDATE");
     }