From 21499eb5f144a5195e826c2ac9aeb87a69bb85bd Mon Sep 17 00:00:00 2001 From: juanan Date: Thu, 5 Oct 2023 19:38:56 +0200 Subject: [PATCH 1/5] Implementation of merging of metadata while importing TRestDataSet --- source/framework/core/inc/TRestDataSet.h | 1 + source/framework/core/src/TRestDataSet.cxx | 89 +++++++++++++++++----- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/source/framework/core/inc/TRestDataSet.h b/source/framework/core/inc/TRestDataSet.h index 94f0a490d..aa36c8696 100644 --- a/source/framework/core/inc/TRestDataSet.h +++ b/source/framework/core/inc/TRestDataSet.h @@ -169,6 +169,7 @@ class TRestDataSet : public TRestMetadata { inline void SetQuantity(const std::map& quantity) { fQuantity = quantity; } TRestDataSet& operator=(TRestDataSet& dS); + Bool_t Merge(TRestDataSet& dS); void Import(const std::string& fileName); void Import(std::vector fileNames); void Export(const std::string& filename); diff --git a/source/framework/core/src/TRestDataSet.cxx b/source/framework/core/src/TRestDataSet.cxx index 4e7d0aa56..f12759110 100644 --- a/source/framework/core/src/TRestDataSet.cxx +++ b/source/framework/core/src/TRestDataSet.cxx @@ -900,6 +900,36 @@ TRestDataSet& TRestDataSet::operator=(TRestDataSet& dS) { return *this; } +/////////////////////////////////////////////// +/// \brief This function merge different TRestDataSet +/// metadata in current dataSet +/// +Bool_t TRestDataSet::Merge(TRestDataSet& dS) { + + auto obsNames = GetObservablesList(); + for (const auto& obs : fObservablesList) { + if (std::find(obsNames.begin(), obsNames.end(), obs) != obsNames.end()) { + RESTError<<"Cannot merge dataSets with different observable list "< REST_StringHelper::StringToTimeStamp(dS.GetFilterStartTime()) )fFilterStartTime = dS.GetFilterStartTime(); + if(REST_StringHelper::StringToTimeStamp(fFilterEndTime) < REST_StringHelper::StringToTimeStamp(dS.GetFilterEndTime()) )fFilterEndTime = dS.GetFilterEndTime(); + if (fStartTime > dS.GetStartTime())fStartTime = dS.GetStartTime(); + if (fEndTime < dS.GetEndTime())fEndTime = dS.GetEndTime(); + + + auto fileSelection = dS.GetFileSelection(); + fFileSelection.insert(fFileSelection.end(), fileSelection.begin(), fileSelection.end()); + + fProcessObservablesList = dS.GetProcessObservablesList(); + + fTotalDuration += dS.GetTotalTimeInSeconds(); + + return true; +} + /////////////////////////////////////////////// /// \brief This function imports metadata from a root file /// it import metadata info from the previous dataSet @@ -956,25 +986,48 @@ void TRestDataSet::Import(std::vector fileNames) { return; } - if (fileNames.size() == 0) return; + int count = 0; + auto it = fileNames.begin(); + while (it != fileNames.end()){ + std::string fileName = *it; + TFile* file = TFile::Open(fileName.c_str(), "READ"); + bool isValid = false; + if (file != nullptr) { + TIter nextkey(file->GetListOfKeys()); + TKey* key; + while ((key = (TKey*)nextkey())) { + std::string kName = key->GetClassName(); + if (REST_Reflection::GetClassQuick(kName.c_str()) != nullptr && + REST_Reflection::GetClassQuick(kName.c_str())->InheritsFrom("TRestDataSet")) { + TRestDataSet* dS = file->Get(key->GetName()); + if (GetVerboseLevel() >= TRestStringOutput::REST_Verbose_Level::REST_Info) + dS->PrintMetadata(); + + if(count == 0){ + *this = *dS; + isValid = true; + } else { + isValid = Merge(*dS); + } + + if(isValid)count++; + } + } + } else { + RESTError << "Cannot open " << fileName << RESTendl; + } + + if (!isValid){ + RESTError << fileName << " is not a valid dataSet skipping..."<GetListOfKeys()); - TKey* key; - while ((key = (TKey*)nextkey())) { - std::string kName = key->GetClassName(); - if (REST_Reflection::GetClassQuick(kName.c_str()) != nullptr && - REST_Reflection::GetClassQuick(kName.c_str())->InheritsFrom("TRestDataSet")) { - TRestDataSet* dS = file->Get(key->GetName()); - if (GetVerboseLevel() >= TRestStringOutput::REST_Verbose_Level::REST_Info) - dS->PrintMetadata(); - *this = *dS; - } - } - } else { - RESTError << "Cannot open " << fileNames[0] << RESTendl; - exit(1); + if (fileNames.empty()) { + RESTError << "File selection is empty, dataSet will not be imported " << RESTendl; + return; } RESTInfo << "Opening list of files. First file: " << fileNames[0] << RESTendl; From 326038c56a7cec874e55eb2f95b3e0e954036e59 Mon Sep 17 00:00:00 2001 From: juanan Date: Thu, 5 Oct 2023 19:39:14 +0200 Subject: [PATCH 2/5] Removal of duplicated line --- source/framework/core/src/TRestDataSet.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/source/framework/core/src/TRestDataSet.cxx b/source/framework/core/src/TRestDataSet.cxx index f12759110..eb3304eac 100644 --- a/source/framework/core/src/TRestDataSet.cxx +++ b/source/framework/core/src/TRestDataSet.cxx @@ -894,7 +894,6 @@ TRestDataSet& TRestDataSet::operator=(TRestDataSet& dS) { fFilterEqualsTo = dS.GetFilterEqualsTo(); fQuantity = dS.GetQuantity(); fTotalDuration = dS.GetTotalTimeInSeconds(); - fFileSelection = dS.GetFileSelection(); fCut = dS.GetCut(); return *this; From 75a76bfe78a9885d35ffdbe9a35b0f5b531d6b0a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 17:41:00 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- source/framework/core/src/TRestDataSet.cxx | 94 +++++++++++----------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/source/framework/core/src/TRestDataSet.cxx b/source/framework/core/src/TRestDataSet.cxx index eb3304eac..412fdb156 100644 --- a/source/framework/core/src/TRestDataSet.cxx +++ b/source/framework/core/src/TRestDataSet.cxx @@ -904,20 +904,22 @@ TRestDataSet& TRestDataSet::operator=(TRestDataSet& dS) { /// metadata in current dataSet /// Bool_t TRestDataSet::Merge(TRestDataSet& dS) { - auto obsNames = GetObservablesList(); - for (const auto& obs : fObservablesList) { + for (const auto& obs : fObservablesList) { if (std::find(obsNames.begin(), obsNames.end(), obs) != obsNames.end()) { - RESTError<<"Cannot merge dataSets with different observable list "< REST_StringHelper::StringToTimeStamp(dS.GetFilterStartTime()) )fFilterStartTime = dS.GetFilterStartTime(); - if(REST_StringHelper::StringToTimeStamp(fFilterEndTime) < REST_StringHelper::StringToTimeStamp(dS.GetFilterEndTime()) )fFilterEndTime = dS.GetFilterEndTime(); - if (fStartTime > dS.GetStartTime())fStartTime = dS.GetStartTime(); - if (fEndTime < dS.GetEndTime())fEndTime = dS.GetEndTime(); + } + if (REST_StringHelper::StringToTimeStamp(fFilterStartTime) > + REST_StringHelper::StringToTimeStamp(dS.GetFilterStartTime())) + fFilterStartTime = dS.GetFilterStartTime(); + if (REST_StringHelper::StringToTimeStamp(fFilterEndTime) < + REST_StringHelper::StringToTimeStamp(dS.GetFilterEndTime())) + fFilterEndTime = dS.GetFilterEndTime(); + if (fStartTime > dS.GetStartTime()) fStartTime = dS.GetStartTime(); + if (fEndTime < dS.GetEndTime()) fEndTime = dS.GetEndTime(); auto fileSelection = dS.GetFileSelection(); fFileSelection.insert(fFileSelection.end(), fileSelection.begin(), fileSelection.end()); @@ -987,46 +989,46 @@ void TRestDataSet::Import(std::vector fileNames) { int count = 0; auto it = fileNames.begin(); - while (it != fileNames.end()){ - std::string fileName = *it; - TFile* file = TFile::Open(fileName.c_str(), "READ"); - bool isValid = false; - if (file != nullptr) { - TIter nextkey(file->GetListOfKeys()); - TKey* key; - while ((key = (TKey*)nextkey())) { - std::string kName = key->GetClassName(); - if (REST_Reflection::GetClassQuick(kName.c_str()) != nullptr && - REST_Reflection::GetClassQuick(kName.c_str())->InheritsFrom("TRestDataSet")) { - TRestDataSet* dS = file->Get(key->GetName()); - if (GetVerboseLevel() >= TRestStringOutput::REST_Verbose_Level::REST_Info) - dS->PrintMetadata(); - - if(count == 0){ - *this = *dS; - isValid = true; - } else { - isValid = Merge(*dS); - } - - if(isValid)count++; - } - } - } else { - RESTError << "Cannot open " << fileName << RESTendl; - } - - if (!isValid){ - RESTError << fileName << " is not a valid dataSet skipping..."<GetListOfKeys()); + TKey* key; + while ((key = (TKey*)nextkey())) { + std::string kName = key->GetClassName(); + if (REST_Reflection::GetClassQuick(kName.c_str()) != nullptr && + REST_Reflection::GetClassQuick(kName.c_str())->InheritsFrom("TRestDataSet")) { + TRestDataSet* dS = file->Get(key->GetName()); + if (GetVerboseLevel() >= TRestStringOutput::REST_Verbose_Level::REST_Info) + dS->PrintMetadata(); + + if (count == 0) { + *this = *dS; + isValid = true; + } else { + isValid = Merge(*dS); + } + + if (isValid) count++; + } + } + } else { + RESTError << "Cannot open " << fileName << RESTendl; + } + + if (!isValid) { + RESTError << fileName << " is not a valid dataSet skipping..." << RESTendl; + it = fileNames.erase(it); + } else { + ++it; + } } if (fileNames.empty()) { - RESTError << "File selection is empty, dataSet will not be imported " << RESTendl; - return; + RESTError << "File selection is empty, dataSet will not be imported " << RESTendl; + return; } RESTInfo << "Opening list of files. First file: " << fileNames[0] << RESTendl; From eb131c14134508e22e57f8bc2aeea398108390a0 Mon Sep 17 00:00:00 2001 From: juanan Date: Mon, 9 Oct 2023 17:25:10 +0200 Subject: [PATCH 4/5] Added const to Merge funcion args --- source/framework/core/inc/TRestDataSet.h | 2 +- source/framework/core/src/TRestDataSet.cxx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/framework/core/inc/TRestDataSet.h b/source/framework/core/inc/TRestDataSet.h index aa36c8696..0f6a6304b 100644 --- a/source/framework/core/inc/TRestDataSet.h +++ b/source/framework/core/inc/TRestDataSet.h @@ -169,7 +169,7 @@ class TRestDataSet : public TRestMetadata { inline void SetQuantity(const std::map& quantity) { fQuantity = quantity; } TRestDataSet& operator=(TRestDataSet& dS); - Bool_t Merge(TRestDataSet& dS); + Bool_t Merge(const TRestDataSet& dS); void Import(const std::string& fileName); void Import(std::vector fileNames); void Export(const std::string& filename); diff --git a/source/framework/core/src/TRestDataSet.cxx b/source/framework/core/src/TRestDataSet.cxx index 412fdb156..3ac437e2d 100644 --- a/source/framework/core/src/TRestDataSet.cxx +++ b/source/framework/core/src/TRestDataSet.cxx @@ -903,7 +903,7 @@ TRestDataSet& TRestDataSet::operator=(TRestDataSet& dS) { /// \brief This function merge different TRestDataSet /// metadata in current dataSet /// -Bool_t TRestDataSet::Merge(TRestDataSet& dS) { +Bool_t TRestDataSet::Merge(const TRestDataSet& dS) { auto obsNames = GetObservablesList(); for (const auto& obs : fObservablesList) { if (std::find(obsNames.begin(), obsNames.end(), obs) != obsNames.end()) { From 8c48a9894fd7c823722c5e492ae3c04c384e6b73 Mon Sep 17 00:00:00 2001 From: juanan Date: Mon, 9 Oct 2023 17:26:05 +0200 Subject: [PATCH 5/5] Removing filter timestamp and processObservableList at Merge --- source/framework/core/src/TRestDataSet.cxx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/framework/core/src/TRestDataSet.cxx b/source/framework/core/src/TRestDataSet.cxx index 3ac437e2d..3b3da25bf 100644 --- a/source/framework/core/src/TRestDataSet.cxx +++ b/source/framework/core/src/TRestDataSet.cxx @@ -912,20 +912,12 @@ Bool_t TRestDataSet::Merge(const TRestDataSet& dS) { } } - if (REST_StringHelper::StringToTimeStamp(fFilterStartTime) > - REST_StringHelper::StringToTimeStamp(dS.GetFilterStartTime())) - fFilterStartTime = dS.GetFilterStartTime(); - if (REST_StringHelper::StringToTimeStamp(fFilterEndTime) < - REST_StringHelper::StringToTimeStamp(dS.GetFilterEndTime())) - fFilterEndTime = dS.GetFilterEndTime(); if (fStartTime > dS.GetStartTime()) fStartTime = dS.GetStartTime(); if (fEndTime < dS.GetEndTime()) fEndTime = dS.GetEndTime(); auto fileSelection = dS.GetFileSelection(); fFileSelection.insert(fFileSelection.end(), fileSelection.begin(), fileSelection.end()); - fProcessObservablesList = dS.GetProcessObservablesList(); - fTotalDuration += dS.GetTotalTimeInSeconds(); return true;