From b641ae599d2e5f4e9d843553da48c5dfee0cfd34 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 21 Sep 2023 10:47:01 +0200 Subject: [PATCH 1/4] improve url checker method --- source/framework/tools/src/TRestTools.cxx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/framework/tools/src/TRestTools.cxx b/source/framework/tools/src/TRestTools.cxx index aa94f34c8..00b1dd58a 100644 --- a/source/framework/tools/src/TRestTools.cxx +++ b/source/framework/tools/src/TRestTools.cxx @@ -49,6 +49,8 @@ #include #include +#include + #ifdef USE_Curl #include #endif @@ -711,11 +713,9 @@ bool TRestTools::isDataSet(const std::string& filename) { /////////////////////////////////////////////// /// \brief Returns true if **filename** is an *http* address. /// -bool TRestTools::isURL(const string& filename) { - if (filename.find("http") == 0) { - return true; - } - return false; +bool TRestTools::isURL(const string& s) { + std::regex pattern("^https?://(.+)"); + return std::regex_match(s, pattern); } /////////////////////////////////////////////// From 9095f13b616fa026be182d2d0e72f11c0873516f Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 21 Sep 2023 11:57:07 +0200 Subject: [PATCH 2/4] working with remote readouts --- source/framework/core/inc/TRestMetadata.h | 2 +- source/framework/core/src/TRestMetadata.cxx | 28 +++++++++++++-------- source/framework/tools/inc/TRestTools.h | 2 +- source/framework/tools/src/TRestTools.cxx | 9 ++++--- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/source/framework/core/inc/TRestMetadata.h b/source/framework/core/inc/TRestMetadata.h index cea8ac1c7..3a4963a9f 100644 --- a/source/framework/core/inc/TRestMetadata.h +++ b/source/framework/core/inc/TRestMetadata.h @@ -72,7 +72,7 @@ class TRestMetadata : public TNamed { void ReadEnvInElement(TiXmlElement* e, bool overwrite = true); void ReadElement(TiXmlElement* e, bool recursive = false); void ReplaceForLoopVars(TiXmlElement* e, std::map forLoopVar); - void ExpandForLoopOnce(TiXmlElement* e, std::map forLoopVar); + void ExpandForLoopOnce(TiXmlElement* e, const std::map& forLoopVar); void ExpandForLoops(TiXmlElement* e, std::map forLoopVar); void ExpandIfSections(TiXmlElement* e); void ExpandIncludeFile(TiXmlElement* e); diff --git a/source/framework/core/src/TRestMetadata.cxx b/source/framework/core/src/TRestMetadata.cxx index 82f31fa72..b1537f3e0 100644 --- a/source/framework/core/src/TRestMetadata.cxx +++ b/source/framework/core/src/TRestMetadata.cxx @@ -1065,8 +1065,10 @@ void TRestMetadata::ExpandIfSections(TiXmlElement* e) { /////////////////////////////////////////////// /// \brief Helper method for TRestMetadata::ExpandForLoops(). -void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, map forLoopVar) { - if (e == nullptr) return; +void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, const map& forLoopVar) { + if (e == nullptr) { + return; + } TiXmlElement* parele = (TiXmlElement*)e->Parent(); TiXmlElement* contentelement = e->FirstChildElement(); @@ -1098,7 +1100,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map forL if (e == nullptr) return; RESTDebug << "Entering ... TRestMetadata::ReplaceForLoopVars" << RESTendl; - std::string parName = ""; + std::string parName; TiXmlAttribute* attr = e->FirstAttribute(); while (attr != nullptr) { const char* val = attr->Value(); @@ -1111,7 +1113,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map forL if (strcmp(name, "name") != 0) { string outputBuffer = val; - if (outputBuffer.find("[") != string::npos || outputBuffer.find("]") != string::npos) { + if (outputBuffer.find('[') != string::npos || outputBuffer.find(']') != string::npos) { RESTError << "TRestMetadata::ReplaceForLoopVars. Old for-loop construction identified" << RESTendl; RESTError << "Please, replace [] variable nomenclature by ${}." << RESTendl; @@ -1134,7 +1136,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map forL string proenv = forLoopVar.count(expression) > 0 ? forLoopVar[expression] : ""; - if (proenv != "") { + if (!proenv.empty()) { outputBuffer.replace(replacePos, replaceLen, proenv); endPosition = 0; } else { @@ -1203,7 +1205,7 @@ void TRestMetadata::ExpandForLoops(TiXmlElement* e, map forloopv if (fVerboseLevel >= TRestStringOutput::REST_Verbose_Level::REST_Extreme) parele->Print(stdout, 0); RESTDebug << "----end of for loop----" << RESTendl; - } else if (_in != "") { + } else if (!_in.empty()) { vector loopvars = Split(_in, ":"); RESTDebug << "----expanding for loop----" << RESTendl; @@ -1250,18 +1252,22 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) { if (_filename == nullptr) return; string filename; - if (string(_filename) == "server") { + if (string(_filename) == "server" || TRestTools::isURL(_filename)) { // Let TRestRun to retrieve data according to run number later-on - // if ((string) this->ClassName() == "TRestRun") return; // match the database, runNumber=0(default data), type="META_RML", tag=
auto url = gDataBase->query_data(DBEntry(0, "META_RML", e->Value())).value; + if (url.empty()) { + // don't really understand this "database" code, this just works + url = _filename; + } + filename = TRestTools::DownloadRemoteFile(url); } else { filename = SearchFile(_filename); - } + }l - if (filename == "") { + if (filename.empty()) { RESTError << "TRestMetadata::ExpandIncludeFile. Include file \"" << _filename << "\" does not exist!" << RESTendl; exit(1); @@ -1369,7 +1375,7 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) { } ele = ele->NextSiblingElement(); } - // more than 1 elements found + // more than 1 element found if (eles.size() > 1) { RESTWarning << "(expand include file): find multiple xml sections with same name!" << RESTendl; diff --git a/source/framework/tools/inc/TRestTools.h b/source/framework/tools/inc/TRestTools.h index df1cf4360..58836b4bf 100644 --- a/source/framework/tools/inc/TRestTools.h +++ b/source/framework/tools/inc/TRestTools.h @@ -125,7 +125,7 @@ class TRestTools { static std::string Execute(std::string cmd); - static std::string DownloadRemoteFile(std::string remoteFile); + static std::string DownloadRemoteFile(const std::string& remoteFile); static int DownloadRemoteFile(std::string remoteFile, std::string localFile); static int UploadToServer(std::string localFile, std::string remoteFile, std::string methodUrl = ""); diff --git a/source/framework/tools/src/TRestTools.cxx b/source/framework/tools/src/TRestTools.cxx index 00b1dd58a..435d0b5a7 100644 --- a/source/framework/tools/src/TRestTools.cxx +++ b/source/framework/tools/src/TRestTools.cxx @@ -1049,9 +1049,10 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) { /// will be used, including scp, wget. Downloads to REST_USER_PATH + "/download/" + filename /// by default. /// -std::string TRestTools::DownloadRemoteFile(string url) { - string purename = TRestTools::GetPureFileName(url); - if (purename == "") { +std::string TRestTools::DownloadRemoteFile(const string& url) { + cout << "TRestTools::DownloadRemoteFile: " << url << endl; + string pureName = TRestTools::GetPureFileName(url); + if (pureName.empty()) { cout << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << endl; cout << "please specify a concrete file name in this url" << endl; cout << "url: " << url << endl; @@ -1061,7 +1062,7 @@ std::string TRestTools::DownloadRemoteFile(string url) { if (url.find("local:") == 0) { return Replace(url, "local:", ""); } else { - string fullpath = REST_USER_PATH + "/download/" + purename; + string fullpath = REST_USER_PATH + "/download/" + pureName; int out; int attempts = 10; do { From 4e0cee21c76519b791d04aac52880362845101bb Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 21 Sep 2023 12:12:07 +0200 Subject: [PATCH 3/4] remote files --- source/framework/core/src/TRestMetadata.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/framework/core/src/TRestMetadata.cxx b/source/framework/core/src/TRestMetadata.cxx index b1537f3e0..43f923a11 100644 --- a/source/framework/core/src/TRestMetadata.cxx +++ b/source/framework/core/src/TRestMetadata.cxx @@ -1209,7 +1209,7 @@ void TRestMetadata::ExpandForLoops(TiXmlElement* e, map forloopv vector loopvars = Split(_in, ":"); RESTDebug << "----expanding for loop----" << RESTendl; - for (string loopvar : loopvars) { + for (const string& loopvar : loopvars) { forloopvar[_name] = loopvar; fVariables[_name] = loopvar; ExpandForLoopOnce(e, forloopvar); @@ -1265,7 +1265,7 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) { filename = TRestTools::DownloadRemoteFile(url); } else { filename = SearchFile(_filename); - }l + } if (filename.empty()) { RESTError << "TRestMetadata::ExpandIncludeFile. Include file \"" << _filename << "\" does not exist!" From 1f7d08adc583e656d8a50d290794f6f3eb7893b7 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 21 Sep 2023 12:39:30 +0200 Subject: [PATCH 4/4] warnings --- source/framework/tools/src/TRestTools.cxx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/framework/tools/src/TRestTools.cxx b/source/framework/tools/src/TRestTools.cxx index 435d0b5a7..0af3237fa 100644 --- a/source/framework/tools/src/TRestTools.cxx +++ b/source/framework/tools/src/TRestTools.cxx @@ -1050,12 +1050,11 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) { /// by default. /// std::string TRestTools::DownloadRemoteFile(const string& url) { - cout << "TRestTools::DownloadRemoteFile: " << url << endl; string pureName = TRestTools::GetPureFileName(url); if (pureName.empty()) { - cout << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << endl; - cout << "please specify a concrete file name in this url" << endl; - cout << "url: " << url << endl; + RESTWarning << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << RESTendl; + RESTWarning << "please specify a concrete file name in this url" << RESTendl; + RESTWarning << "url: " << url << RESTendl; return ""; } @@ -1071,14 +1070,12 @@ std::string TRestTools::DownloadRemoteFile(const string& url) { RESTWarning << "Retrying download in 5 seconds" << RESTendl; std::this_thread::sleep_for(std::chrono::seconds(5)); } else if (attempts < 10) { - RESTSuccess << "Download suceeded after " << 10 - attempts << " attempts" << RESTendl; + RESTSuccess << "Download succeeded after " << 10 - attempts << " attempts" << RESTendl; } attempts--; } while (out == 1024 && attempts > 0); - if (out == 0) { - return fullpath; - } else if (TRestTools::fileExists(fullpath)) { + if (out == 0 || TRestTools::fileExists(fullpath)) { return fullpath; } else { return "";