Skip to content

Commit

Permalink
Merge pull request #480 from rest-for-physics/479-support-downloading…
Browse files Browse the repository at this point in the history 
…-files-over-http-at-addmetadata-definition

Improve support for remote metadata files
  • Loading branch information
@lobis
lobis authored Sep 21, 2023
2 parents c947149 + 1f7d08a commit 200e8f2
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 29 deletions.
2 changes: 1 addition & 1 deletion source/framework/core/inc/TRestMetadata.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class TRestMetadata : public TNamed {
void ReadEnvInElement(TiXmlElement* e, bool overwrite = true);
void ReadElement(TiXmlElement* e, bool recursive = false);
void ReplaceForLoopVars(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandForLoopOnce(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandForLoopOnce(TiXmlElement* e, const std::map<std::string, std::string>& forLoopVar);
void ExpandForLoops(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandIfSections(TiXmlElement* e);
void ExpandIncludeFile(TiXmlElement* e);
Expand Down
28 changes: 17 additions & 11 deletions source/framework/core/src/TRestMetadata.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1065,8 +1065,10 @@ void TRestMetadata::ExpandIfSections(TiXmlElement* e) {

///////////////////////////////////////////////
/// \brief Helper method for TRestMetadata::ExpandForLoops().
void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, map<string, string> forLoopVar) {
if (e == nullptr) return;
void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, const map<string, string>& forLoopVar) {
if (e == nullptr) {
return;
}

TiXmlElement* parele = (TiXmlElement*)e->Parent();
TiXmlElement* contentelement = e->FirstChildElement();
Expand Down Expand Up @@ -1098,7 +1100,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL
if (e == nullptr) return;

RESTDebug << "Entering ... TRestMetadata::ReplaceForLoopVars" << RESTendl;
std::string parName = "";
std::string parName;
TiXmlAttribute* attr = e->FirstAttribute();
while (attr != nullptr) {
const char* val = attr->Value();
Expand All @@ -1111,7 +1113,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL
if (strcmp(name, "name") != 0) {
string outputBuffer = val;

if (outputBuffer.find("[") != string::npos || outputBuffer.find("]") != string::npos) {
if (outputBuffer.find('[') != string::npos || outputBuffer.find(']') != string::npos) {
RESTError << "TRestMetadata::ReplaceForLoopVars. Old for-loop construction identified"
<< RESTendl;
RESTError << "Please, replace [] variable nomenclature by ${}." << RESTendl;
Expand All @@ -1134,7 +1136,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL

string proenv = forLoopVar.count(expression) > 0 ? forLoopVar[expression] : "";

if (proenv != "") {
if (!proenv.empty()) {
outputBuffer.replace(replacePos, replaceLen, proenv);
endPosition = 0;
} else {
Expand Down Expand Up @@ -1203,11 +1205,11 @@ void TRestMetadata::ExpandForLoops(TiXmlElement* e, map<string, string> forloopv

if (fVerboseLevel >= TRestStringOutput::REST_Verbose_Level::REST_Extreme) parele->Print(stdout, 0);
RESTDebug << "----end of for loop----" << RESTendl;
} else if (_in != "") {
} else if (!_in.empty()) {
vector<string> loopvars = Split(_in, ":");

RESTDebug << "----expanding for loop----" << RESTendl;
for (string loopvar : loopvars) {
for (const string& loopvar : loopvars) {
forloopvar[_name] = loopvar;
fVariables[_name] = loopvar;
ExpandForLoopOnce(e, forloopvar);
Expand Down Expand Up @@ -1250,18 +1252,22 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) {
if (_filename == nullptr) return;

string filename;
if (string(_filename) == "server") {
if (string(_filename) == "server" || TRestTools::isURL(_filename)) {
// Let TRestRun to retrieve data according to run number later-on
// if ((string) this->ClassName() == "TRestRun") return;

// match the database, runNumber=0(default data), type="META_RML", tag=<section name>
auto url = gDataBase->query_data(DBEntry(0, "META_RML", e->Value())).value;
if (url.empty()) {
// don't really understand this "database" code, this just works
url = _filename;
}

filename = TRestTools::DownloadRemoteFile(url);
} else {
filename = SearchFile(_filename);
}

if (filename == "") {
if (filename.empty()) {
RESTError << "TRestMetadata::ExpandIncludeFile. Include file \"" << _filename << "\" does not exist!"
<< RESTendl;
exit(1);
Expand Down Expand Up @@ -1369,7 +1375,7 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) {
}
ele = ele->NextSiblingElement();
}
// more than 1 elements found
// more than 1 element found
if (eles.size() > 1) {
RESTWarning << "(expand include file): find multiple xml sections with same name!"
<< RESTendl;
Expand Down
2 changes: 1 addition & 1 deletion source/framework/tools/inc/TRestTools.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ class TRestTools {

static std::string Execute(std::string cmd);

static std::string DownloadRemoteFile(std::string remoteFile);
static std::string DownloadRemoteFile(const std::string& remoteFile);
static int DownloadRemoteFile(std::string remoteFile, std::string localFile);
static int UploadToServer(std::string localFile, std::string remoteFile, std::string methodUrl = "");

Expand Down
30 changes: 14 additions & 16 deletions source/framework/tools/src/TRestTools.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
#include <TSystem.h>
#include <TUrl.h>

#include <regex>

#ifdef USE_Curl
#include <curl/curl.h>
#endif
Expand Down Expand Up @@ -711,11 +713,9 @@ bool TRestTools::isDataSet(const std::string& filename) {
///////////////////////////////////////////////
/// \brief Returns true if **filename** is an *http* address.
///
bool TRestTools::isURL(const string& filename) {
if (filename.find("http") == 0) {
return true;
}
return false;
bool TRestTools::isURL(const string& s) {
std::regex pattern("^https?://(.+)");
return std::regex_match(s, pattern);
}

///////////////////////////////////////////////
Expand Down Expand Up @@ -1049,19 +1049,19 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) {
/// will be used, including scp, wget. Downloads to REST_USER_PATH + "/download/" + filename
/// by default.
///
std::string TRestTools::DownloadRemoteFile(string url) {
string purename = TRestTools::GetPureFileName(url);
if (purename == "") {
cout << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << endl;
cout << "please specify a concrete file name in this url" << endl;
cout << "url: " << url << endl;
std::string TRestTools::DownloadRemoteFile(const string& url) {
string pureName = TRestTools::GetPureFileName(url);
if (pureName.empty()) {
RESTWarning << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << RESTendl;
RESTWarning << "please specify a concrete file name in this url" << RESTendl;
RESTWarning << "url: " << url << RESTendl;
return "";
}

if (url.find("local:") == 0) {
return Replace(url, "local:", "");
} else {
string fullpath = REST_USER_PATH + "/download/" + purename;
string fullpath = REST_USER_PATH + "/download/" + pureName;
int out;
int attempts = 10;
do {
Expand All @@ -1070,14 +1070,12 @@ std::string TRestTools::DownloadRemoteFile(string url) {
RESTWarning << "Retrying download in 5 seconds" << RESTendl;
std::this_thread::sleep_for(std::chrono::seconds(5));
} else if (attempts < 10) {
RESTSuccess << "Download suceeded after " << 10 - attempts << " attempts" << RESTendl;
RESTSuccess << "Download succeeded after " << 10 - attempts << " attempts" << RESTendl;
}
attempts--;
} while (out == 1024 && attempts > 0);

if (out == 0) {
return fullpath;
} else if (TRestTools::fileExists(fullpath)) {
if (out == 0 || TRestTools::fileExists(fullpath)) {
return fullpath;
} else {
return "";
Expand Down

0 comments on commit 200e8f2

Please sign in to comment.