Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve support for remote metadata files #480

Merged
merged 4 commits into from
Sep 21, 2023
Merged

Improve support for remote metadata files #480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b641ae5
improve url checker method
lobis Sep 21, 2023
9095f13
working with remote readouts
lobis Sep 21, 2023
4e0cee2
remote files
lobis Sep 21, 2023
1f7d08a
warnings
lobis Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion source/framework/core/inc/TRestMetadata.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class TRestMetadata : public TNamed {
void ReadEnvInElement(TiXmlElement* e, bool overwrite = true);
void ReadElement(TiXmlElement* e, bool recursive = false);
void ReplaceForLoopVars(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandForLoopOnce(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandForLoopOnce(TiXmlElement* e, const std::map<std::string, std::string>& forLoopVar);
void ExpandForLoops(TiXmlElement* e, std::map<std::string, std::string> forLoopVar);
void ExpandIfSections(TiXmlElement* e);
void ExpandIncludeFile(TiXmlElement* e);
Expand Down
28 changes: 17 additions & 11 deletions source/framework/core/src/TRestMetadata.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1065,8 +1065,10 @@ void TRestMetadata::ExpandIfSections(TiXmlElement* e) {

///////////////////////////////////////////////
/// \brief Helper method for TRestMetadata::ExpandForLoops().
void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, map<string, string> forLoopVar) {
if (e == nullptr) return;
void TRestMetadata::ExpandForLoopOnce(TiXmlElement* e, const map<string, string>& forLoopVar) {
if (e == nullptr) {
return;
}

TiXmlElement* parele = (TiXmlElement*)e->Parent();
TiXmlElement* contentelement = e->FirstChildElement();
Expand Down Expand Up @@ -1098,7 +1100,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL
if (e == nullptr) return;

RESTDebug << "Entering ... TRestMetadata::ReplaceForLoopVars" << RESTendl;
std::string parName = "";
std::string parName;
TiXmlAttribute* attr = e->FirstAttribute();
while (attr != nullptr) {
const char* val = attr->Value();
Expand All @@ -1111,7 +1113,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL
if (strcmp(name, "name") != 0) {
string outputBuffer = val;

if (outputBuffer.find("[") != string::npos || outputBuffer.find("]") != string::npos) {
if (outputBuffer.find('[') != string::npos || outputBuffer.find(']') != string::npos) {
RESTError << "TRestMetadata::ReplaceForLoopVars. Old for-loop construction identified"
<< RESTendl;
RESTError << "Please, replace [] variable nomenclature by ${}." << RESTendl;
Expand All @@ -1134,7 +1136,7 @@ void TRestMetadata::ReplaceForLoopVars(TiXmlElement* e, map<string, string> forL

string proenv = forLoopVar.count(expression) > 0 ? forLoopVar[expression] : "";

if (proenv != "") {
if (!proenv.empty()) {
outputBuffer.replace(replacePos, replaceLen, proenv);
endPosition = 0;
} else {
Expand Down Expand Up @@ -1203,11 +1205,11 @@ void TRestMetadata::ExpandForLoops(TiXmlElement* e, map<string, string> forloopv

if (fVerboseLevel >= TRestStringOutput::REST_Verbose_Level::REST_Extreme) parele->Print(stdout, 0);
RESTDebug << "----end of for loop----" << RESTendl;
} else if (_in != "") {
} else if (!_in.empty()) {
vector<string> loopvars = Split(_in, ":");

RESTDebug << "----expanding for loop----" << RESTendl;
for (string loopvar : loopvars) {
for (const string& loopvar : loopvars) {
forloopvar[_name] = loopvar;
fVariables[_name] = loopvar;
ExpandForLoopOnce(e, forloopvar);
Expand Down Expand Up @@ -1250,18 +1252,22 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) {
if (_filename == nullptr) return;

string filename;
if (string(_filename) == "server") {
if (string(_filename) == "server" || TRestTools::isURL(_filename)) {
// Let TRestRun to retrieve data according to run number later-on
// if ((string) this->ClassName() == "TRestRun") return;

// match the database, runNumber=0(default data), type="META_RML", tag=<section name>
auto url = gDataBase->query_data(DBEntry(0, "META_RML", e->Value())).value;
if (url.empty()) {
// don't really understand this "database" code, this just works
url = _filename;
}

filename = TRestTools::DownloadRemoteFile(url);
} else {
filename = SearchFile(_filename);
}

if (filename == "") {
if (filename.empty()) {
RESTError << "TRestMetadata::ExpandIncludeFile. Include file \"" << _filename << "\" does not exist!"
<< RESTendl;
exit(1);
Expand Down Expand Up @@ -1369,7 +1375,7 @@ void TRestMetadata::ExpandIncludeFile(TiXmlElement* e) {
}
ele = ele->NextSiblingElement();
}
// more than 1 elements found
// more than 1 element found
if (eles.size() > 1) {
RESTWarning << "(expand include file): find multiple xml sections with same name!"
<< RESTendl;
Expand Down
2 changes: 1 addition & 1 deletion source/framework/tools/inc/TRestTools.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ class TRestTools {

static std::string Execute(std::string cmd);

static std::string DownloadRemoteFile(std::string remoteFile);
static std::string DownloadRemoteFile(const std::string& remoteFile);
static int DownloadRemoteFile(std::string remoteFile, std::string localFile);
static int UploadToServer(std::string localFile, std::string remoteFile, std::string methodUrl = "");

Expand Down
30 changes: 14 additions & 16 deletions source/framework/tools/src/TRestTools.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
#include <TSystem.h>
#include <TUrl.h>

#include <regex>

#ifdef USE_Curl
#include <curl/curl.h>
#endif
Expand Down Expand Up @@ -711,11 +713,9 @@ bool TRestTools::isDataSet(const std::string& filename) {
///////////////////////////////////////////////
/// \brief Returns true if **filename** is an *http* address.
///
bool TRestTools::isURL(const string& filename) {
if (filename.find("http") == 0) {
return true;
}
return false;
bool TRestTools::isURL(const string& s) {
std::regex pattern("^https?://(.+)");
return std::regex_match(s, pattern);
}

///////////////////////////////////////////////
Expand Down Expand Up @@ -1049,19 +1049,19 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) {
/// will be used, including scp, wget. Downloads to REST_USER_PATH + "/download/" + filename
/// by default.
///
std::string TRestTools::DownloadRemoteFile(string url) {
string purename = TRestTools::GetPureFileName(url);
if (purename == "") {
cout << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << endl;
cout << "please specify a concrete file name in this url" << endl;
cout << "url: " << url << endl;
std::string TRestTools::DownloadRemoteFile(const string& url) {
string pureName = TRestTools::GetPureFileName(url);
if (pureName.empty()) {
RESTWarning << "error! (TRestTools::DownloadRemoteFile): url is not a file!" << RESTendl;
RESTWarning << "please specify a concrete file name in this url" << RESTendl;
RESTWarning << "url: " << url << RESTendl;
return "";
}

if (url.find("local:") == 0) {
return Replace(url, "local:", "");
} else {
string fullpath = REST_USER_PATH + "/download/" + purename;
string fullpath = REST_USER_PATH + "/download/" + pureName;
int out;
int attempts = 10;
do {
Expand All @@ -1070,14 +1070,12 @@ std::string TRestTools::DownloadRemoteFile(string url) {
RESTWarning << "Retrying download in 5 seconds" << RESTendl;
std::this_thread::sleep_for(std::chrono::seconds(5));
} else if (attempts < 10) {
RESTSuccess << "Download suceeded after " << 10 - attempts << " attempts" << RESTendl;
RESTSuccess << "Download succeeded after " << 10 - attempts << " attempts" << RESTendl;
}
attempts--;
} while (out == 1024 && attempts > 0);

if (out == 0) {
return fullpath;
} else if (TRestTools::fileExists(fullpath)) {
if (out == 0 || TRestTools::fileExists(fullpath)) {
return fullpath;
} else {
return "";
Expand Down