diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 045ea92ed..6b159fcfd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,4 @@ -image: ghcr.io/lobis/root-geant4-garfield:cpp17_ROOT-v6-26-00_Geant4-v10.4.3_Garfield-af4a1451 -#image: nkx1231/root6-geant4-garfield:0.6 +image: ghcr.io/lobis/root-geant4-garfield:rest-for-physics variables: # GIT_SUBMODULE_STRATEGY: recursive diff --git a/CMakeLists.txt b/CMakeLists.txt index 956b88d35..b122d96f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ find_package(ROOT REQUIRED COMPONENTS ${ROOT_REQUIRED_LIBRARIES}) message(STATUS "ROOT LIBRARIES: ${ROOT_LIBRARIES}") -set(external_libs "${external_libs};${ROOT_LIBRARIES}") +set(external_libs "${external_libs};${ROOT_LIBRARIES};-lcurl") set(ROOTCINT_EXECUTABLE ${ROOT_rootcint_CMD}) diff --git a/source/framework/core/src/TRestMetadata.cxx b/source/framework/core/src/TRestMetadata.cxx index 3e48ed19f..e721c1ece 100644 --- a/source/framework/core/src/TRestMetadata.cxx +++ b/source/framework/core/src/TRestMetadata.cxx @@ -2216,6 +2216,8 @@ TString TRestMetadata::GetSearchPath() { if (getenv("configPath")) result += getenv("configPath") + (string) ":"; result += REST_PATH + "/data/:"; + // We give priority to the official /data/ path. + result += REST_USER_PATH + ":"; if (result.back() == ':') result.erase(result.size() - 1); return ReplaceConstants(ReplaceVariables(result)); diff --git a/source/framework/tools/inc/TRestTools.h b/source/framework/tools/inc/TRestTools.h index c71ffd40a..c1065b17e 100644 --- a/source/framework/tools/inc/TRestTools.h +++ b/source/framework/tools/inc/TRestTools.h @@ -50,8 +50,8 @@ class TRestTools { static void LoadRESTLibrary(bool silent = false); - static int ReadASCIITable(string fName, std::vector>& data); - static int ReadASCIITable(string fName, std::vector>& data); + static int ReadASCIITable(string fName, std::vector>& data, Int_t skipLines = 0); + static int ReadASCIITable(string fName, std::vector>& data, Int_t skipLines = 0); template static int ReadBinaryTable(string fName, std::vector>& data, Int_t columns); @@ -68,6 +68,9 @@ class TRestTools { template static int PrintTable(std::vector> data, Int_t start = 0, Int_t end = 0); + template + static int ExportASCIITable(std::string fname, std::vector>& data); + static Int_t isValidFile(const string& path); static bool fileExists(const std::string& filename); static bool isRootFile(const std::string& filename); @@ -87,12 +90,13 @@ class TRestTools { static std::string Execute(string cmd); - static std::string DownloadRemoteFile(string remoteFile); - static int DownloadRemoteFile(string remoteFile, string localFile); - static int UploadToServer(string localfile, string remotefile, string methodurl = ""); + static std::string DownloadRemoteFile(std::string remoteFile); + static int DownloadRemoteFile(std::string remoteFile, std::string localFile); + static int UploadToServer(std::string localfile, std::string remotefile, std::string methodurl = ""); + static std::string POSTRequest(const std::string& url, const std::map& keys); - static void ChangeDirectory( string toDirectory ); - static void ReturnToPreviousDirectory( ); + static void ChangeDirectory(string toDirectory); + static void ReturnToPreviousDirectory(); /// Rest tools class ClassDef(TRestTools, 1); @@ -124,7 +128,7 @@ inline void SetInitLevel(T* name, int level) { struct __##classname##_Init { \ __##classname##_Init() { \ REST_ARGS[#objname] = #classname; \ - if (objname != nullptr) { \ + if (objname != nullptr) { \ if (REST_InitTools::CanOverwrite(objname, level)) { \ delete objname; \ objname = new classname(); \ diff --git a/source/framework/tools/src/TRestTools.cxx b/source/framework/tools/src/TRestTools.cxx index 4f49376c9..270c002c3 100644 --- a/source/framework/tools/src/TRestTools.cxx +++ b/source/framework/tools/src/TRestTools.cxx @@ -45,9 +45,11 @@ #include +#include #include #include #include +#include #include "TClass.h" #include "TRestStringHelper.h" @@ -55,6 +57,8 @@ #include "TSystem.h" #include "TUrl.h" +#include + ClassImp(TRestTools); /////////////////////////////////////////////// @@ -125,6 +129,35 @@ template int TRestTools::PrintTable(std::vector> d template int TRestTools::PrintTable(std::vector> data, Int_t start, Int_t end); +/////////////////////////////////////////////// +/// \brief Writes the contents of the vector table given as argument to `fname`. +/// Allowed types are Int_t, Float_t and Double_t. +/// +template +int TRestTools::ExportASCIITable(std::string fname, std::vector>& data) { + ofstream file(fname); + if (!file.is_open()) { + ferr << "Unable to open file for writting : " << fname << endl; + return 1; + } + + for (int n = 0; n < data.size(); n++) + for (int m = 0; m < data[n].size(); m++) { + file << data[n][m]; + if (m + 1 < data[n].size()) file << "\t"; + if (m + 1 == data[n].size()) file << "\n"; + } + file.close(); + + return 0; +} + +template int TRestTools::ExportASCIITable(std::string fname, std::vector>& data); +template int TRestTools::ExportASCIITable(std::string fname, + std::vector>& data); +template int TRestTools::ExportASCIITable(std::string fname, + std::vector>& data); + /////////////////////////////////////////////// /// \brief Reads a binary file containning a fixed-columns table with values /// @@ -178,7 +211,8 @@ template int TRestTools::ReadBinaryTable(string fName, std::vector(std::vector(std::vector T TRestTools::GetLowestIncreaseFromTable(std::vector> data, Int_t column) { @@ -261,7 +296,7 @@ template Double_t TRestTools::GetLowestIncreaseFromTable(std::vector>& data) { +int TRestTools::ReadASCIITable(string fName, std::vector>& data, Int_t skipLines) { if (!TRestTools::isValidFile((string)fName)) { cout << "TRestTools::ReadASCIITable. Error" << endl; cout << "Cannot open file : " << fName << endl; @@ -276,6 +311,11 @@ int TRestTools::ReadASCIITable(string fName, std::vector>& std::vector> values; for (string line; std::getline(fin, line);) { + if (skipLines > 0) { + skipLines--; + continue; + } + if (line.find("#") == string::npos) { std::istringstream in(line); values.push_back( @@ -307,7 +347,7 @@ int TRestTools::ReadASCIITable(string fName, std::vector>& /// /// This version works with Float_t vector since we use StringToFloat method. /// -int TRestTools::ReadASCIITable(string fName, std::vector>& data) { +int TRestTools::ReadASCIITable(string fName, std::vector>& data, Int_t skipLines) { if (!TRestTools::isValidFile((string)fName)) { cout << "TRestTools::ReadASCIITable. Error" << endl; cout << "Cannot open file : " << fName << endl; @@ -322,6 +362,11 @@ int TRestTools::ReadASCIITable(string fName, std::vector>& std::vector> values; for (string line; std::getline(fin, line);) { + if (skipLines > 0) { + skipLines--; + continue; + } + if (line.find("#") == string::npos) { std::istringstream in(line); values.push_back( @@ -698,7 +743,8 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) { /// /// The file name is given in url format, and is parsed by TUrl. Various methods /// will be used, including scp, wget. Downloads to REST_USER_PATH + "/download/" + filename -/// by default +/// by default. +/// std::string TRestTools::DownloadRemoteFile(string url) { string purename = TRestTools::GetPureFileName(url); if (purename == "") { @@ -712,8 +758,18 @@ std::string TRestTools::DownloadRemoteFile(string url) { return Replace(url, "local:", ""); } else { string fullpath = REST_USER_PATH + "/download/" + purename; + int out; + int attempts = 10; + do { + out = TRestTools::DownloadRemoteFile(url, fullpath); + if (out == 1024) { + warning << "Retrying download in 5 seconds" << endl; + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + attempts--; + } while (out == 1024 && attempts > 0); - if (TRestTools::DownloadRemoteFile(url, fullpath) == 0) { + if (out == 0) { return fullpath; } else if (TRestTools::fileExists(fullpath)) { return fullpath; @@ -729,6 +785,7 @@ std::string TRestTools::DownloadRemoteFile(string url) { /// /// The file name is given in url format, and is parsed by TUrl. Various methods /// will be used, including scp, wget. returns 0 if succeed. +/// int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) { TUrl url(remoteFile.c_str()); @@ -752,8 +809,14 @@ int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) { return 0; } else { ferr << "download failed! (" << remoteFile << ")" << endl; - if (a == 1024) ferr << "Network connection problem?" << endl; - if (a == 2048) ferr << "File does NOT exist in remotely?" << endl; + if (a == 1024) { + ferr << "Network connection problem?" << endl; + return 1024; + } + if (a == 2048) { + ferr << "File does NOT exist in remotely?" << endl; + return 2048; + } } } else if ((string)url.GetProtocol() == "ssh") { string cmd = "scp -P " + ToString(url.GetPort() == 0 ? 22 : url.GetPort()) + " " + url.GetUser() + @@ -771,13 +834,63 @@ int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) { return -1; } +/////////////////////////////////////////////// +/// \brief It performs a POST web protocol request using a set of keys and values given +/// by argument, and returns the result as a string. +/// +std::string TRestTools::POSTRequest(const std::string& url, const std::map& keys) { + CURL* curl; + CURLcode res; + + string filename = REST_USER_PATH + "/download/curl.out"; + + /* In windows, this will init the winsock stuff */ + curl_global_init(CURL_GLOBAL_ALL); + + FILE* f = fopen(filename.c_str(), "wt"); + + std::string request = ""; + int n = 0; + for (auto const& x : keys) { + if (n > 0) request += "&"; + request += x.first + "=" + x.second; + n++; + } + /* get a curl handle */ + curl = curl_easy_init(); + if (curl) { + /* First set the URL that is about to receive our POST. This URL can + just as well be a https:// URL if that is what should receive the + data. */ + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)f); + /* Now specify the POST data */ + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request.c_str()); + + /* Perform the request, res will get the return code */ + res = curl_easy_perform(curl); + /* Check for errors */ + if (res != CURLE_OK) ferr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << endl; + + /* always cleanup */ + curl_easy_cleanup(curl); + } + fclose(f); + curl_global_cleanup(); + + std::string file_content = ""; + std::getline(std::ifstream(filename), file_content, '\0'); + + return file_content; +} + /////////////////////////////////////////////// /// Upload the local file to remote file, method url will overwrite the login information /// inside remotefile. /// /// Example: UploadToServer("/home/nkx/abc.txt", "https://sultan.unizar.es/gasFiles/gases.rml", -/// "ssh://nkx:M123456@:8322") Then, the local file abc.txt will be uploaded to the server, renamed to -/// gases.rml and overwrite it +/// "ssh://nkx:M123456@:8322") Then, the local file abc.txt will be uploaded to the server, +/// renamed to gases.rml and overwrite it int TRestTools::UploadToServer(string filelocal, string remotefile, string methodurl) { if (!TRestTools::fileExists(filelocal)) { cout << "error! local file not exist!" << endl; @@ -815,32 +928,30 @@ int TRestTools::UploadToServer(string filelocal, string remotefile, string metho return 0; } -void TRestTools::ChangeDirectory( string toDirectory ) { - - char originDirectory[256]; - sprintf(originDirectory, "%s", getenv("PWD")); - chdir(toDirectory.c_str()); +void TRestTools::ChangeDirectory(string toDirectory) { + char originDirectory[256]; + sprintf(originDirectory, "%s", getenv("PWD")); + chdir(toDirectory.c_str()); - string fullPath = ""; - if( toDirectory[0] == '/' ) - fullPath = toDirectory; - else - fullPath = (string) originDirectory + "/" + toDirectory; + string fullPath = ""; + if (toDirectory[0] == '/') + fullPath = toDirectory; + else + fullPath = (string)originDirectory + "/" + toDirectory; - setenv ( "PWD", fullPath.c_str(), 1 ); - setenv ( "OLDPWD", originDirectory, 1 ); + setenv("PWD", fullPath.c_str(), 1); + setenv("OLDPWD", originDirectory, 1); } -void TRestTools::ReturnToPreviousDirectory( ) { - - char originDirectory[256]; - sprintf(originDirectory, "%s", getenv("PWD")); +void TRestTools::ReturnToPreviousDirectory() { + char originDirectory[256]; + sprintf(originDirectory, "%s", getenv("PWD")); - char newDirectory[256]; - sprintf(newDirectory, "%s", getenv("OLDPWD")); + char newDirectory[256]; + sprintf(newDirectory, "%s", getenv("OLDPWD")); - setenv ( "PWD", newDirectory, 1 ); - setenv ( "OLDPWD", originDirectory, 1 ); + setenv("PWD", newDirectory, 1); + setenv("OLDPWD", originDirectory, 1); - chdir( newDirectory ); + chdir(newDirectory); }