Skip to content

Commit

Permalink
Merge pull request #172 from rest-for-physics/jgalan_tools_post
Browse files Browse the repository at this point in the history
It adds a method at TRestTools to do http POST requests
  • Loading branch information
jgalan authored Apr 1, 2022
2 parents 938c6d8 + 5f53737 commit 1d60c10
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 46 deletions.
3 changes: 1 addition & 2 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
image: ghcr.io/lobis/root-geant4-garfield:cpp17_ROOT-v6-26-00_Geant4-v10.4.3_Garfield-af4a1451
#image: nkx1231/root6-geant4-garfield:0.6
image: ghcr.io/lobis/root-geant4-garfield:rest-for-physics

variables:
# GIT_SUBMODULE_STRATEGY: recursive
Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ find_package(ROOT REQUIRED COMPONENTS ${ROOT_REQUIRED_LIBRARIES})

message(STATUS "ROOT LIBRARIES: ${ROOT_LIBRARIES}")

set(external_libs "${external_libs};${ROOT_LIBRARIES}")
set(external_libs "${external_libs};${ROOT_LIBRARIES};-lcurl")

set(ROOTCINT_EXECUTABLE ${ROOT_rootcint_CMD})

Expand Down
2 changes: 2 additions & 0 deletions source/framework/core/src/TRestMetadata.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2216,6 +2216,8 @@ TString TRestMetadata::GetSearchPath() {

if (getenv("configPath")) result += getenv("configPath") + (string) ":";
result += REST_PATH + "/data/:";
// We give priority to the official /data/ path.
result += REST_USER_PATH + ":";
if (result.back() == ':') result.erase(result.size() - 1);

return ReplaceConstants(ReplaceVariables(result));
Expand Down
20 changes: 12 additions & 8 deletions source/framework/tools/inc/TRestTools.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ class TRestTools {

static void LoadRESTLibrary(bool silent = false);

static int ReadASCIITable(string fName, std::vector<std::vector<Double_t>>& data);
static int ReadASCIITable(string fName, std::vector<std::vector<Float_t>>& data);
static int ReadASCIITable(string fName, std::vector<std::vector<Double_t>>& data, Int_t skipLines = 0);
static int ReadASCIITable(string fName, std::vector<std::vector<Float_t>>& data, Int_t skipLines = 0);

template <typename T>
static int ReadBinaryTable(string fName, std::vector<std::vector<T>>& data, Int_t columns);
Expand All @@ -68,6 +68,9 @@ class TRestTools {
template <typename T>
static int PrintTable(std::vector<std::vector<T>> data, Int_t start = 0, Int_t end = 0);

template <typename T>
static int ExportASCIITable(std::string fname, std::vector<std::vector<T>>& data);

static Int_t isValidFile(const string& path);
static bool fileExists(const std::string& filename);
static bool isRootFile(const std::string& filename);
Expand All @@ -87,12 +90,13 @@ class TRestTools {

static std::string Execute(string cmd);

static std::string DownloadRemoteFile(string remoteFile);
static int DownloadRemoteFile(string remoteFile, string localFile);
static int UploadToServer(string localfile, string remotefile, string methodurl = "");
static std::string DownloadRemoteFile(std::string remoteFile);
static int DownloadRemoteFile(std::string remoteFile, std::string localFile);
static int UploadToServer(std::string localfile, std::string remotefile, std::string methodurl = "");
static std::string POSTRequest(const std::string& url, const std::map<std::string, std::string>& keys);

static void ChangeDirectory( string toDirectory );
static void ReturnToPreviousDirectory( );
static void ChangeDirectory(string toDirectory);
static void ReturnToPreviousDirectory();

/// Rest tools class
ClassDef(TRestTools, 1);
Expand Down Expand Up @@ -124,7 +128,7 @@ inline void SetInitLevel(T* name, int level) {
struct __##classname##_Init { \
__##classname##_Init() { \
REST_ARGS[#objname] = #classname; \
if (objname != nullptr) { \
if (objname != nullptr) { \
if (REST_InitTools::CanOverwrite(objname, level)) { \
delete objname; \
objname = new classname(); \
Expand Down
181 changes: 146 additions & 35 deletions source/framework/tools/src/TRestTools.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,20 @@

#include <dirent.h>

#include <chrono>
#include <iostream>
#include <limits>
#include <memory>
#include <thread>

#include "TClass.h"
#include "TRestStringHelper.h"
#include "TRestStringOutput.h"
#include "TSystem.h"
#include "TUrl.h"

#include <curl/curl.h>

ClassImp(TRestTools);

///////////////////////////////////////////////
Expand Down Expand Up @@ -125,6 +129,35 @@ template int TRestTools::PrintTable<Float_t>(std::vector<std::vector<Float_t>> d
template int TRestTools::PrintTable<Double_t>(std::vector<std::vector<Double_t>> data, Int_t start,
Int_t end);

///////////////////////////////////////////////
/// \brief Writes the contents of the vector table given as argument to `fname`.
/// Allowed types are Int_t, Float_t and Double_t.
///
template <typename T>
int TRestTools::ExportASCIITable(std::string fname, std::vector<std::vector<T>>& data) {
ofstream file(fname);
if (!file.is_open()) {
ferr << "Unable to open file for writting : " << fname << endl;
return 1;
}

for (int n = 0; n < data.size(); n++)
for (int m = 0; m < data[n].size(); m++) {
file << data[n][m];
if (m + 1 < data[n].size()) file << "\t";
if (m + 1 == data[n].size()) file << "\n";
}
file.close();

return 0;
}

template int TRestTools::ExportASCIITable<Int_t>(std::string fname, std::vector<std::vector<Int_t>>& data);
template int TRestTools::ExportASCIITable<Float_t>(std::string fname,
std::vector<std::vector<Float_t>>& data);
template int TRestTools::ExportASCIITable<Double_t>(std::string fname,
std::vector<std::vector<Double_t>>& data);

///////////////////////////////////////////////
/// \brief Reads a binary file containning a fixed-columns table with values
///
Expand Down Expand Up @@ -178,7 +211,8 @@ template int TRestTools::ReadBinaryTable<Double_t>(string fName, std::vector<std
Int_t columns);

///////////////////////////////////////////////
/// \brief It returns the maximum value for a particular `column` from the table given by argument.
/// \brief It returns the maximum value for a particular `column` from the table given by
/// argument.
///
/// This method is available for tables of type Float_t, Double_t and Int_t.
///
Expand All @@ -200,7 +234,8 @@ template Double_t TRestTools::GetMaxValueFromTable<Double_t>(std::vector<std::ve
Int_t column);

///////////////////////////////////////////////
/// \brief It returns the minimum value for a particular `column` from the table given by argument.
/// \brief It returns the minimum value for a particular `column` from the table given by
/// argument.
///
/// This method is available for tables of type Float_t, Double_t and Int_t.
///
Expand All @@ -222,13 +257,13 @@ template Double_t TRestTools::GetMinValueFromTable<Double_t>(std::vector<std::ve
Int_t column);

///////////////////////////////////////////////
/// \brief It returns the lowest increase, different from zero, between the elements of a particular `column`
/// from the table given by argument.
/// \brief It returns the lowest increase, different from zero, between the elements of a
/// particular `column` from the table given by argument.
///
/// This method is available for tables of type Float_t, Double_t and Int_t.
///
/// \warning This method will not check every possible column element difference. It will only look for
/// consecutive elements steps.
/// \warning This method will not check every possible column element difference. It will only
/// look for consecutive elements steps.
///
template <typename T>
T TRestTools::GetLowestIncreaseFromTable(std::vector<std::vector<T>> data, Int_t column) {
Expand Down Expand Up @@ -261,7 +296,7 @@ template Double_t TRestTools::GetLowestIncreaseFromTable<Double_t>(std::vector<s
///
/// Only works with Double_t vector since we use StringToDouble method.
///
int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Double_t>>& data) {
int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Double_t>>& data, Int_t skipLines) {
if (!TRestTools::isValidFile((string)fName)) {
cout << "TRestTools::ReadASCIITable. Error" << endl;
cout << "Cannot open file : " << fName << endl;
Expand All @@ -276,6 +311,11 @@ int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Double_t>>&
std::vector<std::vector<string>> values;

for (string line; std::getline(fin, line);) {
if (skipLines > 0) {
skipLines--;
continue;
}

if (line.find("#") == string::npos) {
std::istringstream in(line);
values.push_back(
Expand Down Expand Up @@ -307,7 +347,7 @@ int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Double_t>>&
///
/// This version works with Float_t vector since we use StringToFloat method.
///
int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Float_t>>& data) {
int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Float_t>>& data, Int_t skipLines) {
if (!TRestTools::isValidFile((string)fName)) {
cout << "TRestTools::ReadASCIITable. Error" << endl;
cout << "Cannot open file : " << fName << endl;
Expand All @@ -322,6 +362,11 @@ int TRestTools::ReadASCIITable(string fName, std::vector<std::vector<Float_t>>&
std::vector<std::vector<string>> values;

for (string line; std::getline(fin, line);) {
if (skipLines > 0) {
skipLines--;
continue;
}

if (line.find("#") == string::npos) {
std::istringstream in(line);
values.push_back(
Expand Down Expand Up @@ -698,7 +743,8 @@ std::istream& TRestTools::GetLine(std::istream& is, std::string& t) {
///
/// The file name is given in url format, and is parsed by TUrl. Various methods
/// will be used, including scp, wget. Downloads to REST_USER_PATH + "/download/" + filename
/// by default
/// by default.
///
std::string TRestTools::DownloadRemoteFile(string url) {
string purename = TRestTools::GetPureFileName(url);
if (purename == "") {
Expand All @@ -712,8 +758,18 @@ std::string TRestTools::DownloadRemoteFile(string url) {
return Replace(url, "local:", "");
} else {
string fullpath = REST_USER_PATH + "/download/" + purename;
int out;
int attempts = 10;
do {
out = TRestTools::DownloadRemoteFile(url, fullpath);
if (out == 1024) {
warning << "Retrying download in 5 seconds" << endl;
std::this_thread::sleep_for(std::chrono::seconds(5));
}
attempts--;
} while (out == 1024 && attempts > 0);

if (TRestTools::DownloadRemoteFile(url, fullpath) == 0) {
if (out == 0) {
return fullpath;
} else if (TRestTools::fileExists(fullpath)) {
return fullpath;
Expand All @@ -729,6 +785,7 @@ std::string TRestTools::DownloadRemoteFile(string url) {
///
/// The file name is given in url format, and is parsed by TUrl. Various methods
/// will be used, including scp, wget. returns 0 if succeed.
///
int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) {
TUrl url(remoteFile.c_str());

Expand All @@ -752,8 +809,14 @@ int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) {
return 0;
} else {
ferr << "download failed! (" << remoteFile << ")" << endl;
if (a == 1024) ferr << "Network connection problem?" << endl;
if (a == 2048) ferr << "File does NOT exist in remotely?" << endl;
if (a == 1024) {
ferr << "Network connection problem?" << endl;
return 1024;
}
if (a == 2048) {
ferr << "File does NOT exist in remotely?" << endl;
return 2048;
}
}
} else if ((string)url.GetProtocol() == "ssh") {
string cmd = "scp -P " + ToString(url.GetPort() == 0 ? 22 : url.GetPort()) + " " + url.GetUser() +
Expand All @@ -771,13 +834,63 @@ int TRestTools::DownloadRemoteFile(string remoteFile, string localFile) {
return -1;
}

///////////////////////////////////////////////
/// \brief It performs a POST web protocol request using a set of keys and values given
/// by argument, and returns the result as a string.
///
std::string TRestTools::POSTRequest(const std::string& url, const std::map<std::string, std::string>& keys) {
CURL* curl;
CURLcode res;

string filename = REST_USER_PATH + "/download/curl.out";

/* In windows, this will init the winsock stuff */
curl_global_init(CURL_GLOBAL_ALL);

FILE* f = fopen(filename.c_str(), "wt");

std::string request = "";
int n = 0;
for (auto const& x : keys) {
if (n > 0) request += "&";
request += x.first + "=" + x.second;
n++;
}
/* get a curl handle */
curl = curl_easy_init();
if (curl) {
/* First set the URL that is about to receive our POST. This URL can
just as well be a https:// URL if that is what should receive the
data. */
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)f);
/* Now specify the POST data */
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request.c_str());

/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) ferr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << endl;

/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(f);
curl_global_cleanup();

std::string file_content = "";
std::getline(std::ifstream(filename), file_content, '\0');

return file_content;
}

///////////////////////////////////////////////
/// Upload the local file to remote file, method url will overwrite the login information
/// inside remotefile.
///
/// Example: UploadToServer("/home/nkx/abc.txt", "https://sultan.unizar.es/gasFiles/gases.rml",
/// "ssh://nkx:M123456@:8322") Then, the local file abc.txt will be uploaded to the server, renamed to
/// gases.rml and overwrite it
/// "ssh://nkx:M123456@:8322") Then, the local file abc.txt will be uploaded to the server,
/// renamed to gases.rml and overwrite it
int TRestTools::UploadToServer(string filelocal, string remotefile, string methodurl) {
if (!TRestTools::fileExists(filelocal)) {
cout << "error! local file not exist!" << endl;
Expand Down Expand Up @@ -815,32 +928,30 @@ int TRestTools::UploadToServer(string filelocal, string remotefile, string metho
return 0;
}

void TRestTools::ChangeDirectory( string toDirectory ) {

char originDirectory[256];
sprintf(originDirectory, "%s", getenv("PWD"));
chdir(toDirectory.c_str());
void TRestTools::ChangeDirectory(string toDirectory) {
char originDirectory[256];
sprintf(originDirectory, "%s", getenv("PWD"));
chdir(toDirectory.c_str());

string fullPath = "";
if( toDirectory[0] == '/' )
fullPath = toDirectory;
else
fullPath = (string) originDirectory + "/" + toDirectory;
string fullPath = "";
if (toDirectory[0] == '/')
fullPath = toDirectory;
else
fullPath = (string)originDirectory + "/" + toDirectory;

setenv ( "PWD", fullPath.c_str(), 1 );
setenv ( "OLDPWD", originDirectory, 1 );
setenv("PWD", fullPath.c_str(), 1);
setenv("OLDPWD", originDirectory, 1);
}

void TRestTools::ReturnToPreviousDirectory( ) {

char originDirectory[256];
sprintf(originDirectory, "%s", getenv("PWD"));
void TRestTools::ReturnToPreviousDirectory() {
char originDirectory[256];
sprintf(originDirectory, "%s", getenv("PWD"));

char newDirectory[256];
sprintf(newDirectory, "%s", getenv("OLDPWD"));
char newDirectory[256];
sprintf(newDirectory, "%s", getenv("OLDPWD"));

setenv ( "PWD", newDirectory, 1 );
setenv ( "OLDPWD", originDirectory, 1 );
setenv("PWD", newDirectory, 1);
setenv("OLDPWD", originDirectory, 1);

chdir( newDirectory );
chdir(newDirectory);
}

0 comments on commit 1d60c10

Please sign in to comment.