From 1bcd656dcc3fd3dd47d97325b801a2a532bbcd0a Mon Sep 17 00:00:00 2001 From: haoruo Date: Mon, 7 Aug 2023 15:00:36 -0600 Subject: [PATCH 01/12] MPI seems working --- .../MachineLearning/GravityNN/CMakeLists.txt | 1 + .../MachineLearning/GravityNN/tighten.cpp | 68 ++++++++++++++++--- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index ada47e16..c0755a3e 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -46,3 +46,4 @@ add_executable( add_dependencies(tighten proto_dep) target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) target_link_libraries(tighten ${PROTOBUF_LIBRARIES} gravity OpenMP::OpenMP_CXX CLI11) +target_link_libraries(tighten mpi) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 095a0eda..f96c6aa1 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -61,6 +61,51 @@ double bound_neuron(std::string fname, std::string start_node, Bound neuron, con return mult * NN._rel_obj_val; } +void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& fname, const std::string& start_node, const std::vector& global_bounds) { + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + +// std::cout << "Rank " << rank << " of " << size << " started." << std::endl; + + // Determine the range of neurons this rank should handle. + size_t neurons_per_rank = local_bounds.size() / size; + size_t start_idx = rank * neurons_per_rank; + size_t end_idx = (rank == size - 1) ? local_bounds.size() : start_idx + neurons_per_rank; + +// std::cout << "Rank " << rank << " is handling neurons from " << start_idx << " to " << end_idx << std::endl; + + // Each rank computes bounds for its subset of neurons. + for (size_t i = start_idx; i < end_idx; i++) { + Bound& neuron = local_bounds[i]; + auto new_bound = bound_neuron(fname, start_node, neuron, global_bounds); + if (neuron.side == LOWER) { + neuron.value = std::max(neuron.value, new_bound); + } else { + neuron.value = std::min(neuron.value, new_bound); + } + +// std::cout << "Rank " << rank << " updated bound for neuron " << i << " with value " << neuron.value << std::endl; + } + + // Gather the results at rank 0. + if (rank == 0) { + for (int src_rank = 1; src_rank < size; src_rank++) { + size_t src_start_idx = src_rank * neurons_per_rank; + size_t src_end_idx = (src_rank == size - 1) ? local_bounds.size() : src_start_idx + neurons_per_rank; + MPI_Recv(&local_bounds[src_start_idx], (src_end_idx - src_start_idx) * sizeof(Bound), MPI_BYTE, src_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); +// std::cout << "Rank 0 received data from rank " << src_rank << std::endl; + } + } else { + MPI_Send(&local_bounds[start_idx], (end_idx - start_idx) * sizeof(Bound), MPI_BYTE, 0, 0, MPI_COMM_WORLD); +// std::cout << "Rank " << rank << " sent data to rank 0." << std::endl; + } + + // If necessary, rank 0 can then broadcast the complete results back to all other ranks. + MPI_Bcast(&local_bounds[0], local_bounds.size() * sizeof(Bound), MPI_BYTE, 0, MPI_COMM_WORLD); +// std::cout << "Rank " << rank << " completed broadcasting." << std::endl; +} + int main(int argc, char * argv[]) { string fname = string(prj_dir)+"/data_sets/VNN/tll_new_old.onnx"; if(argc >= 2) { @@ -95,6 +140,10 @@ int main(int argc, char * argv[]) { if (rolling_horizon > layers_to_optimize.size()){ rolling_horizon = layers_to_optimize.size(); } + + auto start_time_all = std::chrono::high_resolution_clock::now(); + + MPI_Init(nullptr, nullptr); for (auto lidx = 0; lidx < layers_to_optimize.size(); lidx++) { auto l = layers_to_optimize[lidx]; @@ -140,17 +189,9 @@ int main(int argc, char * argv[]) { } auto num_threads = std::thread::hardware_concurrency(); - -#pragma omp parallel for num_threads(num_threads / 2) - for (auto& neuron: local_bounds) { - auto new_bound = bound_neuron(fname, start_node, neuron, global_bounds); - auto prev_bound = neuron.value; - if (neuron.side == LOWER) { - neuron.value = std::max(neuron.value, new_bound); - } else { - neuron.value = std::min(neuron.value, new_bound); - } - } + + // Use the run_MPI_bound_neuron function for parallel neuron bound calculation + run_MPI_bound_neuron(local_bounds, fname, start_node, global_bounds); auto end_time = std::chrono::high_resolution_clock::now(); fflush(stdout); dup2(bak, 1); @@ -169,6 +210,7 @@ int main(int argc, char * argv[]) { std::cout << "Time: " << time_taken << "ms" << std::endl; global_bounds.insert(global_bounds.end(), local_bounds.begin(), local_bounds.end()); } + MPI_Finalize(); std::cout << "Starting final runs" << std::endl; @@ -176,6 +218,10 @@ int main(int argc, char * argv[]) { std::cout << "########################################" << std::endl; final_run(fname, global_bounds, obj_idx); } + + auto end_time_all = std::chrono::high_resolution_clock::now(); + auto time_taken_all = std::chrono::duration_cast(end_time_all - start_time_all).count(); + std::cout << "Time in total: " << time_taken_all << "ms" << std::endl; return 0; } From acd289aec6a49590c6228597bd301a2822489230 Mon Sep 17 00:00:00 2001 From: haoruo Date: Mon, 7 Aug 2023 15:01:26 -0600 Subject: [PATCH 02/12] remove total time --- examples/MachineLearning/GravityNN/tighten.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index f96c6aa1..26efad1e 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -141,8 +141,6 @@ int main(int argc, char * argv[]) { rolling_horizon = layers_to_optimize.size(); } - auto start_time_all = std::chrono::high_resolution_clock::now(); - MPI_Init(nullptr, nullptr); for (auto lidx = 0; lidx < layers_to_optimize.size(); lidx++) { @@ -218,10 +216,6 @@ int main(int argc, char * argv[]) { std::cout << "########################################" << std::endl; final_run(fname, global_bounds, obj_idx); } - - auto end_time_all = std::chrono::high_resolution_clock::now(); - auto time_taken_all = std::chrono::duration_cast(end_time_all - start_time_all).count(); - std::cout << "Time in total: " << time_taken_all << "ms" << std::endl; return 0; } From 8772b0b44a0d12c4ded2237c91d641caacd7403f Mon Sep 17 00:00:00 2001 From: haoruo Date: Wed, 9 Aug 2023 15:57:04 -0600 Subject: [PATCH 03/12] working MPI version on the cluster --- .../GravityNN/network/Layers/Linear.hpp | 3 +- .../GravityNN/network/Layers/Unary.hpp | 3 +- .../GravityNN/network/NeuralNet.hpp | 2 +- .../GravityNN/network/types.hpp | 8 + .../MachineLearning/GravityNN/tighten.cpp | 165 +++++++++++++----- 5 files changed, 130 insertions(+), 51 deletions(-) diff --git a/examples/MachineLearning/GravityNN/network/Layers/Linear.hpp b/examples/MachineLearning/GravityNN/network/Layers/Linear.hpp index 9ab4861a..d1e62f3b 100644 --- a/examples/MachineLearning/GravityNN/network/Layers/Linear.hpp +++ b/examples/MachineLearning/GravityNN/network/Layers/Linear.hpp @@ -109,7 +109,8 @@ class GEMM : public Layer { NN.add(Gemm.in(inds["Constr"]) == 0); // add IBP bounds auto f = x.in(inds["In"])*w.in(inds["B"]) + w.in(inds["C"]); - x.in(inds["Out"]).copy_bounds(*f._all_range); + // FIXME: add tighter bounds between existing one and the IBP one. + // x.in(inds["Out"]).copy_bounds(*f._all_range); } Tensor *A, *B, *C = nullptr; // Inputs diff --git a/examples/MachineLearning/GravityNN/network/Layers/Unary.hpp b/examples/MachineLearning/GravityNN/network/Layers/Unary.hpp index 8cd36916..4499f0b8 100644 --- a/examples/MachineLearning/GravityNN/network/Layers/Unary.hpp +++ b/examples/MachineLearning/GravityNN/network/Layers/Unary.hpp @@ -150,7 +150,8 @@ class Relu : public Layer { // add IBP bounds gravity::func g = x.in(inds["In"]); auto f = ReLU(g); - x.in(inds["Out"]).copy_bounds(*f._all_range); + // FIXME: add tighter bounds between existing one and the IBP one. + // x.in(inds["Out"]).copy_bounds(*f._all_range); } diff --git a/examples/MachineLearning/GravityNN/network/NeuralNet.hpp b/examples/MachineLearning/GravityNN/network/NeuralNet.hpp index 3f619b78..c42660b8 100644 --- a/examples/MachineLearning/GravityNN/network/NeuralNet.hpp +++ b/examples/MachineLearning/GravityNN/network/NeuralNet.hpp @@ -378,4 +378,4 @@ class NeuralNet { std::map output_to_layer; std::set folded_layers; -}; \ No newline at end of file +}; diff --git a/examples/MachineLearning/GravityNN/network/types.hpp b/examples/MachineLearning/GravityNN/network/types.hpp index a2f6aacb..909af419 100644 --- a/examples/MachineLearning/GravityNN/network/types.hpp +++ b/examples/MachineLearning/GravityNN/network/types.hpp @@ -46,6 +46,14 @@ class Bound { this->side = side; this->old_value = value; } + + Bound(std::string layer_name, std::string neuron_name, double value, double old_value, Side side) { + this->layer_name = layer_name; + this->neuron_name = neuron_name; + this->value = value; + this->side = side; + this->old_value = old_value; + } std::string layer_name; std::string neuron_name; diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 26efad1e..44052439 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -8,10 +8,15 @@ #include #include #include +#include using namespace gravity; void final_run(std::string fname, const std::vector& global_bounds, size_t obj_idx) { + // char hostname[128]; + // gethostname(hostname, sizeof(hostname)); + // std::cout << "Running final_run on host: " << hostname << " with process ID: " << getpid() << std::endl; + NeuralNet nn(fname); nn.set_aux_bounds(global_bounds); @@ -28,6 +33,10 @@ void final_run(std::string fname, const std::vector& global_bounds, size_ } double bound_neuron(std::string fname, std::string start_node, Bound neuron, const std::vector& global_bounds) { + // char hostname[128]; + // gethostname(hostname, sizeof(hostname)); + // std::cout << "Running bound_neuron on host: " << hostname << " with process ID: " << getpid() << " bounding neuron" << neuron.neuron_name << std::endl; + NeuralNet nn(fname); nn.set_aux_bounds(global_bounds); @@ -61,21 +70,55 @@ double bound_neuron(std::string fname, std::string start_node, Bound neuron, con return mult * NN._rel_obj_val; } +constexpr int MAX_LAYER_NAME_LEN = 100; +constexpr int MAX_NEURON_NAME_LEN = 100; + +MPI_Datatype MPI_PROXY_BOUND_TYPE; +const int nitems = 5; +int blocklengths[5] = {MAX_LAYER_NAME_LEN, MAX_LAYER_NAME_LEN, 1, 1, 1}; +MPI_Datatype types[5] = {MPI_CHAR, MPI_CHAR, MPI_DOUBLE, MPI_DOUBLE, MPI_INT}; +MPI_Aint offsets[5]; + +class ProxyBound { +public: + char layer_name[MAX_LAYER_NAME_LEN]; + char neuron_name[MAX_NEURON_NAME_LEN]; + double value; + double old_value; + Side side; + + ProxyBound(const Bound& b) { + strncpy(layer_name, b.layer_name.c_str(), MAX_LAYER_NAME_LEN); + layer_name[MAX_LAYER_NAME_LEN - 1] = '\0'; + + strncpy(neuron_name, b.neuron_name.c_str(), MAX_NEURON_NAME_LEN); + neuron_name[MAX_NEURON_NAME_LEN - 1] = '\0'; + + value = b.value; + old_value = b.old_value; + side = b.side; + } + + Bound toBound() const { + return Bound(std::string(layer_name), std::string(neuron_name), value, old_value, side); + } +}; + void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& fname, const std::string& start_node, const std::vector& global_bounds) { int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); -// std::cout << "Rank " << rank << " of " << size << " started." << std::endl; - - // Determine the range of neurons this rank should handle. - size_t neurons_per_rank = local_bounds.size() / size; - size_t start_idx = rank * neurons_per_rank; - size_t end_idx = (rank == size - 1) ? local_bounds.size() : start_idx + neurons_per_rank; + // Adjust how many neurons each rank handles + size_t total_bounds = local_bounds.size(); + size_t neurons_per_rank = total_bounds / size; + size_t extra_neurons = total_bounds % size; -// std::cout << "Rank " << rank << " is handling neurons from " << start_idx << " to " << end_idx << std::endl; + size_t start_idx = rank * neurons_per_rank + std::min(static_cast(rank), extra_neurons); + size_t end_idx = start_idx + neurons_per_rank + (rank < extra_neurons); - // Each rank computes bounds for its subset of neurons. + // Compute bounds for local neurons + #pragma omp parallel for for (size_t i = start_idx; i < end_idx; i++) { Bound& neuron = local_bounds[i]; auto new_bound = bound_neuron(fname, start_node, neuron, global_bounds); @@ -84,26 +127,33 @@ void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& f } else { neuron.value = std::min(neuron.value, new_bound); } + } -// std::cout << "Rank " << rank << " updated bound for neuron " << i << " with value " << neuron.value << std::endl; + // Convert local_bounds to ProxyBound for MPI communication + std::vector proxyLocalBounds; + for (const auto& bound : local_bounds) { + proxyLocalBounds.emplace_back(bound); } - // Gather the results at rank 0. + // Gather the results at rank 0 if (rank == 0) { for (int src_rank = 1; src_rank < size; src_rank++) { - size_t src_start_idx = src_rank * neurons_per_rank; - size_t src_end_idx = (src_rank == size - 1) ? local_bounds.size() : src_start_idx + neurons_per_rank; - MPI_Recv(&local_bounds[src_start_idx], (src_end_idx - src_start_idx) * sizeof(Bound), MPI_BYTE, src_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); -// std::cout << "Rank 0 received data from rank " << src_rank << std::endl; + size_t src_start_idx = src_rank * neurons_per_rank + std::min(static_cast(src_rank), extra_neurons); + size_t src_end_idx = src_start_idx + neurons_per_rank + (src_rank < extra_neurons); + + MPI_Recv(&proxyLocalBounds[src_start_idx], src_end_idx - src_start_idx, MPI_PROXY_BOUND_TYPE, src_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else { - MPI_Send(&local_bounds[start_idx], (end_idx - start_idx) * sizeof(Bound), MPI_BYTE, 0, 0, MPI_COMM_WORLD); -// std::cout << "Rank " << rank << " sent data to rank 0." << std::endl; + MPI_Send(&proxyLocalBounds[start_idx], end_idx - start_idx, MPI_PROXY_BOUND_TYPE, 0, 0, MPI_COMM_WORLD); } - // If necessary, rank 0 can then broadcast the complete results back to all other ranks. - MPI_Bcast(&local_bounds[0], local_bounds.size() * sizeof(Bound), MPI_BYTE, 0, MPI_COMM_WORLD); -// std::cout << "Rank " << rank << " completed broadcasting." << std::endl; + // Broadcast the updated bounds back to all ranks + MPI_Bcast(&proxyLocalBounds[0], proxyLocalBounds.size(), MPI_PROXY_BOUND_TYPE, 0, MPI_COMM_WORLD); + + // Convert back the received ProxyBounds to Bounds + for (size_t i = 0; i < local_bounds.size(); ++i) { + local_bounds[i] = proxyLocalBounds[i].toBound(); + } } int main(int argc, char * argv[]) { @@ -112,6 +162,10 @@ int main(int argc, char * argv[]) { fname = argv[1]; } + MPI_Init(nullptr, nullptr); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + NeuralNet nn(fname); std::vector layers_to_optimize; @@ -128,26 +182,27 @@ int main(int argc, char * argv[]) { // include last layer layers_to_optimize.push_back(nn._all_layers[nn._all_layers.size()-1]); - - std::cout << "Optimizing layers:" << std::endl; - for (auto l: layers_to_optimize) { - std::cout << l->lname() << std::endl; + if (rank == 0) { + std::cout << "Optimizing layers:" << std::endl; + for (auto l: layers_to_optimize) { + std::cout << l->lname() << std::endl; + } } std::vector global_bounds; - int rolling_horizon = 2; + int rolling_horizon = 3; if (rolling_horizon > layers_to_optimize.size()){ rolling_horizon = layers_to_optimize.size(); } - - MPI_Init(nullptr, nullptr); for (auto lidx = 0; lidx < layers_to_optimize.size(); lidx++) { auto l = layers_to_optimize[lidx]; - std::cout << "################################################" << std::endl; - std::cout << "Optimizing layer: " << l->lname() << std::endl; - std::cout << "Layer " << lidx+1 << "/" << layers_to_optimize.size() << std::endl; + if (rank == 0) { + std::cout << "################################################" << std::endl; + std::cout << "Optimizing layer: " << l->lname() << std::endl; + std::cout << "Layer " << lidx+1 << "/" << layers_to_optimize.size() << std::endl; + } std::vector local_bounds; for (auto o: l->outputs) { for (auto i = 0; i < o->numel; i++) { @@ -165,8 +220,10 @@ int main(int argc, char * argv[]) { } } - std::cout << "Number of neurons to optimize: " << local_bounds.size()/2 << std::endl; - + if (rank == 0) { + std::cout << "Number of neurons to optimize: " << local_bounds.size()/2 << std::endl; + } + // skip the rest of this loop iteration if local_bounds is of size 0 if (local_bounds.size() == 0) { continue; @@ -186,9 +243,17 @@ int main(int argc, char * argv[]) { start_node = layers_to_optimize[lidx - (rolling_horizon - 1)]->name; } - auto num_threads = std::thread::hardware_concurrency(); - // Use the run_MPI_bound_neuron function for parallel neuron bound calculation + + offsets[0] = offsetof(ProxyBound, layer_name); + offsets[1] = offsetof(ProxyBound, neuron_name); + offsets[2] = offsetof(ProxyBound, value); + offsets[3] = offsetof(ProxyBound, old_value); + offsets[4] = offsetof(ProxyBound, side); + + MPI_Type_create_struct(nitems, blocklengths, offsets, types, &MPI_PROXY_BOUND_TYPE); + MPI_Type_commit(&MPI_PROXY_BOUND_TYPE); + run_MPI_bound_neuron(local_bounds, fname, start_node, global_bounds); auto end_time = std::chrono::high_resolution_clock::now(); fflush(stdout); @@ -196,26 +261,30 @@ int main(int argc, char * argv[]) { close(bak); // print out the final bounds - for (int i = 0; i < local_bounds.size()-1; i+=2) { - auto lb = local_bounds[i]; - auto ub = local_bounds[i+1]; - std::cout << lb.neuron_name << ": "; - std::cout << "[" << ftostr(lb.old_value) << ", " << ftostr(ub.old_value) << "] -> "; - std::cout << "[" << ftostr(lb.value) << ", " << ftostr(ub.value) << "]"; - std::cout << std::endl; + if (rank == 0) { + for (int i = 0; i < local_bounds.size()-1; i+=2) { + auto lb = local_bounds[i]; + auto ub = local_bounds[i+1]; + std::cout << lb.neuron_name << ": "; + std::cout << "[" << ftostr(lb.old_value) << ", " << ftostr(ub.old_value) << "] -> "; + std::cout << "[" << ftostr(lb.value) << ", " << ftostr(ub.value) << "]"; + std::cout << std::endl; + } + auto time_taken = std::chrono::duration_cast(end_time - start_time).count(); + std::cout << "Time: " << time_taken << "ms" << std::endl; } - auto time_taken = std::chrono::duration_cast(end_time - start_time).count(); - std::cout << "Time: " << time_taken << "ms" << std::endl; global_bounds.insert(global_bounds.end(), local_bounds.begin(), local_bounds.end()); } - MPI_Finalize(); + MPI_Type_free(&MPI_PROXY_BOUND_TYPE); - std::cout << "Starting final runs" << std::endl; - - for (size_t obj_idx = 0; obj_idx < nn.obj_spec->shape[0]; obj_idx++) { - std::cout << "########################################" << std::endl; - final_run(fname, global_bounds, obj_idx); + if (rank == 0) { + std::cout << "Starting final runs" << std::endl; + for (size_t obj_idx = 0; obj_idx < nn.obj_spec->shape[0]; obj_idx++) { + std::cout << "########################################" << std::endl; + final_run(fname, global_bounds, obj_idx); + } } + MPI_Finalize(); return 0; } From 70dbc25da9a40da36783efe91e4374730fb46ab1 Mon Sep 17 00:00:00 2001 From: haoruo Date: Wed, 9 Aug 2023 16:15:48 -0600 Subject: [PATCH 04/12] update print message --- examples/MachineLearning/GravityNN/tighten.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 44052439..00451387 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -35,7 +35,7 @@ void final_run(std::string fname, const std::vector& global_bounds, size_ double bound_neuron(std::string fname, std::string start_node, Bound neuron, const std::vector& global_bounds) { // char hostname[128]; // gethostname(hostname, sizeof(hostname)); - // std::cout << "Running bound_neuron on host: " << hostname << " with process ID: " << getpid() << " bounding neuron" << neuron.neuron_name << std::endl; + // std::cout << "Running bound_neuron on host: " << hostname << " with process ID: " << getpid() << " bounding " << neuron.side << " of neuron " << neuron.neuron_name << std::endl; NeuralNet nn(fname); nn.set_aux_bounds(global_bounds); @@ -191,7 +191,7 @@ int main(int argc, char * argv[]) { std::vector global_bounds; - int rolling_horizon = 3; + int rolling_horizon = 2; if (rolling_horizon > layers_to_optimize.size()){ rolling_horizon = layers_to_optimize.size(); } @@ -244,7 +244,6 @@ int main(int argc, char * argv[]) { } // Use the run_MPI_bound_neuron function for parallel neuron bound calculation - offsets[0] = offsetof(ProxyBound, layer_name); offsets[1] = offsetof(ProxyBound, neuron_name); offsets[2] = offsetof(ProxyBound, value); From 269d8ad3668ae93115e9e3f40f18ae4037481354 Mon Sep 17 00:00:00 2001 From: haoruo Date: Wed, 9 Aug 2023 16:44:43 -0600 Subject: [PATCH 05/12] record some useful gurobi parameters --- examples/MachineLearning/GravityNN/tighten.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 00451387..46844e3b 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -28,6 +28,18 @@ void final_run(std::string fname, const std::vector& global_bounds, size_ grb_mod->set(GRB_IntParam_Threads, thread::hardware_concurrency() / 2); grb_mod->set(GRB_IntParam_OutputFlag, 1); grb_mod->set(GRB_IntParam_NonConvex, 2); + +// grb_mod->set(GRB_DoubleParam_Heuristics, 0); +// grb_mod->set(GRB_IntParam_CutPasses, 5); +// grb_mod->set(GRB_IntParam_VarBranch, 0); +// +// double tolerance = grb_mod->get(GRB_DoubleParam_MIPGap); +// +// // Stop the program if the lower bound exceeds 0 +// grb_mod->set(GRB_DoubleParam_BestBdStop, tolerance); +// +// // Stop the program if a feasible solution with objective less than 0 is found +// grb_mod->set(GRB_DoubleParam_BestObjStop, -tolerance); int retval = S.run(); } From 6c25290a9c09a45dfc800746fead73b6fab23aad Mon Sep 17 00:00:00 2001 From: haoruo Date: Wed, 9 Aug 2023 17:38:44 -0600 Subject: [PATCH 06/12] parallelize final run for multiple objs --- .../MachineLearning/GravityNN/tighten.cpp | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 46844e3b..4e6dec32 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -175,8 +175,9 @@ int main(int argc, char * argv[]) { } MPI_Init(nullptr, nullptr); - int rank; + int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); NeuralNet nn(fname); std::vector layers_to_optimize; @@ -290,12 +291,22 @@ int main(int argc, char * argv[]) { if (rank == 0) { std::cout << "Starting final runs" << std::endl; - for (size_t obj_idx = 0; obj_idx < nn.obj_spec->shape[0]; obj_idx++) { - std::cout << "########################################" << std::endl; - final_run(fname, global_bounds, obj_idx); - } } + size_t total_objs = nn.obj_spec->shape[0]; + size_t objs_per_rank = total_objs / size; + size_t extra_objs = total_objs % size; + + size_t start_obj_idx = rank * objs_per_rank + std::min(static_cast(rank), extra_objs); + size_t end_obj_idx = start_obj_idx + objs_per_rank + (rank < extra_objs); + + for (size_t obj_idx = start_obj_idx; obj_idx < end_obj_idx; obj_idx++) { + std::cout << "########################################" << std::endl; + final_run(fname, global_bounds, obj_idx); + } + + // FIXME: gather final_run results + MPI_Finalize(); return 0; } From ae4a00f5b4833cac5fa2b000dfea040954ba5ab5 Mon Sep 17 00:00:00 2001 From: "Zhao, Haoruo" Date: Tue, 15 Aug 2023 18:21:27 -0600 Subject: [PATCH 07/12] trying to compile on M2 Apple chip --- CMakeLists.txt | 6 ++-- .../MachineLearning/GravityNN/CMakeLists.txt | 22 +++++++++--- .../GravityNN/network/Tensor.hpp | 2 +- .../MachineLearning/GravityNN/tighten.cpp | 35 ++++++++++++------- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1819d423..7bc95d09 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,9 +86,9 @@ if (OpenMPI) message("Compilers used: ${CMAKE_C_COMPILER} and ${CMAKE_CXX_COMPILER}") message(STATUS "Enable OpenMPI") add_definitions(-DUSE_MPI) -# find_package(OpenMPI REQUIRED) -# include_directories(${OpenMPI_INCLUDE_DIRS}) -# set(LIBS ${LIBS} ${OpenMPI_LIBRARIES}) + find_package(OpenMPI REQUIRED) + include_directories(${OpenMPI_INCLUDE_DIRS}) + set(LIBS ${LIBS} ${OpenMPI_LIBRARIES}) endif(OpenMPI) diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index c0755a3e..7ccc598c 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -1,13 +1,24 @@ -find_package(Protobuf REQUIRED) +find_package(Protobuf REQUIRED HINTS /opt/homebrew/Cellar) file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.proto3" "${CMAKE_CURRENT_BINARY_DIR}/onnx.proto") -PROTOBUF_GENERATE_CPP(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto) + +#protobuf_generate( +# TARGET gravity +# PROTOS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto +# OUT_VAR PROTO_GENERATED_FILES +# IMPORT_DIRS ${PROTO_IMPORT_DIRS} +# PROTOC_OUT_DIR "${PROTO_BINARY_DIR}") +#set_source_files_properties(${PROTO_GENERATED_FILES} PROPERTIES SKIP_UNITY_BUILD_INCLUSION on) + + +#PROTOBUF_GENERATE_CPP(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto) add_custom_target(proto_dep DEPENDS ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) find_package(PkgConfig QUIET REQUIRED) pkg_check_modules(PC_Protobuf protobuf) message(${PC_Protobuf_INCLUDEDIR}) +message(${PC_Protobuf_LIBRARIES}) include_directories(${PC_Protobuf_INCLUDEDIR} ${ONNX_PROTO_HDRS} ${ONNX_PROTO_SRCS}) @@ -43,7 +54,10 @@ add_executable( ${ONNX_PROTO_HDRS} ${NN_SOURCES} ) +#target_link_libraries(tighten absl::base absl::synchronization absl::strings) +#target_link_libraries(tighten ${ABSL_BASE_LIBRARY}) + add_dependencies(tighten proto_dep) target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(tighten ${PROTOBUF_LIBRARIES} gravity OpenMP::OpenMP_CXX CLI11) -target_link_libraries(tighten mpi) +target_link_libraries(tighten ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity OpenMP::OpenMP_CXX CLI11) +#target_link_libraries(tighten mpi) diff --git a/examples/MachineLearning/GravityNN/network/Tensor.hpp b/examples/MachineLearning/GravityNN/network/Tensor.hpp index 34f4eb63..04245d45 100644 --- a/examples/MachineLearning/GravityNN/network/Tensor.hpp +++ b/examples/MachineLearning/GravityNN/network/Tensor.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 4e6dec32..97dbaa59 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -84,13 +84,14 @@ double bound_neuron(std::string fname, std::string start_node, Bound neuron, con constexpr int MAX_LAYER_NAME_LEN = 100; constexpr int MAX_NEURON_NAME_LEN = 100; - -MPI_Datatype MPI_PROXY_BOUND_TYPE; const int nitems = 5; int blocklengths[5] = {MAX_LAYER_NAME_LEN, MAX_LAYER_NAME_LEN, 1, 1, 1}; -MPI_Datatype types[5] = {MPI_CHAR, MPI_CHAR, MPI_DOUBLE, MPI_DOUBLE, MPI_INT}; -MPI_Aint offsets[5]; +#ifdef USE_MPI +MPI_Datatype MPI_PROXY_BOUND_TYPE; +MPI_Datatype types[5] = {MPI_CHAR, MPI_CHAR, MPI_DOUBLE, MPI_DOUBLE, MPI_INT}; +#endif +int offsets[5]; class ProxyBound { public: char layer_name[MAX_LAYER_NAME_LEN]; @@ -118,9 +119,10 @@ class ProxyBound { void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& fname, const std::string& start_node, const std::vector& global_bounds) { int rank, size; +#ifdef USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - +#endif // Adjust how many neurons each rank handles size_t total_bounds = local_bounds.size(); size_t neurons_per_rank = total_bounds / size; @@ -152,15 +154,19 @@ void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& f for (int src_rank = 1; src_rank < size; src_rank++) { size_t src_start_idx = src_rank * neurons_per_rank + std::min(static_cast(src_rank), extra_neurons); size_t src_end_idx = src_start_idx + neurons_per_rank + (src_rank < extra_neurons); - +#ifdef USE_MPI MPI_Recv(&proxyLocalBounds[src_start_idx], src_end_idx - src_start_idx, MPI_PROXY_BOUND_TYPE, src_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); +#endif } } else { +#ifdef USE_MPI MPI_Send(&proxyLocalBounds[start_idx], end_idx - start_idx, MPI_PROXY_BOUND_TYPE, 0, 0, MPI_COMM_WORLD); +#endif } - +#ifdef USE_MPI // Broadcast the updated bounds back to all ranks MPI_Bcast(&proxyLocalBounds[0], proxyLocalBounds.size(), MPI_PROXY_BOUND_TYPE, 0, MPI_COMM_WORLD); +#endif // Convert back the received ProxyBounds to Bounds for (size_t i = 0; i < local_bounds.size(); ++i) { @@ -174,11 +180,12 @@ int main(int argc, char * argv[]) { fname = argv[1]; } - MPI_Init(nullptr, nullptr); int rank, size; +#ifdef USE_MPI + MPI_Init(nullptr, nullptr); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - +#endif NeuralNet nn(fname); std::vector layers_to_optimize; @@ -262,10 +269,10 @@ int main(int argc, char * argv[]) { offsets[2] = offsetof(ProxyBound, value); offsets[3] = offsetof(ProxyBound, old_value); offsets[4] = offsetof(ProxyBound, side); - +#ifdef USE_MPI MPI_Type_create_struct(nitems, blocklengths, offsets, types, &MPI_PROXY_BOUND_TYPE); MPI_Type_commit(&MPI_PROXY_BOUND_TYPE); - +#endif run_MPI_bound_neuron(local_bounds, fname, start_node, global_bounds); auto end_time = std::chrono::high_resolution_clock::now(); fflush(stdout); @@ -287,8 +294,9 @@ int main(int argc, char * argv[]) { } global_bounds.insert(global_bounds.end(), local_bounds.begin(), local_bounds.end()); } +#ifdef USE_MPI MPI_Type_free(&MPI_PROXY_BOUND_TYPE); - +#endif if (rank == 0) { std::cout << "Starting final runs" << std::endl; } @@ -306,7 +314,8 @@ int main(int argc, char * argv[]) { } // FIXME: gather final_run results - +#ifdef USE_MPI MPI_Finalize(); +#endif return 0; } From 982d6fbac29cf9f0afe3aee76d8613cdbb82f73a Mon Sep 17 00:00:00 2001 From: Haoruo Zhao Date: Tue, 15 Aug 2023 23:47:47 -0600 Subject: [PATCH 08/12] compiles on macbook --- .../MachineLearning/GravityNN/CMakeLists.txt | 26 ++++++++++++++----- .../GravityNN/network/Tensor.hpp | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index 7ccc598c..ffc73a52 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -1,4 +1,10 @@ + +find_package(PkgConfig QUIET REQUIRED) +pkg_check_modules(PC_Protobuf Protobuf) find_package(Protobuf REQUIRED HINTS /opt/homebrew/Cellar) +message(${PC_Protobuf_INCLUDEDIR}) +message(${PC_Protobuf_LIBRARIES}) + file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.proto3" "${CMAKE_CURRENT_BINARY_DIR}/onnx.proto") @@ -15,10 +21,6 @@ file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.prot add_custom_target(proto_dep DEPENDS ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -find_package(PkgConfig QUIET REQUIRED) -pkg_check_modules(PC_Protobuf protobuf) -message(${PC_Protobuf_INCLUDEDIR}) -message(${PC_Protobuf_LIBRARIES}) include_directories(${PC_Protobuf_INCLUDEDIR} ${ONNX_PROTO_HDRS} ${ONNX_PROTO_SRCS}) @@ -40,11 +42,23 @@ add_executable( ${NN_SOURCES} ) +find_library(ONNX_LIB + libonnx.a + HINTS /usr/local/lib + HINTS $ENV{HOME}/Utils/onnx/build + ) + +find_library(ONNX_PROTO_LIB + libonnx_proto.a + HINTS /usr/local/lib + HINTS $ENV{HOME}/Utils/onnx/build + ) + add_dependencies(GravityNN proto_dep) target_include_directories(GravityNN PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(GravityNN ${PROTOBUF_LIBRARIES} gravity CLI11) +target_link_libraries(GravityNN ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) find_package(OpenMP) add_executable( @@ -59,5 +73,5 @@ add_executable( add_dependencies(tighten proto_dep) target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(tighten ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity OpenMP::OpenMP_CXX CLI11) +target_link_libraries(tighten ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) #target_link_libraries(tighten mpi) diff --git a/examples/MachineLearning/GravityNN/network/Tensor.hpp b/examples/MachineLearning/GravityNN/network/Tensor.hpp index 04245d45..34f4eb63 100644 --- a/examples/MachineLearning/GravityNN/network/Tensor.hpp +++ b/examples/MachineLearning/GravityNN/network/Tensor.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include From 5f9d30c73b998ad1f9c4d87feb11b3031dfc7739 Mon Sep 17 00:00:00 2001 From: "Zhao, Haoruo" Date: Wed, 16 Aug 2023 01:23:27 -0600 Subject: [PATCH 09/12] compiled on M2, links ok --- .../MachineLearning/GravityNN/CMakeLists.txt | 22 ++++++++++++------- .../GravityNN/utils/subgraph_extraction.hpp | 4 ++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index ffc73a52..37df35aa 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -1,12 +1,12 @@ find_package(PkgConfig QUIET REQUIRED) pkg_check_modules(PC_Protobuf Protobuf) -find_package(Protobuf REQUIRED HINTS /opt/homebrew/Cellar) +find_package(Protobuf REQUIRED HINTS /usr/local/) message(${PC_Protobuf_INCLUDEDIR}) message(${PC_Protobuf_LIBRARIES}) -file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.proto3" "${CMAKE_CURRENT_BINARY_DIR}/onnx.proto") +#file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.proto3" "${CMAKE_CURRENT_BINARY_DIR}/onnx.proto") #protobuf_generate( # TARGET gravity @@ -18,8 +18,8 @@ file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.prot #PROTOBUF_GENERATE_CPP(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto) -add_custom_target(proto_dep DEPENDS ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) +#add_custom_target(proto_dep DEPENDS ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS}) +#include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${PC_Protobuf_INCLUDEDIR} ${ONNX_PROTO_HDRS} ${ONNX_PROTO_SRCS}) @@ -54,11 +54,17 @@ find_library(ONNX_PROTO_LIB HINTS $ENV{HOME}/Utils/onnx/build ) -add_dependencies(GravityNN proto_dep) +find_library(PROTOBUF_LIB + libprotobuf.a + HINTS /usr/local/lib + HINTS $ENV{HOME}/Utils/onnx/build +) + +#add_dependencies(GravityNN proto_dep) target_include_directories(GravityNN PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(GravityNN ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) +target_link_libraries(GravityNN ${PROTOBUF_LIB} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) find_package(OpenMP) add_executable( @@ -71,7 +77,7 @@ add_executable( #target_link_libraries(tighten absl::base absl::synchronization absl::strings) #target_link_libraries(tighten ${ABSL_BASE_LIBRARY}) -add_dependencies(tighten proto_dep) +#add_dependencies(tighten proto_dep) target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(tighten ${PROTOBUF_LIBRARIES} ${PC_Protobuf_LIBRARIES} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) +target_link_libraries(tighten ${PROTOBUF_LIB} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) #target_link_libraries(tighten mpi) diff --git a/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp b/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp index 3a58660b..c85f6325 100644 --- a/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp +++ b/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -175,4 +175,4 @@ std::vector subgraph_extraction(std::vector layers, std::string } return subgraph; -} \ No newline at end of file +} From f03bea4be0bf5b1c2f979c57d9a16955d6b45290 Mon Sep 17 00:00:00 2001 From: Haydn Jones Date: Thu, 17 Aug 2023 17:11:27 -0400 Subject: [PATCH 10/12] Hopefully have fixed protobuf issues --- .../MachineLearning/GravityNN/CMakeLists.txt | 74 +++---------------- .../MachineLearning/GravityNN/GravityNN.cpp | 2 +- .../MachineLearning/GravityNN/cmake/CPM.cmake | 33 +++++++++ .../GravityNN/network/Layers/LayerBase.hpp | 2 +- .../GravityNN/network/NeuralNet.hpp | 2 +- .../GravityNN/network/Tensor.hpp | 2 +- .../MachineLearning/GravityNN/tighten.cpp | 2 +- .../GravityNN/utils/subgraph_extraction.hpp | 2 +- 8 files changed, 51 insertions(+), 68 deletions(-) create mode 100644 examples/MachineLearning/GravityNN/cmake/CPM.cmake diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index 37df35aa..18b78f0f 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -1,39 +1,14 @@ - -find_package(PkgConfig QUIET REQUIRED) -pkg_check_modules(PC_Protobuf Protobuf) -find_package(Protobuf REQUIRED HINTS /usr/local/) -message(${PC_Protobuf_INCLUDEDIR}) -message(${PC_Protobuf_LIBRARIES}) - - -#file(DOWNLOAD "https://raw.githubusercontent.com/onnx/onnx/master/onnx/onnx.proto3" "${CMAKE_CURRENT_BINARY_DIR}/onnx.proto") - -#protobuf_generate( -# TARGET gravity -# PROTOS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto -# OUT_VAR PROTO_GENERATED_FILES -# IMPORT_DIRS ${PROTO_IMPORT_DIRS} -# PROTOC_OUT_DIR "${PROTO_BINARY_DIR}") -#set_source_files_properties(${PROTO_GENERATED_FILES} PROPERTIES SKIP_UNITY_BUILD_INCLUSION on) - - -#PROTOBUF_GENERATE_CPP(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${CMAKE_CURRENT_BINARY_DIR}/onnx.proto) -#add_custom_target(proto_dep DEPENDS ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS}) -#include_directories(${CMAKE_CURRENT_BINARY_DIR}) - - -include_directories(${PC_Protobuf_INCLUDEDIR} ${ONNX_PROTO_HDRS} ${ONNX_PROTO_SRCS}) +include(${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/cmake/CPM.cmake) include(FetchContent) -FetchContent_Declare( - cli11 - GIT_REPOSITORY https://github.com/CLIUtils/CLI11 - GIT_TAG v2.3.2 +CPMAddPackage("gh:CLIUtils/CLI11#v2.3.2") +CPMAddPackage("gh:onnx/onnx#v1.14.0") + +file(GLOB NN_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/*.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers/*.hpp ) -file(GLOB NN_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers/*.hpp) -FetchContent_MakeAvailable(cli11) -message(${cli11_SOURCE_DIR}) add_executable( GravityNN MachineLearning/GravityNN/GravityNN.cpp @@ -42,42 +17,17 @@ add_executable( ${NN_SOURCES} ) -find_library(ONNX_LIB - libonnx.a - HINTS /usr/local/lib - HINTS $ENV{HOME}/Utils/onnx/build - ) - -find_library(ONNX_PROTO_LIB - libonnx_proto.a - HINTS /usr/local/lib - HINTS $ENV{HOME}/Utils/onnx/build - ) - -find_library(PROTOBUF_LIB - libprotobuf.a - HINTS /usr/local/lib - HINTS $ENV{HOME}/Utils/onnx/build -) - -#add_dependencies(GravityNN proto_dep) - - target_include_directories(GravityNN PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(GravityNN ${PROTOBUF_LIB} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) +target_link_libraries(GravityNN gravity CLI11 onnx) find_package(OpenMP) add_executable( tighten MachineLearning/GravityNN/tighten.cpp - ${ONNX_PROTO_SRCS} - ${ONNX_PROTO_HDRS} - ${NN_SOURCES} + ${ONNX_PROTO_SRCS} + ${ONNX_PROTO_HDRS} + ${NN_SOURCES} ) -#target_link_libraries(tighten absl::base absl::synchronization absl::strings) -#target_link_libraries(tighten ${ABSL_BASE_LIBRARY}) -#add_dependencies(tighten proto_dep) target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) -target_link_libraries(tighten ${PROTOBUF_LIB} gravity CLI11 ${ONNX_LIB} ${ONNX_PROTO_LIB}) -#target_link_libraries(tighten mpi) +target_link_libraries(tighten gravity CLI11 onnx) diff --git a/examples/MachineLearning/GravityNN/GravityNN.cpp b/examples/MachineLearning/GravityNN/GravityNN.cpp index ce04df70..97deec73 100644 --- a/examples/MachineLearning/GravityNN/GravityNN.cpp +++ b/examples/MachineLearning/GravityNN/GravityNN.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/cmake/CPM.cmake b/examples/MachineLearning/GravityNN/cmake/CPM.cmake new file mode 100644 index 00000000..f49d7434 --- /dev/null +++ b/examples/MachineLearning/GravityNN/cmake/CPM.cmake @@ -0,0 +1,33 @@ +set(CPM_DOWNLOAD_VERSION 0.38.2) + +if(CPM_SOURCE_CACHE) + set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +elseif(DEFINED ENV{CPM_SOURCE_CACHE}) + set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +else() + set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +endif() + +# Expand relative path. This is important if the provided path contains a tilde (~) +get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE) + +function(download_cpm) + message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}") + file(DOWNLOAD + https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake + ${CPM_DOWNLOAD_LOCATION} + ) +endfunction() + +if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION})) + download_cpm() +else() + # resume download if it previously failed + file(READ ${CPM_DOWNLOAD_LOCATION} check) + if("${check}" STREQUAL "") + download_cpm() + endif() + unset(check) +endif() + +include(${CPM_DOWNLOAD_LOCATION}) diff --git a/examples/MachineLearning/GravityNN/network/Layers/LayerBase.hpp b/examples/MachineLearning/GravityNN/network/Layers/LayerBase.hpp index 5ab267d3..46cc237d 100644 --- a/examples/MachineLearning/GravityNN/network/Layers/LayerBase.hpp +++ b/examples/MachineLearning/GravityNN/network/Layers/LayerBase.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/network/NeuralNet.hpp b/examples/MachineLearning/GravityNN/network/NeuralNet.hpp index c42660b8..a63b1223 100644 --- a/examples/MachineLearning/GravityNN/network/NeuralNet.hpp +++ b/examples/MachineLearning/GravityNN/network/NeuralNet.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/network/Tensor.hpp b/examples/MachineLearning/GravityNN/network/Tensor.hpp index 34f4eb63..e0672088 100644 --- a/examples/MachineLearning/GravityNN/network/Tensor.hpp +++ b/examples/MachineLearning/GravityNN/network/Tensor.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index 97dbaa59..ad0a4b7c 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp b/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp index c85f6325..b8ab2754 100644 --- a/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp +++ b/examples/MachineLearning/GravityNN/utils/subgraph_extraction.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include From 51151fc3ff138aed34e6df7c2814d067862195ee Mon Sep 17 00:00:00 2001 From: "Zhao, Haoruo" Date: Mon, 21 Aug 2023 12:19:54 -0400 Subject: [PATCH 11/12] mpi config --- CMakeLists.txt | 6 ++++-- examples/MachineLearning/GravityNN/CMakeLists.txt | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bc95d09..773138a4 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,8 +81,10 @@ option(OpenMPI "Link to OpenMPI libraries" OFF) option(OPT_PARSER "Link to OPT_PARSER libraries" OFF) if (OpenMPI) - SET(CMAKE_CXX_COMPILER mpicxx) - SET(CMAKE_C_COMPILER mpicc) + find_program(MPICXX NAMES mpicxx HINTS /usr/local/bin) + find_program(MPICC NAMES mpicc HINTS /usr/local/bin) + SET(CMAKE_CXX_COMPILER ${MPICXX}) + SET(CMAKE_C_COMPILER ${MPICC}) message("Compilers used: ${CMAKE_C_COMPILER} and ${CMAKE_CXX_COMPILER}") message(STATUS "Enable OpenMPI") add_definitions(-DUSE_MPI) diff --git a/examples/MachineLearning/GravityNN/CMakeLists.txt b/examples/MachineLearning/GravityNN/CMakeLists.txt index 18b78f0f..3c2f09b8 100644 --- a/examples/MachineLearning/GravityNN/CMakeLists.txt +++ b/examples/MachineLearning/GravityNN/CMakeLists.txt @@ -31,3 +31,7 @@ add_executable( target_include_directories(tighten PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network ${CMAKE_CURRENT_SOURCE_DIR}/MachineLearning/GravityNN/network/Layers ${ONNX_PROTO_HDRS} ${cli11_SOURCE_DIR}/include) target_link_libraries(tighten gravity CLI11 onnx) +if (OpenMPI) + include_directories(${MPI_INCLUDE_PATH}) + target_link_libraries(tighten ${MPI_LIBRARIES}) +endif(OpenMPI) \ No newline at end of file From 0fbd049dbbd7af6211e3e298d6b30a0228f17c12 Mon Sep 17 00:00:00 2001 From: "Zhao, Haoruo" Date: Mon, 21 Aug 2023 12:56:36 -0400 Subject: [PATCH 12/12] clean up tighten --- .../MachineLearning/GravityNN/tighten.cpp | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/examples/MachineLearning/GravityNN/tighten.cpp b/examples/MachineLearning/GravityNN/tighten.cpp index ad0a4b7c..ccbb1310 100644 --- a/examples/MachineLearning/GravityNN/tighten.cpp +++ b/examples/MachineLearning/GravityNN/tighten.cpp @@ -45,9 +45,9 @@ void final_run(std::string fname, const std::vector& global_bounds, size_ } double bound_neuron(std::string fname, std::string start_node, Bound neuron, const std::vector& global_bounds) { - // char hostname[128]; - // gethostname(hostname, sizeof(hostname)); - // std::cout << "Running bound_neuron on host: " << hostname << " with process ID: " << getpid() << " bounding " << neuron.side << " of neuron " << neuron.neuron_name << std::endl; +// char hostname[128]; +// gethostname(hostname, sizeof(hostname)); +// std::cout << "Running bound_neuron on host: " << hostname << " with process ID: " << getpid() << " bounding " << neuron.side << " of neuron " << neuron.neuron_name << std::endl; NeuralNet nn(fname); nn.set_aux_bounds(global_bounds); @@ -90,8 +90,7 @@ int blocklengths[5] = {MAX_LAYER_NAME_LEN, MAX_LAYER_NAME_LEN, 1, 1, 1}; #ifdef USE_MPI MPI_Datatype MPI_PROXY_BOUND_TYPE; MPI_Datatype types[5] = {MPI_CHAR, MPI_CHAR, MPI_DOUBLE, MPI_DOUBLE, MPI_INT}; -#endif -int offsets[5]; +MPI_Aint offsets[5]; class ProxyBound { public: char layer_name[MAX_LAYER_NAME_LEN]; @@ -116,23 +115,26 @@ class ProxyBound { return Bound(std::string(layer_name), std::string(neuron_name), value, old_value, side); } }; +#endif void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& fname, const std::string& start_node, const std::vector& global_bounds) { - int rank, size; -#ifdef USE_MPI + int rank = 0, size = 1; // Default values for non-MPI case + size_t start_idx = 0; // Default start index + size_t end_idx = local_bounds.size(); // Default end index + + #ifdef USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); -#endif // Adjust how many neurons each rank handles size_t total_bounds = local_bounds.size(); size_t neurons_per_rank = total_bounds / size; size_t extra_neurons = total_bounds % size; - size_t start_idx = rank * neurons_per_rank + std::min(static_cast(rank), extra_neurons); - size_t end_idx = start_idx + neurons_per_rank + (rank < extra_neurons); + start_idx = rank * neurons_per_rank + std::min(static_cast(rank), extra_neurons); + end_idx = start_idx + neurons_per_rank + (rank < extra_neurons); + #endif // Compute bounds for local neurons - #pragma omp parallel for for (size_t i = start_idx; i < end_idx; i++) { Bound& neuron = local_bounds[i]; auto new_bound = bound_neuron(fname, start_node, neuron, global_bounds); @@ -143,7 +145,9 @@ void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& f } } + // Convert local_bounds to ProxyBound for MPI communication + #ifdef USE_MPI std::vector proxyLocalBounds; for (const auto& bound : local_bounds) { proxyLocalBounds.emplace_back(bound); @@ -154,24 +158,20 @@ void run_MPI_bound_neuron(std::vector& local_bounds, const std::string& f for (int src_rank = 1; src_rank < size; src_rank++) { size_t src_start_idx = src_rank * neurons_per_rank + std::min(static_cast(src_rank), extra_neurons); size_t src_end_idx = src_start_idx + neurons_per_rank + (src_rank < extra_neurons); -#ifdef USE_MPI MPI_Recv(&proxyLocalBounds[src_start_idx], src_end_idx - src_start_idx, MPI_PROXY_BOUND_TYPE, src_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); -#endif } } else { -#ifdef USE_MPI MPI_Send(&proxyLocalBounds[start_idx], end_idx - start_idx, MPI_PROXY_BOUND_TYPE, 0, 0, MPI_COMM_WORLD); -#endif } -#ifdef USE_MPI // Broadcast the updated bounds back to all ranks MPI_Bcast(&proxyLocalBounds[0], proxyLocalBounds.size(), MPI_PROXY_BOUND_TYPE, 0, MPI_COMM_WORLD); -#endif + // Convert back the received ProxyBounds to Bounds for (size_t i = 0; i < local_bounds.size(); ++i) { local_bounds[i] = proxyLocalBounds[i].toBound(); } + #endif } int main(int argc, char * argv[]) { @@ -180,12 +180,12 @@ int main(int argc, char * argv[]) { fname = argv[1]; } - int rank, size; -#ifdef USE_MPI + int rank = 0, size = 1; // Default values for non-MPI case + #ifdef USE_MPI MPI_Init(nullptr, nullptr); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); -#endif + #endif NeuralNet nn(fname); std::vector layers_to_optimize; @@ -264,15 +264,16 @@ int main(int argc, char * argv[]) { } // Use the run_MPI_bound_neuron function for parallel neuron bound calculation + #ifdef USE_MPI offsets[0] = offsetof(ProxyBound, layer_name); offsets[1] = offsetof(ProxyBound, neuron_name); offsets[2] = offsetof(ProxyBound, value); offsets[3] = offsetof(ProxyBound, old_value); offsets[4] = offsetof(ProxyBound, side); -#ifdef USE_MPI + MPI_Type_create_struct(nitems, blocklengths, offsets, types, &MPI_PROXY_BOUND_TYPE); MPI_Type_commit(&MPI_PROXY_BOUND_TYPE); -#endif + #endif run_MPI_bound_neuron(local_bounds, fname, start_node, global_bounds); auto end_time = std::chrono::high_resolution_clock::now(); fflush(stdout);