Merge pull request #212 from jtwhite79/feat_ftx

improvements to file transfer info in rmr file

benchmarks/basic_tests.py

@@ -1007,17 +1007,17 @@ def mf6_v5_sen_test():
     msngrp_file = msn_file.replace(".msn",".group.msn")
     assert os.path.exists(msngrp_file),msngrp_file
 
-    jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("hostname") and f.lower().endswith(".jcb")]
+    jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".jcb")]
     print(len(jcb_files))
     assert len(jcb_files) == 52
     for jcb_file in jcb_files:
         j = pyemu.Jco.from_binary(os.path.join(m_d,jcb_file))
 
-    lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("hostname") and f.lower().endswith(".lst")]
+    lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".lst")]
     print(len(lst_files))
     assert len(lst_files) == 52
 
-    cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("hostname") and f.lower().endswith(".cbc")]
+    cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx") and f.lower().endswith(".cbc")]
     print(len(cbc_files))
     assert len(cbc_files) == 52
 
@@ -1294,7 +1294,7 @@ def ins_missing_e_test():
     #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
     #tplins1_test()
     #mf6_v5_ies_test()
-    #mf6_v5_sen_test()
+    mf6_v5_sen_test()
 
 
 
@@ -1306,5 +1306,5 @@ def ins_missing_e_test():
     #basic_sqp_test()
     #mf6_v5_ies_test()
     #fr_timeout_test()
-    fr_fail_test()
+    #fr_fail_test()
     #tplins1_test()

src/libs/common/config_os.h

@@ -2,7 +2,7 @@
 #define CONFIG_OS_H_
 
 
-#define PESTPP_VERSION "5.1.23";
+#define PESTPP_VERSION "5.1.24";
 
 #if defined(_WIN32) || defined(_WIN64)
 #define OS_WIN

src/libs/pestpp_common/Ensemble.cpp

@@ -3633,18 +3633,48 @@ void ObservationEnsemble::draw(int num_reals, Covariance &cov, PerformanceLog *p
 	ObservationInfo oi = pest_scenario_ptr->get_ctl_observation_info();
 	map<string, vector<string>> grouper;
 	vector<string> ogroups = pest_scenario_ptr->get_ctl_ordered_obs_group_names();
+	vector<string> vars_in_group;
+	vector<string> sorted_var_names;
 	if (pest_scenario_ptr->get_pestpp_options().get_ies_group_draws())
 	{
 		for (auto group : ogroups)
 		{
-			for (auto &oname : var_names)
-			{
-				if (oi.get_group(oname) == group)
-					grouper[group].push_back(oname);
-			}
-		}
+            vars_in_group.clear();
+            for (auto &oname : var_names)
+            {
+                if (oi.get_group(oname) == group)
+                    vars_in_group.push_back(oname);
+            }
+            if (vars_in_group.size() == 0)
+                continue;
+            sort(vars_in_group.begin(), vars_in_group.end());
+            sorted_var_names.insert(sorted_var_names.end(), vars_in_group.begin(), vars_in_group.end());
+            grouper[group] = vars_in_group;
+        }
 	}
-	Ensemble::draw(num_reals, cov, obs, pest_scenario_ptr->get_ctl_ordered_nz_obs_names(), grouper, plog, level);
+
+    //check
+    bool same = true;
+    if (var_names.size() == sorted_var_names.size()) {
+        //throw_ensemble_error("sorted obs names not equal to org obs names");
+        for (int i = 0; i < var_names.size(); i++)
+            if (var_names[i] != sorted_var_names[i]) {
+                same = false;
+                break;
+            }
+    }
+    if (!same)
+    {
+        plog->log_event("observations not grouped by observation groups, reordering obs ensemble");
+        cout << "observations not grouped by observations groups, reordering obs ensemble" << endl;
+        var_names = sorted_var_names;
+    }
+    else
+    {
+        sorted_var_names = var_names;
+    }
+
+	Ensemble::draw(num_reals, cov, obs, sorted_var_names, grouper, plog, level);
 
 	//apply any bounds that were supplied
 	map<string, double> lower_bnd = pest_scenario_ptr->get_ext_file_double_map("observation data external", "lower_bound");

src/libs/pestpp_common/EnsembleMethodUtils.cpp

@@ -2115,6 +2115,11 @@ string L2PhiHandler::get_summary_header()
 	return ss.str();
 }
 
+bool cmp_pair(pair<string,double>& first, pair<string,double>& second)
+{
+    return first.second > second.second;
+}
+
 
 void L2PhiHandler::report_group(bool echo) {
 
@@ -2204,6 +2209,7 @@ void L2PhiHandler::report_group(bool echo) {
     ss.str("");
     ss << "  ---  observation group phi summary ---  " << endl;
     ss << "       (computed using 'actual' phi)" << endl;
+    ss << "           (sorted by mean phi)" << endl;
     ss << left << setw(len) << "group" << right << setw(15) << "mean" << setw(15) << "std";
     ss << setw(15) << "min" << setw(15) << "max";
     ss << setw(15) << "percent" << setw(15) << "std" << setw(15) << "min" << setw(15) << "max" << endl;
@@ -2212,9 +2218,19 @@ void L2PhiHandler::report_group(bool echo) {
         cout << ss.str();
 
     vector<string> nzgroups(snzgroups.begin(),snzgroups.end());
-    sort(nzgroups.begin(),nzgroups.end());
-    for (auto& g : nzgroups)
+    //sort(nzgroups.begin(),nzgroups.end());
+    vector<pair<string,double>> pairs;
+    for (auto& it : mn_map)
+        pairs.push_back(it);
+
+
+    sort(pairs.begin(),pairs.end(),cmp_pair);
+
+    c = 0;
+    string g;
+    for (auto& pair : pairs)
     {
+        g = pair.first;
         ss.str("");
         ss << left << setw(len) << pest_utils::lower_cp(g);
         ss << right << setw(15) << mn_map[g];
@@ -2229,12 +2245,16 @@ void L2PhiHandler::report_group(bool echo) {
         ss << endl;
 
         f << ss.str();
-        if (echo)
+        if ((echo) && (c <= 10))
             cout << ss.str();
+        c++;
     }
     f << "    Note: 'percent' is the percentage of the actual phi for each realization." << endl << endl;
     if (echo)
-        cout << "    Note: 'percent' is the percentage of the actual phi for each realization." << endl << endl;
+        cout << "    Note: 'percent' is the percentage of the actual phi for each realization." << endl;
+    if (c > 10)
+        cout << "    Note: only the first 10 highest mean phi groups are listed. See rec file for full listing" << endl;
+    cout << endl;
 }
 
 void L2PhiHandler::report(bool echo, bool group_report)
@@ -4020,19 +4040,26 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 				message(1, ss.str());
 				message(1, " the realization names are compatible");
 				message(1, "re-indexing obs+noise en to align with par en...");
-//				cout << "oe names: " << endl;
-//				for (auto& name : oe_names)
-//					cout << name << endl;
-//				oe_names = pe.get_real_names();
-//				cout << endl << "pe names: " << endl;
-//				for (auto& name : oe_names)
-//					cout << name << endl;
-				oe_base.reorder(pe.get_real_names(), vector<string>());
-				//oe_names = oe_base.get_real_names();
-//				cout << "new oe names: " << endl;
-//				for (auto& name : oe_names)
-//					cout << name << endl;
-//				cout << endl;
+                if (verbose_level > 1) {
+                    cout << "oe names: " << endl;
+                    for (auto &name : oe_names)
+                        cout << name << endl;
+                    oe_names = pe.get_real_names();
+                    cout << endl << "pe names: " << endl;
+                    for (auto &name : oe_names)
+                        cout << name << endl;
+                }
+
+                    oe_base.reorder(pe.get_real_names(), vector<string>());
+
+                if (verbose_level > 1)
+                {
+                    oe_names = oe_base.get_real_names();
+                    cout << "new oe names: " << endl;
+                    for (auto &name : oe_names)
+                        cout << name << endl;
+                    cout << endl;
+                }
 			}
 			else
 			{

src/libs/run_managers/yamr/PantherAgent.cpp

@@ -284,7 +284,7 @@ void PANTHERAgent::transfer_files(const vector<string>& tfiles, int group, int r
             in.close();
             continue;
         }
-        string filename_desc = desc + " agent_filename="+filename + " " + tag;
+        string filename_desc = desc + " agent_filename:"+filename + " " + tag;
 
         pack = NetPackage(NetPackage::PackType::START_FILE_WRKR2MSTR,group,run_id,filename_desc);
         send_message(pack);
@@ -1091,7 +1091,7 @@ void PANTHERAgent::start_impl(const string &host, const string &port)
 				ss << "results of run_id " << run_id << " sent successfully";
 				report(ss.str(), true);
                 transfer_files(pest_scenario.get_pestpp_options().get_panther_transfer_on_finish(), group_id,
-                               run_id,info_txt, "run_status=completed");
+                               run_id,info_txt, "run_status:completed");
 
             }
 			else if (final_run_status.first == NetPackage::PackType::RUN_FAILED)
@@ -1112,7 +1112,7 @@ void PANTHERAgent::start_impl(const string &host, const string &port)
 					terminate_or_restart(-1);
 				}
                 transfer_files(pest_scenario.get_pestpp_options().get_panther_transfer_on_fail(), group_id,
-                               run_id,info_txt, "run_status=failed");
+                               run_id,info_txt, "run_status:failed");
 				if (pest_scenario.get_pestpp_options().get_panther_debug_fail_freeze())
 				{
 					ss.str("");
@@ -1158,7 +1158,7 @@ void PANTHERAgent::start_impl(const string &host, const string &port)
 					terminate_or_restart(-1);
 				}
                 transfer_files(pest_scenario.get_pestpp_options().get_panther_transfer_on_fail(), group_id,
-                               run_id,info_txt, "run_status=failed");
+                               run_id,info_txt, "run_status:failed");
 			}
 
 			else if (final_run_status.first == NetPackage::PackType::CORRUPT_MESG)