Merge remote-tracking branch 'upstream/develop'

benchmarks/basic_tests.py

@@ -1198,7 +1198,7 @@ def fr_timeout_test():
     with open(os.path.join(new_d,"run.py"),'w') as f:
         f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n")
         f.write("if not os.path.exists('run.info'):\n    exit()\n")
-        f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split('=')[-1])\n")
+        f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n")
         f.write("if rnum % 2 == 0:\n    time.sleep(10000000)\n")
     pst.model_command = "python run.py"
     oe_file = os.path.join(new_d, "pest.0.obs.csv")
@@ -1285,7 +1285,7 @@ def ins_missing_e_test():
 
     #da_prep_4_mf6_freyberg_seq_tbl()
     #da_mf6_freyberg_test_2()
-    shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
+    #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe"))
     #tplins1_test()
     #mf6_v5_ies_test()
     #mf6_v5_sen_test()

documentation/pestpp_users_manual.md

@@ -1,7 +1,7 @@
 
  <img src="0d3cb7750c90b712af04ea3a51c8ecb968d784cc.png" style="width:6.26806in;height:1.68194in" alt="A close up of a purple sign Description automatically generated" />
 
-# <a id='s1' />Version 5.1.20
+# <a id='s1' />Version 5.1.21
 
 <img src="0e14ec9848f78a9809081572ca785af9990c2d38.png" style="width:6.26806in;height:3.05972in" />
 
@@ -70,7 +70,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 # Table of Contents
 
-- [Version 5.1.20](#s1)
+- [Version 5.1.21](#s1)
 - [Acknowledgements](#s2)
 - [Preface](#s3)
 - [License](#s4)
@@ -2998,11 +2998,11 @@ Note the following.
 
 PESTPP-SWP can fill in values for fixed and tied parameters if these are missing from its input file. Actually, it can provide values for other missing parameters as well if the *sweep_forgive()* control variable is set to *true*. These missing values are taken from the PEST control file which is read by PESTPP-SWP.
 
-PESTPP-SWP writes a model output file whose name is provided through the *sweep_output_csv_file()* control variable. If this variable is not provided, PESTPP-SWP employs the name *sweep_out.csv* for its output file.
+PESTPP-SWP writes a model output file whose name is provided through the *sweep_output_csv_file()* control variable. If this variable is not provided, PESTPP-SWP employs the name *sweep_out.csv* for its output file. This fiile contains input and output run identifiers, objective function information, and the simulated value for each control file “observation” for each run. Note that, by default, PESTPP-SWP will not calculate the contribution of prior information equations to the total objective function reported in the sweep output file. This can be overridden with the *sweep_include_regul_phi* option.
 
 The control variable *sweep_chunk()* pertains to parallelization of model runs. Runs are done in bundles of size *N*, where *N* is the value supplied for this variable. (A chunk of 500 is the default). This number should be chosen wisely. It should be a multiple of the number of agents that PESTPP-SWP can use for carrying out model runs.
 
-Also note that PESTPP-SWP can be particularly useful if users need complete model output files for a given set of runs. In this case, the file transfer capabilities of the parallel run manager can be used with PESTPP-SWP to run a sweep of parameter values and model output files can be transferred back to the master directory.
+PESTPP-SWP can be particularly useful if users need complete model output files for a given set of runs. In this case, the file transfer capabilities of the parallel run manager can be used with PESTPP-SWP to run a sweep of parameter values and model output files can be transferred back to the master directory.
 
 ## <a id='s14-3' />10.3 Summary of Control Variables
 
@@ -3020,6 +3020,7 @@ The number of control variables may change with time. Refer to the PEST++ web si
 | *enforce_tied_bounds(false)*             | Boolean  | Flag to enforce parameter bounds on any tied parameters                                                                                                                                                                                                 |
 | *tie_by_group(false)*                    | Boolean  | Flag to tie all adjustable parameters together within each parameter group. Initial parameter ratios are maintained as parameters are adjusted. Parameters that are designated as already tied, or that have parameters tied to them, are not affected. |
 | *ensemble_output_precision*              | int      | Number of significant digits to use in ASCII format ensemble files. Default is 6                                                                                                                                                                        |
+| *Sweep_include_regul_phi*                | Boolean  | Flag to include the contribution of prior information equations in the total objective function information reported in the sweep output file, Default is false.                                                                                        |
 
 Table 10.1 PESTPP-SWP control variables. Parallel run management variables can be supplied in addition to these; see section 5.3.6
 

etc/environment.yml

@@ -10,4 +10,4 @@ dependencies:
   - matplotlib
   - coverage
   - scipy
-  - shapely
+  - shapely>=1.8

src/libs/common/config_os.h

@@ -2,7 +2,7 @@
 #define CONFIG_OS_H_
 
 
-#define PESTPP_VERSION "5.1.20";
+#define PESTPP_VERSION "5.1.21";
 
 #if defined(_WIN32) || defined(_WIN64)
 #define OS_WIN

src/libs/common/network_package.cpp

@@ -45,6 +45,7 @@ string NetPackage::get_info_txt()
 {
 	string info_txt = extract_string(desc, DESC_LEN);
 	info_txt.erase(std::find(info_txt.begin(), info_txt.end(), '\0'), info_txt.end());
+    std::transform(info_txt.begin(), info_txt.end(), info_txt.begin(), ::tolower);
 	return info_txt;
 }
 

src/libs/pestpp_common/Ensemble.cpp

@@ -2195,7 +2195,7 @@ map<int,int> ParameterEnsemble::add_runs(RunManagerAbstract *run_mgr_ptr,const v
 	//if (da_cycle != NetPackage::NULL_DA_CYCLE)
 	{
 		stringstream ss;
-		ss << " da_cycle=" << da_cycle << " ";
+		ss << " da_cycle:" << da_cycle << " ";
 		info_txt = ss.str();
 	}
 	for (auto &rname : run_real_names)
@@ -2220,7 +2220,7 @@ map<int,int> ParameterEnsemble::add_runs(RunManagerAbstract *run_mgr_ptr,const v
 				ss << n << ",";
 			throw_ensemble_error(ss.str());
 		}
-		run_id = run_mgr_ptr->add_run(pars_real,info_txt+" realization="+rname);
+		run_id = run_mgr_ptr->add_run(pars_real,info_txt+"  realization:"+rname);
 		real_run_ids[idx]  = run_id;
 	}
 	return real_run_ids;

src/libs/pestpp_common/Ensemble.h

@@ -147,6 +147,7 @@ class FixedParInfo
 	void update_par_values(const map<string, double>& pval_map);
 	void clear() { fixed_info.clear(); fixed_names.clear(); }
 	void fill_fixed(map<string, double>& fixed_map, vector<string>& rnames);
+	int get_map_size() {return fixed_info.size();}
 private:
 	bool initialized;
 	vector<string> fixed_names;

src/libs/pestpp_common/EnsembleMethodUtils.cpp

@@ -3675,7 +3675,8 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 	else
 	{
 		message(1, "initializing localizer");
-		use_localizer = localizer.initialize(performance_log);
+		bool forgive_missing = pest_scenario.get_pestpp_options().get_ies_localizer_forgive_missing();
+		use_localizer = localizer.initialize(performance_log, forgive_missing);
 	}
 	num_threads = pest_scenario.get_pestpp_options().get_ies_num_threads();
 	if (!use_localizer)
@@ -3922,20 +3923,23 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 			frec << ma << endl;
 	}
 
+    int num_par_elements = pest_scenario.get_ctl_parameters().size();
 	if ((pp_args.find("IES_ORDERED_BINARY") == pp_args.end()) && (pp_args.find("DA_ORDERED_BINARY") == pp_args.end()))
 	{
-		if ((pe.shape().second > 100000) && (pest_scenario.get_pestpp_options().get_save_binary()))
+
+		//if ((pe.shape().second > 100000) && (pest_scenario.get_pestpp_options().get_save_binary()))
+        if ((num_par_elements > 100000) && (pest_scenario.get_pestpp_options().get_save_binary()))
 		{
-			message(1, "'ies_ordered_binary' was not passed, but 'ies_save_binary' is true and num adj pars > 100,000, switching to unordered binary...");
+			message(1, "'ies_ordered_binary' was not passed, but 'ies_save_binary' is true and npar > 100,000, switching to unordered binary...");
 			pest_scenario.get_pestpp_options_ptr()->set_ies_ordered_binary(false);
 		}
 	}
 
 	if ((pp_args.find("IES_UPGRADES_IN_MEMORY") == pp_args.end()) && (pp_args.find("DA_UPGRADES_IN_MEMORY") == pp_args.end()))
 	{
-		if (pe.shape().second > 100000)
+		if (num_par_elements > 100000)
 		{
-			message(1, "'ies_upgrades_in_memory' was not passed, but num adj pars > 100,000, switching ies_upgrades_in_memory to false...");
+			message(1, "'ies_upgrades_in_memory' was not passed, but npar > 100,000, switching ies_upgrades_in_memory to false...");
 			pest_scenario.get_pestpp_options_ptr()->set_ies_upgrades_in_memory(false);
 		}
 	}
@@ -4243,6 +4247,8 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 			if (ppo->get_ies_localizer().size() > 0)
 			{
 				message(1, "updating localizer");
+				if (localizer.get_use())
+				    localizer.get_orgmat_ptr()->clear_names();
 				use_localizer = localizer.initialize(performance_log, true);
 			}
 

src/libs/pestpp_common/Jacobian.cpp

@@ -280,7 +280,7 @@ bool Jacobian::build_runs(Parameters &ctl_pars, Observations &ctl_obs, vector<st
 	debug_msg("Jacobian::build_runs method: begin");
 	// add base run
 	Parameters model_pars = par_transform.ctl2model_cp(ctl_pars);
-	int run_id = run_manager.add_run(model_pars, "__base__", 0);
+	int run_id = run_manager.add_run(model_pars, "par_name:__base__", 0);
 
 	if (!calc_init_obs) {
 		const Observations &observations = ctl_obs;
@@ -307,7 +307,7 @@ bool Jacobian::build_runs(Parameters &ctl_pars, Observations &ctl_obs, vector<st
 			{
 				numeric_parameters.update_rec(ipar_name, ipar_val);
 				Parameters model_parameters = par_transform.numeric2model_cp(numeric_parameters);
-				run_manager.add_run(model_parameters, ipar_name, ipar_val);
+				run_manager.add_run(model_parameters, "par_name:"+ipar_name, ipar_val);
 			}
 		}
 		else
@@ -353,7 +353,7 @@ bool Jacobian::process_runs(ParamTransformSeq &par_transform,
 		bool debug_fail)
 {
 	// calculate jacobian
-  base_sim_obs_names = run_manager.get_obs_name_vec();
+    base_sim_obs_names = run_manager.get_obs_name_vec();
 	vector<string> prior_info_name = prior_info.get_keys();
 	base_sim_obs_names.insert(base_sim_obs_names.end(), prior_info_name.begin(), prior_info_name.end());
 	std::vector<Eigen::Triplet<double> > triplet_list;
@@ -388,7 +388,9 @@ bool Jacobian::process_runs(ParamTransformSeq &par_transform,
 	{
 		run_list.push_back(JacobianRun());
 				run_manager. get_info(i_run, r_status, cur_par_name, cur_numeric_par_value);
-		run_manager.get_model_parameters(i_run,  run_list.back().ctl_pars);
+            //this is to strip off the "par_name:" tag
+            cur_par_name = cur_par_name.substr(9,cur_par_name.size());
+            run_manager.get_model_parameters(i_run,  run_list.back().ctl_pars);
 			bool success = run_manager.get_observations_vec(i_run, run_list.back().obs_vec);
 			if ((debug_fail) && (i_run == 1))
 			{
@@ -410,6 +412,8 @@ bool Jacobian::process_runs(ParamTransformSeq &par_transform,
 		if (i_run+1<nruns)
 		{
 			run_manager.get_info(i_run+1, run_status_next, par_name_next, par_value_next);
+			//again the par_name: tag
+			par_name_next = par_name_next.substr(9,par_name_next.size());
 		}
 
 		if (i_run + 1 >= nruns || (cur_par_name != par_name_next))

src/libs/pestpp_common/Jacobian_1to1.cpp

@@ -63,7 +63,7 @@ bool Jacobian_1to1::build_runs(Parameters &ctl_pars, Observations &ctl_obs, vect
 	failed_ctl_parameters.clear();
 	par_run_map.clear();
 	// add base run
-	int run_id = run_manager.add_run(model_parameters, "__base__", 0);
+	int run_id = run_manager.add_run(model_parameters, "par_name:__base__", 0);
 	par_run_map["__base__"] = vector<int>{ run_id };
 	//if base run is has already been complete, update it and mark it as complete
 	// compute runs for to jacobain calculation as it is influenced by derivative type( forward or central)
@@ -97,7 +97,7 @@ bool Jacobian_1to1::build_runs(Parameters &ctl_pars, Observations &ctl_obs, vect
 				{
 					model_parameters[ipar.first] = ipar.second;
 				}
-				int id = run_manager.add_run(model_parameters, i_name, par);
+				int id = run_manager.add_run(model_parameters, "par_name:"+i_name, par);
 				if (par_run_map.count(i_name) == 0)
 					par_run_map[i_name] = vector<int>{ id };
 				else
@@ -143,7 +143,7 @@ bool Jacobian_1to1::build_runs(ModelRun &init_model_run, vector<string> numeric_
 	failed_ctl_parameters.clear();
 	par_run_map.clear();
 	// add base run
-	int run_id = run_manager.add_run(model_parameters, "", 0);
+	int run_id = run_manager.add_run(model_parameters, "par_name:__base__", 0);
 	par_run_map["__base__"] = vector<int>{ run_id };
 	//if base run is has already been complete, update it and mark it as complete
 	// compute runs for to jacobain calculation as it is influenced by derivative type( forward or central)
@@ -177,7 +177,7 @@ bool Jacobian_1to1::build_runs(ModelRun &init_model_run, vector<string> numeric_
 				{
 					model_parameters[ipar.first] = ipar.second;
 				}
-				int id = run_manager.add_run(model_parameters, i_name, par);
+				int id = run_manager.add_run(model_parameters, "par_name:"+i_name, par);
 
 				if (par_run_map.count(i_name) == 0)
 					par_run_map[i_name] = vector<int>{ id };
@@ -284,15 +284,15 @@ bool Jacobian_1to1::process_runs(ParamTransformSeq &par_transform,
 	double par_value_next;
 	double cur_numeric_par_value;
 	list<JacobianRun> run_list;
-	
+
 
 	for (auto par_run : par_run_map)
-	{
-
+    {
 		for (auto rid : par_run.second)
 		{
 			run_list.push_back(JacobianRun());
 			run_manager.get_info(par_run.second[0], r_status, cur_par_name, cur_numeric_par_value);
+			cur_par_name = cur_par_name.substr(9,cur_par_name.size());
 			run_manager.get_model_parameters(par_run.second[0], run_list.back().ctl_pars);
 			bool success = run_manager.get_observations_vec(par_run.second[0], run_list.back().obs_vec);
 			run_list.back().numeric_derivative_par = cur_numeric_par_value;

src/libs/pestpp_common/Localizer.cpp

@@ -19,7 +19,7 @@ bool Localizer::initialize(PerformanceLog *performance_log, bool forgive_missing
 	how = How::OBSERVATIONS; //set this for the case with no localization
 	string loc_typ;
 	string how_str;
-		
+
 	par2col_map.clear();
 	obs2row_map.clear();
 	colname2col_map.clear();
@@ -74,6 +74,7 @@ bool Localizer::initialize(PerformanceLog *performance_log, bool forgive_missing
 		return use;
 	}
 	performance_log->log_event("loading localizer matrix from file " + filename);
+
 	org_mat.from_file(filename);
 
 	performance_log->log_event("processing localizer matrix");

src/libs/pestpp_common/Localizer.h

@@ -75,6 +75,7 @@ class Localizer
 	LocTyp get_loctyp() { return loctyp; }
 	void report(ofstream &f_rec);
 	bool is_initialized() { return initialized; }
+	Mat* get_orgmat_ptr() {return &org_mat;}
 private:
 	bool use;
 	bool autoadaloc;

src/libs/pestpp_common/MOEA.cpp

@@ -1036,7 +1036,10 @@ map<string, map<string, double>> MOEA::decvar_change_report(map<string, map<stri
     double change, percent_change;
 
     stringstream ss;
-    int max_len = get_max_len_obj_name();
+    int max_len = 19;
+    for (auto& n : dv_names)
+        max_len = max(max_len,(int)n.size());
+
     ss << left << setw(max_len) << "decision variable" << right << setw(11) << "mean change";
     ss << setw(11) << "% change";
     ss << setw(11) << "max change" << setw(11) << "% change";

src/libs/pestpp_common/SVDASolver.cpp

@@ -526,7 +526,7 @@ ModelRun SVDASolver::iteration_upgrd(RunManagerAbstract &run_manager, Terminatio
 					0, i_lambda, i_scale, scaled_ctl_pars);
 
 				stringstream ss;
-				ss << "scale(" << std::fixed << std::setprecision(2) << i_scale << ")";
+				ss << "scale:" << std::fixed << std::setprecision(2) << i_scale << "";
 				par_transform.numeric2model_ip(scaled_pars);
 				int run_id = run_manager.add_run(scaled_pars, ss.str(), i_lambda);
 				num_lam_runs++;

src/libs/pestpp_common/covariance.cpp

@@ -422,6 +422,8 @@ void Mat::from_csv(const string &filename)
 		throw runtime_error("error reading header (first) line from csv file :");
 	pest_utils::strip_ip(line);
 	pest_utils::upper_ip(line);
+
+
 	pest_utils::tokenize(line, col_names, ",", false);
 	col_names.erase(col_names.begin()); //drop the index label
 	vector<Eigen::Triplet<double>> triplet_list;
@@ -479,6 +481,7 @@ void Mat::from_csv(const string &filename)
 	irow++;
 	}
 	matrix.resize(row_names.size(), col_names.size());
+	matrix.setZero();
 	matrix.setFromTriplets(triplet_list.begin(), triplet_list.end());
 
 }

src/libs/pestpp_common/covariance.h

@@ -30,6 +30,7 @@ class Mat
 
 	vector<string> get_row_names(){ return row_names; }
 	vector<string> get_col_names(){ return col_names; }
+	void clear_names(){row_names.clear();col_names.clear();}
 	const vector<string>* rn_ptr();
 	const vector<string>* cn_ptr();
 	Eigen::SparseMatrix<double> get_matrix(){ return matrix; }

src/libs/pestpp_common/pest_data_structs.cpp

@@ -546,8 +546,13 @@ PestppOptions::ARG_STATUS PestppOptions::assign_value_by_key(string key, const s
 	{
 		sweep_base_run = pest_utils::parse_string_arg_to_bool(value);
 	}
+    else if (key == "SWEEP_INCLUDE_REGUL_PHI")
+    {
+        sweep_include_regul_phi = pest_utils::parse_string_arg_to_bool(value);
+    }
+
 
-	else if (key == "TIE_BY_GROUP")
+    else if (key == "TIE_BY_GROUP")
 	{
 		tie_by_group = pest_utils::parse_string_arg_to_bool(value);
 	}
@@ -1121,9 +1126,13 @@ bool PestppOptions::assign_ies_value_by_key(const string& key, const string& val
         convert_ip(value,ies_multimodal_alpha);
         return true;
     }
-
-
-
+    else if ((key == "IES_LOCALIZER_FORGIVE_MISSING") || (key == "IES_LOCALIZER_FORGIVE_EXTRA"))
+    {
+        passed_args.insert("IES_LOCALIZER_FORGIVE_MISSING");
+        passed_args.insert("IES_LOCALIZER_FORGIVE_EXTRA");
+        ies_localizer_forgive_missing = pest_utils::parse_string_arg_to_bool(value);
+        return true;
+    }
 	return false;
 }
 
@@ -1607,6 +1616,7 @@ void PestppOptions::summary(ostream& os) const
 	os << "sweep_chunk: " << sweep_chunk << endl;
 	os << "sweep_forgive: " << sweep_forgive << endl;
 	os << "sweep_base_run: " << sweep_base_run << endl;
+	os << "sweep_include_regul_phi: " << sweep_include_regul_phi << endl;
 
 	os << endl << "...pestpp-opt options:" << endl;
 	os << "opt_objective_function: " <<  opt_obj_func << endl;
@@ -1730,6 +1740,7 @@ void PestppOptions::summary(ostream& os) const
 	os << "ies_upgrades_in_memory: " << ies_upgrades_in_memory << endl;
 	os << "ies_ordered_binary: " << ies_ordered_binary << endl;
 	os << "ies_multimodal_alpha: " << ies_multimodal_alpha << endl;
+	os << "ies_localizer_forgive_extra: " << ies_localizer_forgive_missing << endl;
 
 
 	os << endl << "pestpp-sen options: " << endl;
@@ -1804,6 +1815,7 @@ void PestppOptions::set_defaults()
 	set_sweep_chunk(500);
 	set_tie_by_group(false);
 	set_enforce_tied_bounds(false);
+    set_sweep_include_regul_phi(false);
 
 
 	set_opt_obj_func("");
@@ -1907,6 +1919,7 @@ void PestppOptions::set_defaults()
 	set_ies_ordered_binary(true);
     set_ies_multimodal_alpha(1.0);
     set_ensemble_output_precision(6);
+    set_ies_localizer_forgive_missing(false);
 
 	// DA parameters
 	//set_da_use_ies(false);