From d00e4a825435f845d73f81d95a92c96dd56f9c84 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:13:53 +0100
Subject: [PATCH 01/42] reprted by code scanning

---
 tests/test_mcmc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_mcmc.py b/tests/test_mcmc.py
index 761f0e09..70d97865 100644
--- a/tests/test_mcmc.py
+++ b/tests/test_mcmc.py
@@ -3,7 +3,6 @@
 import numpy as np
 import chaospy as cp
 import json
-import matplotlib.pyplot as plt
 import pytest
 import sys
 from easyvvuq.actions import ExecutePython, Actions

From 32d0682731fc5f708fb21ded2df1d714124b8283 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Fri, 10 Nov 2023 13:16:05 +0000
Subject: [PATCH 02/42] Removed unused variables (comment below describes them
 already).

---
 easyvvuq/db/sql.py  | 4 +---
 tutorials/fusion.py | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/easyvvuq/db/sql.py b/easyvvuq/db/sql.py
index b21c323d..f3053c95 100644
--- a/easyvvuq/db/sql.py
+++ b/easyvvuq/db/sql.py
@@ -9,10 +9,8 @@
 import numpy as np
 from sqlalchemy.sql import case
 from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, declarative_base
 from sqlalchemy import MetaData
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import text
 from sqlalchemy.engine import Engine
 from sqlalchemy import event
diff --git a/tutorials/fusion.py b/tutorials/fusion.py
index c00cf90f..17bf4c91 100755
--- a/tutorials/fusion.py
+++ b/tutorials/fusion.py
@@ -121,7 +121,7 @@ def solve_Te(Qe_tot=2e6, H0=0, Hw=0.1, Te_bc=100, chi=1, a0=1, R0=3, E0=1.5, b_p
     return Te.value, ne.value, mesh.cellCenters.value[0], mesh.cellCenters.value[0]/a
 
 if __name__ == '__main__':
-    Te, ne, rho, rho_norm = solve_Te()
+    solve_Te()
 
 """
 to test:

From d23c9d8efc0f8079ec5c343331df1f189aafaf64 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:20:00 +0100
Subject: [PATCH 03/42] reprted by code scanning: print a message rather than
 completely ignore

---
 easyvvuq/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/_version.py b/easyvvuq/_version.py
index 304dd10d..2e1f03c0 100644
--- a/easyvvuq/_version.py
+++ b/easyvvuq/_version.py
@@ -154,7 +154,7 @@ def git_get_keywords(versionfile_abs):
                     keywords["date"] = mo.group(1)
         f.close()
     except EnvironmentError:
-        pass
+        print('EnvironmentError raised and ignored')
     return keywords
 
 

From dd2a98311257729688b7a9061929df697a8d3340 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:25:05 +0100
Subject: [PATCH 04/42] reported by code scanning

---
 easyvvuq/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/_version.py b/easyvvuq/_version.py
index 2e1f03c0..3246f9e1 100644
--- a/easyvvuq/_version.py
+++ b/easyvvuq/_version.py
@@ -513,7 +513,7 @@ def get_versions():
         if cfg.parentdir_prefix:
             return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     return {"version": "0+unknown", "full-revisionid": None,
             "dirty": None,

From d63699484350336ba337ccb5070239493004e0b1 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:44:14 +0100
Subject: [PATCH 05/42] reported by code scanning

---
 easyvvuq/actions/execute_local.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/easyvvuq/actions/execute_local.py b/easyvvuq/actions/execute_local.py
index d9fcee85..95375c76 100644
--- a/easyvvuq/actions/execute_local.py
+++ b/easyvvuq/actions/execute_local.py
@@ -233,6 +233,8 @@ def start(self, previous=None):
         self.ret = subprocess.run(
             self.full_cmd, cwd=target_dir,
             stdout=stdout, stderr=stderr)
+        if isinstance(self.stdout, str): close(stdout)
+        if isinstance(self.stderr, str): close(stderr)
         return previous
 
     def finished(self):

From 0a731907cb9c362b9bfbb8542c4f8244b3d2e9d6 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:46:31 +0100
Subject: [PATCH 06/42] reported by code scanning

---
 easyvvuq/actions/execute_local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/actions/execute_local.py b/easyvvuq/actions/execute_local.py
index 95375c76..8e328c97 100644
--- a/easyvvuq/actions/execute_local.py
+++ b/easyvvuq/actions/execute_local.py
@@ -116,7 +116,7 @@ def start(self, previous=None):
         try:
             previous['encoder_filename'] = self.encoder.target_filename
         except AttributeError:
-            pass
+            print('AttributeError raised and ignored')
         return previous
 
     def finished(self):

From b8793d709a305866ee03e51a42e01be28d2fc64b Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:48:57 +0100
Subject: [PATCH 07/42] reported by code scanning

---
 easyvvuq/sampling/mcmc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easyvvuq/sampling/mcmc.py b/easyvvuq/sampling/mcmc.py
index 4d71768c..6447fa61 100644
--- a/easyvvuq/sampling/mcmc.py
+++ b/easyvvuq/sampling/mcmc.py
@@ -142,12 +142,12 @@ def update(self, result, invalid):
                 ignored_runs += list(result.loc[result[('chain_id', 0)]
                                                 == chain_id]['run_id'].values)
             except KeyError:
-                pass
+                print('KeyError raised and ignored')
             try:
                 ignored_runs += list(invalid.loc[invalid[('chain_id', 0)]
                                                  == chain_id]['run_id'].values)
             except KeyError:
-                pass
+                print('KeyError raised and ignored')
         ignored_runs = [run[0] for run in ignored_runs]
         self.iteration += 1
         return ignored_runs

From 6cb5238a83666809269d88fda2104a138a81d5fc Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Fri, 10 Nov 2023 13:55:39 +0000
Subject: [PATCH 08/42] Fixed test_sobol and made it a proper pytest.

---
 tests/sc/test_sobol.py | 44 ++++++++++++++----------------------------
 1 file changed, 15 insertions(+), 29 deletions(-)

diff --git a/tests/sc/test_sobol.py b/tests/sc/test_sobol.py
index c1e0ca5b..84733cfa 100644
--- a/tests/sc/test_sobol.py
+++ b/tests/sc/test_sobol.py
@@ -4,7 +4,7 @@
 import easyvvuq as uq
 import numpy as np
 import chaospy as cp
-
+from easyvvuq.actions import CreateRunDirectory, Encode, Decode, CleanUp, ExecuteLocal, Actions
 
 def print_exact_sobols():
     V_i = np.zeros(d)
@@ -18,9 +18,7 @@ def print_exact_sobols():
     print('Exact 1st-order Sobol indices: ', V_i / V)
 
 
-if __name__ == '__main__':
-    import matplotlib.pyplot as plt
-    plt.close('all')
+def test_sobol_basic():
 
     # number of unknown variables
     d = 5
@@ -79,20 +77,17 @@ def print_exact_sobols():
         target_filename='sobol_in.json')
     decoder = uq.decoders.SimpleCSV(target_filename=output_filename,
                                     output_columns=output_columns)
+    execute = ExecuteLocal(f"{HOME}/sobol_model.py sobol_in.json")
 
-    # Add the SC app (automatically set as current app)
-    my_campaign.add_app(name="sc",
-                        params=params,
-                        encoder=encoder,
-                        decoder=decoder)
+    actions = Actions(CreateRunDirectory(HOME),
+                  Encode(encoder), execute, Decode(decoder))
+
+    campaign = uq.Campaign(name='sc', params=params, actions=actions)
 
     # Create the sampler
     vary = {
         "x1": cp.Uniform(0.0, 1.0),
-        "x2": cp.Uniform(0.0, 1.0),
-        "x3": cp.Uniform(0.0, 1.0),
-        "x4": cp.Uniform(0.0, 1.0),
-        "x5": cp.Uniform(0.0, 1.0)}
+        "x2": cp.Uniform(0.0, 1.0)}
 
     """
     SPARSE GRID PARAMETERS
@@ -102,32 +97,23 @@ def print_exact_sobols():
       of 1D collocation points per level. Used to make e.g. clenshaw-curtis
       quadrature nested.
     """
+
     my_sampler = uq.sampling.SCSampler(vary=vary, polynomial_order=2,
                                        quadrature_rule="G", sparse=False,
                                        growth=False)
 
-    # Associate the sampler with the campaign
-    my_campaign.set_sampler(my_sampler)
+    campaign.set_sampler(my_sampler)
 
-    # Will draw all (of the finite set of samples)
-    my_campaign.draw_samples()
-    my_campaign.populate_runs_dir()
-
-    # Use this instead to run the samples using EasyVVUQ on the localhost
-    my_campaign.apply_for_each_run_dir(uq.actions.ExecuteLocal(
-        "tests/sc/sobol_model.py sobol_in.json"))
-
-    my_campaign.collate()
+    campaign.execute().collate()
 
     # Post-processing analysis
     analysis = uq.analysis.SCAnalysis(sampler=my_sampler, qoi_cols=output_columns)
 
-    my_campaign.apply_analysis(analysis)
-
-    results = my_campaign.get_last_analysis()
+    results = campaign.analyse(qoi_cols=output_columns)
 
     print(results.sobols_first())
 
-    print_exact_sobols()
+    assert results.sobols_first()["f"]["x1"] > 0.4
 
-    plt.show()
+if __name__ == '__main__':
+    test_sobol_basic()

From 2f9f39d8de9c6b745f1ceddbd0f06e4914a45935 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Fri, 10 Nov 2023 13:56:08 +0000
Subject: [PATCH 09/42] Sharpened the assert criterion in test_sobol.

---
 tests/sc/test_sobol.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/sc/test_sobol.py b/tests/sc/test_sobol.py
index 84733cfa..3e0a75f2 100644
--- a/tests/sc/test_sobol.py
+++ b/tests/sc/test_sobol.py
@@ -113,7 +113,7 @@ def test_sobol_basic():
 
     print(results.sobols_first())
 
-    assert results.sobols_first()["f"]["x1"] > 0.4
+    assert results.sobols_first()["f"]["x1"] > 0.5
 
 if __name__ == '__main__':
     test_sobol_basic()

From 5dd28410b4843570a8cea9c624ad60814b807dd4 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 10 Nov 2023 14:58:20 +0100
Subject: [PATCH 10/42] reported by code scanning

---
 versioneer.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/versioneer.py b/versioneer.py
index 64fea1c8..5e3e79e2 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -328,7 +328,7 @@ def get_root():
             print("Warning: build in %s is using versioneer.py from %s"
                   % (os.path.dirname(me), versioneer_py))
     except NameError:
-        pass
+        print('NameError raised and ignored')
     return root
 
 
@@ -574,7 +574,7 @@ def git_get_keywords(versionfile_abs):
                     keywords["date"] = mo.group(1)
         f.close()
     except EnvironmentError:
-        pass
+        print('EnvironmentError raised and ignored')
     return keywords
 
 
@@ -908,7 +908,7 @@ def get_versions():
         return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
                                           verbose)
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     try:
         root = os.path.realpath(__file__)
@@ -927,13 +927,13 @@ def get_versions():
         pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
         return render(pieces, cfg.style)
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     try:
         if cfg.parentdir_prefix:
             return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     return {"version": "0+unknown", "full-revisionid": None,
             "dirty": None,
@@ -966,7 +966,7 @@ def git_get_keywords(versionfile_abs):
                     keywords["date"] = mo.group(1)
         f.close()
     except EnvironmentError:
-        pass
+        print('EnvironmentError raised and ignored')
     return keywords
 
 
@@ -1146,7 +1146,7 @@ def do_vcs_install(manifest_in, versionfile_source, ipy):
                     present = True
         f.close()
     except EnvironmentError:
-        pass
+        print('EnvironmentError raised and ignored')
     if not present:
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
@@ -1437,7 +1437,7 @@ def get_versions(verbose=False):
                 print("got version from expanded keyword %s" % ver)
             return ver
         except NotThisMethod:
-            pass
+            print('NotThisMethod raised and ignored')
 
     try:
         ver = versions_from_file(versionfile_abs)
@@ -1445,7 +1445,7 @@ def get_versions(verbose=False):
             print("got version from file %s %s" % (versionfile_abs, ver))
         return ver
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     from_vcs_f = handlers.get("pieces_from_vcs")
     if from_vcs_f:
@@ -1456,7 +1456,7 @@ def get_versions(verbose=False):
                 print("got version from VCS %s" % ver)
             return ver
         except NotThisMethod:
-            pass
+            print('NotThisMethod raised and ignored')
 
     try:
         if cfg.parentdir_prefix:
@@ -1465,7 +1465,7 @@ def get_versions(verbose=False):
                 print("got version from parentdir %s" % ver)
             return ver
     except NotThisMethod:
-        pass
+        print('NotThisMethod raised and ignored')
 
     if verbose:
         print("unable to compute version")
@@ -1750,7 +1750,8 @@ def do_setup():
                     for include in line.split()[1:]:
                         simple_includes.add(include)
     except EnvironmentError:
-        pass
+        print('EnvironmentError raised and ignored')
+
     # That doesn't cover everything MANIFEST.in can do
     # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
     # it might give some false negatives. Appending redundant 'include'

From 1947edad762cf2f7581e0c8b0eb4c644594f1fd3 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Fri, 10 Nov 2023 14:04:18 +0000
Subject: [PATCH 11/42] Removed several more code scanning issues.

---
 tests/test_actions_execute_slurm.py | 2 +-
 tests/test_ensemble_boot.py         | 2 +-
 tests/test_sampling_mc.py           | 1 -
 tests/test_sampling_qmc.py          | 1 -
 tests/test_vector.py                | 7 +------
 5 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/tests/test_actions_execute_slurm.py b/tests/test_actions_execute_slurm.py
index 01465934..7f3dce59 100644
--- a/tests/test_actions_execute_slurm.py
+++ b/tests/test_actions_execute_slurm.py
@@ -32,6 +32,6 @@ def test_action_status_slurm(mock_subprocess_run):
     mock_subprocess_run.return_value = slurm_result
 
     action = ExecuteSLURM('tutorials/epidemic/example.slurm', '$target_dir')
-    previous = {'rundir': '/tmp'}
+    # previous = {'rundir': '/tmp'}
     # action.start(previous)
     # assert(status.job_id == '65541')
diff --git a/tests/test_ensemble_boot.py b/tests/test_ensemble_boot.py
index fef3fb7b..b1592eb9 100644
--- a/tests/test_ensemble_boot.py
+++ b/tests/test_ensemble_boot.py
@@ -30,7 +30,7 @@
 def test_confidence_interval():
     dist = np.array([])
     with pytest.raises(ValueError):
-        stat, low, high = confidence_interval(dist, 0.0, 0.05)
+        confidence_interval(dist, 0.0, 0.05)
     dist = np.array([0.0])
     stat, low, high = confidence_interval(dist, 0.0, 0.05)
     assert (stat == low == high == 0.0)
diff --git a/tests/test_sampling_mc.py b/tests/test_sampling_mc.py
index f0c61f75..fe9c42c8 100644
--- a/tests/test_sampling_mc.py
+++ b/tests/test_sampling_mc.py
@@ -1,7 +1,6 @@
 import pytest
 import chaospy as cp
 from easyvvuq.sampling import MCSampler
-from easyvvuq.sampling.base import Vary
 
 
 def test_sampling():
diff --git a/tests/test_sampling_qmc.py b/tests/test_sampling_qmc.py
index f76eb0bb..d21cf4f3 100644
--- a/tests/test_sampling_qmc.py
+++ b/tests/test_sampling_qmc.py
@@ -1,7 +1,6 @@
 import pytest
 import chaospy as cp
 from easyvvuq.sampling import QMCSampler
-from easyvvuq.sampling.base import Vary
 
 
 def test_init():
diff --git a/tests/test_vector.py b/tests/test_vector.py
index 7c7ffea6..bc8782ee 100644
--- a/tests/test_vector.py
+++ b/tests/test_vector.py
@@ -1,13 +1,8 @@
 import easyvvuq as uq
 import chaospy as cp
 import os
-import sys
-import pytest
 import logging
 from pprint import pformat, pprint
-from .gauss.encoder_gauss import GaussEncoder
-from .gauss.decoder_gauss import GaussDecoder
-from easyvvuq.decoders.json import JSONDecoder
 from easyvvuq.actions import Actions, Encode, Decode, CreateRunDirectory
 
 __copyright__ = """
@@ -88,7 +83,7 @@ def test_gauss_vector_sc(tmpdir):
     data = my_campaign.get_collation_result()
     analysis = uq.analysis.SCAnalysis(sampler=sampler, qoi_cols=["numbers"])
     my_campaign.apply_analysis(analysis)
-    results = my_campaign.get_last_analysis()
+    my_campaign.get_last_analysis()
 
 
 def test_gauss_vector_pce(tmpdir):

From ebea00c24a2ad17baeb38af3dade31682e6250f3 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Fri, 10 Nov 2023 14:34:46 +0000
Subject: [PATCH 12/42] Code scanning.

---
 tests/test_campaign.py     | 2 --
 tests/test_multiapp.py     | 1 -
 tests/test_multiencoder.py | 2 --
 tests/test_multisampler.py | 1 -
 4 files changed, 6 deletions(-)

diff --git a/tests/test_campaign.py b/tests/test_campaign.py
index 2464405e..8164913f 100644
--- a/tests/test_campaign.py
+++ b/tests/test_campaign.py
@@ -4,8 +4,6 @@
 import os
 import logging
 import pytest
-import shutil
-from easyvvuq.db.sql import CampaignTable
 
 TEST_PATH = os.path.dirname(os.path.realpath(__file__))
 LOGGER = logging.getLogger(__name__)
diff --git a/tests/test_multiapp.py b/tests/test_multiapp.py
index 0c0d4ba6..e87a0469 100644
--- a/tests/test_multiapp.py
+++ b/tests/test_multiapp.py
@@ -2,7 +2,6 @@
 from easyvvuq.actions import Actions, Encode, Decode, CreateRunDirectory
 import chaospy as cp
 import os
-import sys
 import pytest
 
 __copyright__ = """
diff --git a/tests/test_multiencoder.py b/tests/test_multiencoder.py
index 8042195d..735dd17b 100644
--- a/tests/test_multiencoder.py
+++ b/tests/test_multiencoder.py
@@ -1,8 +1,6 @@
 import easyvvuq as uq
 from easyvvuq.actions import Actions, Encode, Decode, CreateRunDirectory
-import chaospy as cp
 import os
-import sys
 import pytest
 
 __copyright__ = """
diff --git a/tests/test_multisampler.py b/tests/test_multisampler.py
index 9344d394..ca83b031 100644
--- a/tests/test_multisampler.py
+++ b/tests/test_multisampler.py
@@ -2,7 +2,6 @@
 from easyvvuq.actions import Actions, Encode, Decode, CreateRunDirectory
 import chaospy as cp
 import os
-import sys
 import pytest
 
 __copyright__ = """

From 73ab8ceeed9bccfd488881e428430cfd0b930a17 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 1 Dec 2023 15:13:53 +0100
Subject: [PATCH 13/42] remove unnecessary import of CleanUp

---
 tests/sc/test_sobol.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/sc/test_sobol.py b/tests/sc/test_sobol.py
index 3e0a75f2..ce717a3c 100644
--- a/tests/sc/test_sobol.py
+++ b/tests/sc/test_sobol.py
@@ -4,7 +4,7 @@
 import easyvvuq as uq
 import numpy as np
 import chaospy as cp
-from easyvvuq.actions import CreateRunDirectory, Encode, Decode, CleanUp, ExecuteLocal, Actions
+from easyvvuq.actions import CreateRunDirectory, Encode, Decode, ExecuteLocal, Actions
 
 def print_exact_sobols():
     V_i = np.zeros(d)

From f9e561874b11483ad0890996265a1890bac1a5c3 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 1 Dec 2023 15:32:58 +0100
Subject: [PATCH 14/42] fix problem from code scanning report

---
 easyvvuq/_version.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/easyvvuq/_version.py b/easyvvuq/_version.py
index 3246f9e1..820530f1 100644
--- a/easyvvuq/_version.py
+++ b/easyvvuq/_version.py
@@ -138,21 +138,20 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
+        with open(versionfile_abs, "r") as f:
+            for line in f.readlines():
+                if line.strip().startswith("git_refnames ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["refnames"] = mo.group(1)
+                if line.strip().startswith("git_full ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["full"] = mo.group(1)
+                if line.strip().startswith("git_date ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["date"] = mo.group(1)
     except EnvironmentError:
         print('EnvironmentError raised and ignored')
     return keywords

From 986f0b761f6502370ee7eb37df0eb195bdb1dc55 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 1 Dec 2023 15:58:29 +0100
Subject: [PATCH 15/42] state_file was removed as an option

---
 tutorials/easyvvuq_fusion_analysis.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tutorials/easyvvuq_fusion_analysis.py b/tutorials/easyvvuq_fusion_analysis.py
index ef72d212..f753a8a8 100755
--- a/tutorials/easyvvuq_fusion_analysis.py
+++ b/tutorials/easyvvuq_fusion_analysis.py
@@ -4,6 +4,7 @@
 import time
 import numpy as np
 import matplotlib.pylab as plt
+import os
 
 def sobols(P, coefficients):
     A = np.array(P.coefficients)!=0
@@ -38,7 +39,8 @@ def sobols(P, coefficients):
 
 # Read an old campaign
 time_start = time.time()
-old_campaign = uq.Campaign(state_file="campaign_state.json", work_dir=".")
+DIR = '[[DIR]]'   ## replace with directory containing campaign.db
+old_campaign = uq.campaign(name="solps_pce.", db_location= f'{os.path.abspath(os.curdir)}/{DIR}/campaign.db')
 time_end = time.time()
 print('Time for phase 1 = %.3f' % (time_end-time_start))
 

From 885e882cd28b69301885a8d5f950a0ba942daa5f Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Fri, 1 Dec 2023 16:39:02 +0100
Subject: [PATCH 16/42] minor cleanup for .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index ffd86c39..6f81966f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,4 +21,3 @@ pep.sh
 *.csv
 env/
 .ipynb_checkpoints/
-fusion_pce.*/
\ No newline at end of file

From da44ce8bb4cc9080d9c8c03690b4b505f5a4be1c Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 10:47:41 +0100
Subject: [PATCH 17/42] decreased the number of cases from 1010 to 910 so that
 the limit of 999 Maximum Number Of Host Parameters In A Single SQL Statement
 is not reached

---
 tests/test_db.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_db.py b/tests/test_db.py
index 5a52d1e0..4157db1e 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -55,7 +55,7 @@ def campaign(tmp_path, app_info):
     campaign = CampaignDB(location='sqlite:///{}/test.sqlite'.format(tmp_path))
     campaign.create_campaign(info)
     campaign.tmp_path = str(tmp_path)
-    runs = [RunInfo('run', '.', 1, {'a': 1}, 1, 1) for _ in range(1010)]
+    runs = [RunInfo('run', '.', 1, {'a': 1}, 1, 1) for _ in range(910)]
     campaign.add_runs(runs)
     campaign.add_app(app_info)
     return campaign
@@ -66,14 +66,14 @@ def test_db_file_created(campaign):
 
 
 def test_get_and_set_status(campaign):
-    run_ids = list(range(1, 1011))
+    run_ids = list(range(1, 911))
     assert (all([campaign.get_run_status(id_) == Status.NEW for id_ in run_ids]))
     campaign.set_run_statuses(run_ids, Status.ENCODED)
     assert (all([campaign.get_run_status(id_) == Status.ENCODED for id_ in run_ids]))
 
 
 def test_get_num_runs(campaign):
-    assert (campaign.get_num_runs() == 1010)
+    assert (campaign.get_num_runs() == 910)
 
 
 def test_app(campaign):
@@ -130,7 +130,7 @@ def test_collation(campaign):
     assert (list(result.columns) == [('run_id', 0), ('iteration', 0),
                                      ('a', 0), ('b', 0), ('c', 0), ('c', 1)])
     assert (list(result.iloc[100].values) == [101, 0, 1, 100, 101, 102])
-    assert (result.count()[0] == 1010)
+    assert (result.count()[0] == 910)
 
 
 def test_mv_collation(tmp_path, app_info):

From a87239aef036c0690ad4d5aebb7d3f6467b5d9b4 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 10:18:54 +0000
Subject: [PATCH 18/42] Attempted fix to #409.

---
 easyvvuq/encoders/jinja_encoder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/encoders/jinja_encoder.py b/easyvvuq/encoders/jinja_encoder.py
index 8b811bbe..eaac6680 100644
--- a/easyvvuq/encoders/jinja_encoder.py
+++ b/easyvvuq/encoders/jinja_encoder.py
@@ -64,7 +64,7 @@ def encode(self, params={}, target_dir=''):
         try:
             with open(self.template_fname, 'r') as template_file:
                 template_txt = template_file.read()
-                self.template = Template(template_txt)
+                self.template = Template(template_txt, autoescape=False)
         except FileNotFoundError:
             raise RuntimeError(
                 "the template file specified ({}) does not exist".format(self.template_fname))

From 63d76b2abb9aa86bcfcc2bd2edc87fcf4332467b Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 10:50:00 +0000
Subject: [PATCH 19/42] Set autoescape to True. D'oh!

---
 easyvvuq/encoders/jinja_encoder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/encoders/jinja_encoder.py b/easyvvuq/encoders/jinja_encoder.py
index eaac6680..5016c29c 100644
--- a/easyvvuq/encoders/jinja_encoder.py
+++ b/easyvvuq/encoders/jinja_encoder.py
@@ -64,7 +64,7 @@ def encode(self, params={}, target_dir=''):
         try:
             with open(self.template_fname, 'r') as template_file:
                 template_txt = template_file.read()
-                self.template = Template(template_txt, autoescape=False)
+                self.template = Template(template_txt, autoescape=True)
         except FileNotFoundError:
             raise RuntimeError(
                 "the template file specified ({}) does not exist".format(self.template_fname))

From 29d87e90edfc127d463ec3e625b3228541382ea2 Mon Sep 17 00:00:00 2001
From: Jon McCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 11:19:03 +0000
Subject: [PATCH 20/42] Update of installation instructions process for
 EasyVVUQ, remove defunct code and ensure test case is built

---
 README.md           |  9 ++-------
 install_EasyVVUQ.sh | 13 +++++++++++++
 setup.py            | 35 -----------------------------------
 3 files changed, 15 insertions(+), 42 deletions(-)
 create mode 100644 install_EasyVVUQ.sh

diff --git a/README.md b/README.md
index 93a8927a..ceef4db8 100644
--- a/README.md
+++ b/README.md
@@ -89,16 +89,11 @@ git clone https://github.com/UCL-CCS/EasyVVUQ.git
 
 Note: As above, you need to be sure you are installing for Python 3 - if necessary replace `pip` with `pip3` and `python` with `python3` in the commands below.
 
-We are trying to keep dependencies at a minimum but a few are inevitable, to install them use:
+We are trying to keep dependencies at a minimum but a few are inevitable, to install these, install the EasyVVUQ library itself and build a test case use:
 ```
 cd EasyVVUQ/
 
-pip install --use-feature=2020-resolver -r requirements.txt
-```
-
-Then the library can be installed using:
-```buildoutcfg
-python setup.py install
+bash install_EasyVVUQ.sh
 ```
 
 ## API
diff --git a/install_EasyVVUQ.sh b/install_EasyVVUQ.sh
new file mode 100644
index 00000000..b4cf56f9
--- /dev/null
+++ b/install_EasyVVUQ.sh
@@ -0,0 +1,13 @@
+#1) Install Requirements
+echo 'Install Requirements'
+pip install -r requirements.txt
+
+#2) Install EasyVVUQ
+echo 'Installing EasyVVUQ'
+python -m pip install .
+
+#3) Build cannonsim test
+echo 'Building cannonsim'
+cd tests/cannonsim/src
+make 
+cd ../../..
diff --git a/setup.py b/setup.py
index 43706433..256de293 100644
--- a/setup.py
+++ b/setup.py
@@ -5,39 +5,6 @@
 import versioneer
 import subprocess
 
-
-class BuildCannonsimCommand(distutils.cmd.Command):
-    description = 'build cannonsim'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        self.announce('Building cannonsim')
-        subprocess.check_call(['make'], cwd=path.abspath('./tests/cannonsim/src'))
-
-
-class BuildNotebooks(distutils.cmd.Command):
-    description = 'build tutorials as Jupyter notebooks'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        self.announce('Building tutorial as Jupyter notebooks')
-        tutorials = ['basic_tutorial.rst', 'cooling_coffee_cup.rst']
-        for tutorial in tutorials:
-            subprocess.check_call(['rst2ipynb', tutorial, '-o', path.splitext(tutorial)[0] + '.ipynb'], cwd=path.abspath('./docs'))
-
-
 class BuildPyCommand(setuptools.command.build_py.build_py):
     def run(self):
         setuptools.command.build_py.build_py.run(self)
@@ -48,8 +15,6 @@ def run(self):
     long_description = f.read()
 
 cmdclass = versioneer.get_cmdclass()
-cmdclass['build_cannonsim'] = BuildCannonsimCommand
-cmdclass['build_notebooks'] = BuildNotebooks
 cmdclass['build_py'] = BuildPyCommand
 
 setup(

From b8343a036d82dec87cd429a244ecc51b2174e36d Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 11:25:28 +0000
Subject: [PATCH 21/42] Attempt to remove one warning.

---
 easyvvuq/actions/execute_local.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/easyvvuq/actions/execute_local.py b/easyvvuq/actions/execute_local.py
index 8e328c97..96e64341 100644
--- a/easyvvuq/actions/execute_local.py
+++ b/easyvvuq/actions/execute_local.py
@@ -243,6 +243,8 @@ def finished(self):
     def finalise(self):
         """Performs clean-up if necessary. In this case it isn't. I think.
         """
+        stdout.close()
+        stderr.close()
         pass
 
     def succeeded(self):

From 4d1d5d511d2d8ab306f7c9d1820033310f7cfc97 Mon Sep 17 00:00:00 2001
From: JonMcCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 11:43:13 +0000
Subject: [PATCH 22/42] Update coveralls.yml

Update build steps for EasyVVUQ and cannonsim
---
 .github/workflows/coveralls.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml
index b2b5bb66..5050dff3 100644
--- a/.github/workflows/coveralls.yml
+++ b/.github/workflows/coveralls.yml
@@ -28,8 +28,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python setup.py develop
-        python setup.py build_cannonsim
+        python -m pip install --editable .
+        cd tests/cannonsim/src; make; cd ../../..
         pip install pytest-cov
         pip install coveralls
     - name: Create coverage report

From 5c6c117371a4c3bdb19444a21f70f245141ff2dc Mon Sep 17 00:00:00 2001
From: JonMcCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 11:44:09 +0000
Subject: [PATCH 23/42] Update python-package.yml

Update build instructions for EasyVVUQ and cannonsim
---
 .github/workflows/python-package.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 0dfdfcde..331adc67 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -28,8 +28,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python setup.py develop
-        python setup.py build_cannonsim
+        python -m pip install --editable .
+        cd tests/cannonsim/src; make; cd ../../..
         pip install pytest-cov
         pip install coveralls
     - name: Lint with flake8

From f152d70f89c5336fadd22b058b358ff93330d492 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 12:44:55 +0000
Subject: [PATCH 24/42] Addressing code scanning alert #146.

---
 easyvvuq/sampling/mc_sampler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easyvvuq/sampling/mc_sampler.py b/easyvvuq/sampling/mc_sampler.py
index cfab7d9f..5627588c 100644
--- a/easyvvuq/sampling/mc_sampler.py
+++ b/easyvvuq/sampling/mc_sampler.py
@@ -54,7 +54,6 @@ def __init__(self, vary, n_mc_samples, **kwargs):
         self.n_params = len(vary)
         # the number of MC samples, for each of the n_params + 2 input matrices
         self.n_mc_samples = n_mc_samples
-        self.vary = Vary(vary)
         self.count = 0
         # joint distribution
         self.joint = cp.J(*list(vary.values()))

From 80067d796003c4a1c3dbbcaedd876cb2c85e9fea Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 12:46:34 +0000
Subject: [PATCH 25/42] Code scanning issue #147.

---
 easyvvuq/sampling/mc_sampler.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/easyvvuq/sampling/mc_sampler.py b/easyvvuq/sampling/mc_sampler.py
index 5627588c..5c6d311a 100644
--- a/easyvvuq/sampling/mc_sampler.py
+++ b/easyvvuq/sampling/mc_sampler.py
@@ -49,12 +49,11 @@ def __init__(self, vary, n_mc_samples, **kwargs):
         None.
 
         """
-        super().__init__(vary=vary, max_num=n_mc_samples, **kwargs)
+        super().__init__(vary=vary, count=0, max_num=n_mc_samples, **kwargs)
         # the number of uncertain inputs
         self.n_params = len(vary)
         # the number of MC samples, for each of the n_params + 2 input matrices
         self.n_mc_samples = n_mc_samples
-        self.count = 0
         # joint distribution
         self.joint = cp.J(*list(vary.values()))
         # create the Saltelli sampling plan

From 134dd302a890ae21890f3bb4fbfa82be2e731958 Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Mon, 4 Dec 2023 12:52:20 +0000
Subject: [PATCH 26/42] Fixed several unused variable issues.

---
 tests/test_actions_execute_local.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tests/test_actions_execute_local.py b/tests/test_actions_execute_local.py
index 9c24c67b..2377e8aa 100644
--- a/tests/test_actions_execute_local.py
+++ b/tests/test_actions_execute_local.py
@@ -5,19 +5,18 @@
 
 def test_create_run_directory(tmpdir):
     action = CreateRunDirectory(tmpdir)
-    previous = {'campaign_dir': 'test', 'run_id': 123456789, 'run_info': {'id': 123456789}}
-    previous = action.start(previous)
+    action.start({'campaign_dir': 'test', 'run_id': 123456789, 'run_info': {'id': 123456789}})
     assert (os.path.exists(
         os.path.join(
             tmpdir, 'test', 'runs', 'runs_100000000-200000000', 'runs_123000000-124000000',
             'runs_123450000-123460000', 'runs_123456700-123456800', 'run_123456789')))
-    previous = {'campaign_dir': 'test', 'run_id': 0, 'run_info': {'id': 0}}
-    previous = action.start(previous)
+    
+    action.start({'campaign_dir': 'test', 'run_id': 0, 'run_info': {'id': 0}})
     assert (os.path.exists(
         os.path.join(
             tmpdir, 'test', 'runs', 'runs_0-100000000', 'runs_0-1000000', 'runs_0-10000',
             'runs_0-100')))
-    previous = {'campaign_dir': 'test', 'run_id': 100, 'run_info': {'id': 100}}
+
     action = CreateRunDirectory(tmpdir, flatten=True)
-    previous = action.start(previous)
+    action.start({'campaign_dir': 'test', 'run_id': 100, 'run_info': {'id': 100}})
     assert (os.path.exists(os.path.join(tmpdir, 'test', 'runs', 'run_100')))

From 256d835350aed039c5fa722cd6f520a80d530076 Mon Sep 17 00:00:00 2001
From: Jon McCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 13:17:58 +0000
Subject: [PATCH 27/42] Update sql.py to remove legacy implementation warnings

---
 easyvvuq/db/sql.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/easyvvuq/db/sql.py b/easyvvuq/db/sql.py
index f3053c95..b15a8fc1 100644
--- a/easyvvuq/db/sql.py
+++ b/easyvvuq/db/sql.py
@@ -319,7 +319,8 @@ def update_sampler(self, sampler_id, sampler_element):
             The sampler that should be used as the new state
         """
 
-        selected = self.session.query(SamplerTable).get(sampler_id)
+        selected = self.session.get(SamplerTable,sampler_id)
+        #selected = self.session.query(SamplerTable).get(sampler_id)
         selected.sampler = easyvvuq_serialize(sampler_element)
         self.session.commit()
 
@@ -338,7 +339,8 @@ def resurrect_sampler(self, sampler_id):
             The 'live' sampler object, deserialized from the state in the db
         """
         try:
-            serialized_sampler = self.session.query(SamplerTable).get(sampler_id).sampler
+            serialized_sampler = self.session.get(SamplerTable,sampler_id).sampler
+            #serialized_sampler = self.session.query(SamplerTable).get(sampler_id).sampler
             sampler = easyvvuq_deserialize(serialized_sampler.encode('utf-8'))
         except AttributeError:
             sampler = None
@@ -579,7 +581,8 @@ def get_sampler_id(self, campaign_id):
         int
             The id of the sampler set for the specified campaign
         """
-        sampler_id = self.session.query(CampaignTable).get(campaign_id).sampler
+        sampler_id = self.session.get(CampaignTable,campaign_id).sampler
+        #sampler_id = self.session.query(CampaignTable).get(campaign_id).sampler
         return sampler_id
 
     def set_sampler(self, campaign_id, sampler_id):
@@ -592,7 +595,8 @@ def set_sampler(self, campaign_id, sampler_id):
         sampler_id: int
             ID of the sampler.
         """
-        self.session.query(CampaignTable).get(campaign_id).sampler = sampler_id
+        self.session.get(CampaignTable,campaign_id).sampler = sampler_id
+        #self.session.query(CampaignTable).get(campaign_id).sampler = sampler_id
         self.session.commit()
 
     def campaign_dir(self, campaign_name=None):

From 6d330006d17672b2ddecf5ee9ed784a6cf57f98a Mon Sep 17 00:00:00 2001
From: Jon McCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 13:28:20 +0000
Subject: [PATCH 28/42] Clean sql.py file

---
 easyvvuq/db/sql.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/easyvvuq/db/sql.py b/easyvvuq/db/sql.py
index b15a8fc1..7c07d10b 100644
--- a/easyvvuq/db/sql.py
+++ b/easyvvuq/db/sql.py
@@ -320,7 +320,6 @@ def update_sampler(self, sampler_id, sampler_element):
         """
 
         selected = self.session.get(SamplerTable,sampler_id)
-        #selected = self.session.query(SamplerTable).get(sampler_id)
         selected.sampler = easyvvuq_serialize(sampler_element)
         self.session.commit()
 
@@ -339,8 +338,7 @@ def resurrect_sampler(self, sampler_id):
             The 'live' sampler object, deserialized from the state in the db
         """
         try:
-            serialized_sampler = self.session.get(SamplerTable,sampler_id).sampler
-            #serialized_sampler = self.session.query(SamplerTable).get(sampler_id).sampler
+            serialized_sampler = self.session.get(SamplerTable,sampler_id).sampler 
             sampler = easyvvuq_deserialize(serialized_sampler.encode('utf-8'))
         except AttributeError:
             sampler = None
@@ -582,7 +580,6 @@ def get_sampler_id(self, campaign_id):
             The id of the sampler set for the specified campaign
         """
         sampler_id = self.session.get(CampaignTable,campaign_id).sampler
-        #sampler_id = self.session.query(CampaignTable).get(campaign_id).sampler
         return sampler_id
 
     def set_sampler(self, campaign_id, sampler_id):
@@ -596,7 +593,6 @@ def set_sampler(self, campaign_id, sampler_id):
             ID of the sampler.
         """
         self.session.get(CampaignTable,campaign_id).sampler = sampler_id
-        #self.session.query(CampaignTable).get(campaign_id).sampler = sampler_id
         self.session.commit()
 
     def campaign_dir(self, campaign_name=None):

From aa13ffc6b04b06f16404b88dca47495a8eb0609f Mon Sep 17 00:00:00 2001
From: Jon McCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 13:43:36 +0000
Subject: [PATCH 29/42] Edit qmc.py to correct legacy deprecation warning

---
 easyvvuq/sampling/qmc.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/easyvvuq/sampling/qmc.py b/easyvvuq/sampling/qmc.py
index 0c5e6ead..e6a88a62 100644
--- a/easyvvuq/sampling/qmc.py
+++ b/easyvvuq/sampling/qmc.py
@@ -2,7 +2,8 @@
 """
 
 import chaospy as cp
-from SALib.sample import saltelli
+from SALib.sample import sobol
+#from SALib.sample import saltelli
 from .base import BaseSamplingElement, Vary
 import logging
 
@@ -80,7 +81,7 @@ def __init__(self, vary, n_mc_samples, count=0):
             "bounds": [[0, 1]] * self.n_params
         }
 
-        nodes = saltelli.sample(problem, n_mc_samples, calc_second_order=False)
+        nodes = sobol.sample(problem, n_mc_samples, calc_second_order=False,scramble=True)
 
         self._samples = self.distribution.inv(dist_U.fwd(nodes.transpose()))
 

From 99aa7c2457d7f3a37204613525be0deaaa2a1b3d Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:47:36 +0100
Subject: [PATCH 30/42] only print diagnostic information when EasyVVUQ_Debug
 set

---
 easyvvuq/_version.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easyvvuq/_version.py b/easyvvuq/_version.py
index 820530f1..271f3d3d 100644
--- a/easyvvuq/_version.py
+++ b/easyvvuq/_version.py
@@ -153,7 +153,7 @@ def git_get_keywords(versionfile_abs):
                     if mo:
                         keywords["date"] = mo.group(1)
     except EnvironmentError:
-        print('EnvironmentError raised and ignored')
+        if os.getenv("EasyVVUQ_Debug"): print('EnvironmentError raised and ignored')
     return keywords
 
 
@@ -512,7 +512,7 @@ def get_versions():
         if cfg.parentdir_prefix:
             return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
     except NotThisMethod:
-        print('NotThisMethod raised and ignored')
+        if os.getenv("EasyVVUQ_Debug"): print('NotThisMethod raised and ignored')
 
     return {"version": "0+unknown", "full-revisionid": None,
             "dirty": None,

From 1614a29bd62f571b930e67a425b140894ea2563b Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:48:06 +0100
Subject: [PATCH 31/42] only print diagnostic information when EasyVVUQ_Debug
 set

---
 easyvvuq/actions/execute_local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easyvvuq/actions/execute_local.py b/easyvvuq/actions/execute_local.py
index 96e64341..e2659ff1 100644
--- a/easyvvuq/actions/execute_local.py
+++ b/easyvvuq/actions/execute_local.py
@@ -116,7 +116,7 @@ def start(self, previous=None):
         try:
             previous['encoder_filename'] = self.encoder.target_filename
         except AttributeError:
-            print('AttributeError raised and ignored')
+            if os.getenv("EasyVVUQ_Debug"): print('AttributeError raised and ignored')
         return previous
 
     def finished(self):

From c477522f4f1cce0549fea61101b4b80df8d3477b Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:49:02 +0100
Subject: [PATCH 32/42] replace .append by .concat

---
 easyvvuq/analysis/mcmc.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/easyvvuq/analysis/mcmc.py b/easyvvuq/analysis/mcmc.py
index 6ad30d07..700b6b7a 100644
--- a/easyvvuq/analysis/mcmc.py
+++ b/easyvvuq/analysis/mcmc.py
@@ -41,9 +41,7 @@ def plot_hist(self, input_parameter, chain=None, skip=0, merge=True):
         input_parameter = (input_parameter, 0)
         if merge:
             chain_keys = list(self.chains.keys())
-            df = self.chains[chain_keys[0]][input_parameter].iloc[skip:]
-            for chain in chain_keys[1:]:
-                df.append(self.chains[chain][input_parameter].iloc[skip:])
+            df = pd.concat([self.chains[ck][input_parameter].iloc[skip:] for ck in chain_keys])
             plt.hist(df, 20)
         else:
             plt.hist(self.chains[chain][input_parameter].iloc[skip:], 20)

From bb5c0178e38e7501c3a8123eee7fab3f93998272 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:49:24 +0100
Subject: [PATCH 33/42] only print diagnostic information when EasyVVUQ_Debug
 set

---
 easyvvuq/sampling/mcmc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easyvvuq/sampling/mcmc.py b/easyvvuq/sampling/mcmc.py
index 6447fa61..d20799b5 100644
--- a/easyvvuq/sampling/mcmc.py
+++ b/easyvvuq/sampling/mcmc.py
@@ -1,6 +1,6 @@
 from .base import BaseSamplingElement
 import numpy as np
-
+import os
 
 class MCMCSampler(BaseSamplingElement, sampler_name='mcmc_sampler'):
     """A Metropolis-Hastings MCMC Sampler.
@@ -142,12 +142,12 @@ def update(self, result, invalid):
                 ignored_runs += list(result.loc[result[('chain_id', 0)]
                                                 == chain_id]['run_id'].values)
             except KeyError:
-                print('KeyError raised and ignored')
+                if os.getenv("EasyVVUQ_Debug"): print('KeyError raised and ignored')
             try:
                 ignored_runs += list(invalid.loc[invalid[('chain_id', 0)]
                                                  == chain_id]['run_id'].values)
             except KeyError:
-                print('KeyError raised and ignored')
+                if os.getenv("EasyVVUQ_Debug"): print('KeyError raised and ignored')
         ignored_runs = [run[0] for run in ignored_runs]
         self.iteration += 1
         return ignored_runs

From 0dba9c5575c48ddd6753eea5c1fe3b8bffb2ec3c Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:50:09 +0100
Subject: [PATCH 34/42] got this one working again

---
 tests/test_mcmc.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_mcmc.py b/tests/test_mcmc.py
index 70d97865..e70ae949 100644
--- a/tests/test_mcmc.py
+++ b/tests/test_mcmc.py
@@ -7,6 +7,7 @@
 import sys
 from easyvvuq.actions import ExecutePython, Actions
 from dask.distributed import Client
+import matplotlib.pyplot as plt
 
 
 HOME = os.path.abspath(os.path.dirname(__file__))
@@ -19,13 +20,12 @@ def rosenbrock(inputs):
     return {'value': 300.0 - y}
 
 
-@pytest.mark.skip(reason="Broke due to pandas update. See issue #393.")
+###@pytest.mark.skip(reason="Broke due to pandas update. See issue #393.")
 def test_mcmc(tmp_path):
     campaign = uq.Campaign(name="mcmc", work_dir=tmp_path)
     params = {
         "x1": {"type": "float", "default": 0.0},
         "x2": {"type": "float", "default": 0.0},
-        "out_file": {"type": "string", "default": "output.json"},
         "chain_id": {"type": "integer", "default": 0}
     }
     encoder = uq.encoders.GenericEncoder(template_fname=os.path.abspath(
@@ -49,7 +49,7 @@ def q(x, b=1):
     df = campaign.get_collation_result()
     analysis = uq.analysis.MCMCAnalysis(sampler)
     result = analysis.analyse(df)
-    result.plot_hist('x1')
-    result.plot_hist('x2')
-    result.plot_chains('x1')
-    result.plot_chains('x2')
+    plt.clf(); result.plot_hist('x1'); plt.savefig('/tmp/test_mcmc_hist_x1.png')
+    plt.clf(); result.plot_hist('x2'); plt.savefig('/tmp/test_mcmc_hist_x2.png')
+    plt.clf(); result.plot_chains('x1'); plt.savefig('/tmp/test_mcmc_chains_x1.png')
+    plt.clf(); result.plot_chains('x2'); plt.savefig('/tmp/test_mcmc_chains_x2.png')

From 7f8ea9bd15b0730a09f16637c597462ef3f55fcb Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 14:50:41 +0100
Subject: [PATCH 35/42] tomli seems to be required

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d7febe31..02bcaae8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,4 +23,5 @@ dill
 tqdm
 qcg-pilotjob~=0.13.0
 qcg-pilotjob-executor-api~=0.13.0
-h5py
\ No newline at end of file
+h5py
+tomli

From 9f70130c46524c97f4d69b859133fb397394b806 Mon Sep 17 00:00:00 2001
From: Jon McCullough <j.mccullough@ucl.ac.uk>
Date: Mon, 4 Dec 2023 14:23:16 +0000
Subject: [PATCH 36/42] Modify test value of N to remove warning on number of
 samples not being 2^n

---
 tests/test_analysis_qmc_analysis.py |  2 +-
 tests/test_sampling_qmc.py          | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/test_analysis_qmc_analysis.py b/tests/test_analysis_qmc_analysis.py
index b32e78b4..01dc8113 100644
--- a/tests/test_analysis_qmc_analysis.py
+++ b/tests/test_analysis_qmc_analysis.py
@@ -13,7 +13,7 @@ def test_analyse():
         "a": cp.Uniform(0.0, 1.0),
         "b": cp.Uniform(0.0, 1.0)
     }
-    sampler = QMCSampler(vary, 100)
+    sampler = QMCSampler(vary, 128)
     samples = {('run_id', 0): [], ('a', 0): [], ('b', 0): [], ('a+b', 0): []}
     for i, sample in enumerate(sampler):
         samples[('run_id', 0)].append(i)
diff --git a/tests/test_sampling_qmc.py b/tests/test_sampling_qmc.py
index d21cf4f3..6b55b1f0 100644
--- a/tests/test_sampling_qmc.py
+++ b/tests/test_sampling_qmc.py
@@ -5,21 +5,21 @@
 
 def test_init():
     with pytest.raises(RuntimeError):
-        QMCSampler({}, 100)
+        QMCSampler({}, 128)
     with pytest.raises(RuntimeError):
-        QMCSampler([], 100)
+        QMCSampler([], 128)
 
 
 def test_is_finite():
     vary = {'a': cp.Uniform(-5, 3), 'b': cp.Uniform(2, 10)}
-    sampler = QMCSampler(vary, 100)
+    sampler = QMCSampler(vary, 128)
     assert (sampler.is_finite())
 
 
 def test_sampling():
     vary = {'a': cp.Uniform(-5, 0), 'b': cp.Uniform(2, 10)}
-    sampler = QMCSampler(vary, 100)
-    assert (sampler.n_samples == 400)
+    sampler = QMCSampler(vary, 128)
+    assert (sampler.n_samples == 512)
     for _ in range(sampler.n_samples):
         sample = next(sampler)
         assert (sample['a'] >= -5 and sample['a'] <= 0)
@@ -30,8 +30,8 @@ def test_sampling():
 
 def test_resume():
     vary = {'a': cp.Uniform(-5, 0), 'b': cp.Uniform(2, 10)}
-    sampler = QMCSampler(vary, 100, 390)
-    for _ in range(10):
+    sampler = QMCSampler(vary, 128, 500)
+    for _ in range(12):
         sample = next(sampler)
     with pytest.raises(StopIteration):
         next(sampler)

From 232d1e98ac8a316b8479d125e0b1ace763b3bdb6 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 16:34:03 +0100
Subject: [PATCH 37/42] drop a nuisance field from the params

---
 tests/test_surrogate_workflow.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/test_surrogate_workflow.py b/tests/test_surrogate_workflow.py
index 323df998..f47df946 100644
--- a/tests/test_surrogate_workflow.py
+++ b/tests/test_surrogate_workflow.py
@@ -9,7 +9,7 @@
 from tests.sc.sobol_model import sobol_g_func
 from easyvvuq.analysis.sc_analysis import SCAnalysisResults
 from easyvvuq.actions import CreateRunDirectory, Encode, Decode, ExecuteLocal, Actions, ExecutePython
-
+import matplotlib.pyplot as plt
 
 VARY = {
     "Pe": cp.Uniform(100.0, 200.0),
@@ -17,7 +17,7 @@
 }
 
 
-@pytest.mark.skip(reason="Broke due to pandas update. See issue #395.")
+###@pytest.mark.skip(reason="Broke due to pandas update. See issue #395.")
 @pytest.mark.parametrize('sampler', [
     uq.sampling.RandomSampler(
         vary=VARY, max_num=100, analysis_class=uq.analysis.GaussianProcessSurrogate),
@@ -72,6 +72,7 @@ def test_surrogate_workflow(tmpdir, sampler):
         assert np.max(np.abs(surrogate_y - model_y)) < 1e-6
 
     # Attempt callibration with MCMC
+    del params['out_file']  # eliminate this (now) nuisance field
     campaign.add_app(name='surrogate', params=params, actions=Actions(ExecutePython(surrogate)))
     db_location = campaign.db_location
     campaign = None
@@ -145,6 +146,7 @@ def proposal(x):
 
     def loglikelihood(x):
         return -((u - x) ** 2).sum()
+
     init = {'Pe': [110.0], 'f': [2.0]}
     reloaded_campaign.set_sampler(uq.sampling.MCMCSampler(init, proposal, 'u', 1, loglikelihood))
     iterator = reloaded_campaign.iterate(mark_invalid=True)
@@ -154,3 +156,7 @@ def loglikelihood(x):
     assert (len(df) > 0)
     assert (len(df) <= 100)
     results = reloaded_campaign.analyse()
+    plt.clf(); results.plot_hist('Pe'); plt.savefig('/tmp/test_mcmc_hist_Pe.png')
+    plt.clf(); results.plot_hist('f'); plt.savefig('/tmp/test_mcmc_hist_f.png')
+    plt.clf(); results.plot_chains('Pe'); plt.savefig('/tmp/test_mcmc_chains_Pe.png')
+    plt.clf(); results.plot_chains('f'); plt.savefig('/tmp/test_mcmc_chains_f.png')

From f690e2c0845e5f331d33079301d73b99bb51cc9c Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 17:10:53 +0100
Subject: [PATCH 38/42] add fipy as a requirement for one of the tests

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 02bcaae8..d022ddf9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,3 +25,4 @@ qcg-pilotjob~=0.13.0
 qcg-pilotjob-executor-api~=0.13.0
 h5py
 tomli
+fipy

From f4c2a880dbea29c9fdea43280d7c19b3f5a846cf Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Mon, 4 Dec 2023 17:11:25 +0100
Subject: [PATCH 39/42] test case now works

---
 tests/test_action_replace.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_action_replace.py b/tests/test_action_replace.py
index a02cb6b1..1e8f0c67 100644
--- a/tests/test_action_replace.py
+++ b/tests/test_action_replace.py
@@ -11,7 +11,7 @@
 import pytest
 
 
-@pytest.mark.skip(reason="fipy nonsense")
+###@pytest.mark.skip(reason="fipy nonsense")
 def test_action_replace():
     params = {
         "Qe_tot": {"type": "float", "min": 1.0e6, "max": 50.0e6, "default": 2e6},

From 5511dc091aadfb198b8fbe22ae8b4856db35afde Mon Sep 17 00:00:00 2001
From: Derek Groen <djgroennl@gmail.com>
Date: Tue, 5 Dec 2023 09:29:07 +0000
Subject: [PATCH 40/42] Simple typo fix.

---
 tutorials/easyvvuq_fusion_analysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tutorials/easyvvuq_fusion_analysis.py b/tutorials/easyvvuq_fusion_analysis.py
index f753a8a8..538f76bd 100755
--- a/tutorials/easyvvuq_fusion_analysis.py
+++ b/tutorials/easyvvuq_fusion_analysis.py
@@ -40,7 +40,7 @@ def sobols(P, coefficients):
 # Read an old campaign
 time_start = time.time()
 DIR = '[[DIR]]'   ## replace with directory containing campaign.db
-old_campaign = uq.campaign(name="solps_pce.", db_location= f'{os.path.abspath(os.curdir)}/{DIR}/campaign.db')
+old_campaign = uq.Campaign(name="solps_pce.", db_location= f'{os.path.abspath(os.curdir)}/{DIR}/campaign.db')
 time_end = time.time()
 print('Time for phase 1 = %.3f' % (time_end-time_start))
 

From dcecc7ac9edd37e4b7c9c1a56bf465bfa9315d11 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Tue, 5 Dec 2023 11:17:48 +0100
Subject: [PATCH 41/42] place the 'Sensitivity Analysis: Derivative based' code
 in a try/except structure so that assertion failures don't prevent returning
 the rest of the statistical data

---
 easyvvuq/analysis/pce_analysis.py | 49 +++++++++++++++++--------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/easyvvuq/analysis/pce_analysis.py b/easyvvuq/analysis/pce_analysis.py
index cea7e7e2..8a20c0cf 100644
--- a/easyvvuq/analysis/pce_analysis.py
+++ b/easyvvuq/analysis/pce_analysis.py
@@ -10,6 +10,7 @@
 from .base import BaseAnalysisElement
 from .results import AnalysisResults
 from .qmc_analysis import QMCAnalysisResults
+import traceback
 
 __author__ = 'Jalal Lakhlili'
 __license__ = "LGPL"
@@ -475,29 +476,33 @@ def build_surrogate_der(Y_hat, verbose=False):
                 results['sobols_total'][k] = ST
 
             # Sensitivity Analysis: Derivative based
-            dY_hat = build_surrogate_der(fit, verbose=False)
-            derivatives_first_dict = {}
-            Ndimensions = len(self.sampler.vary.vary_dict)
-            for i, param_name in enumerate(self.sampler.vary.vary_dict):
-                if self.sampler.nominal_value:
-                    # Evaluate dY_hat['param'] at the nominal value of the parameters
-                    values = self.sampler.nominal_value
-                    logging.info(f"Using nominal value of the parameters to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v for v in values.values()])
-                elif all([type(v) == type(cp.Normal()) for v in self.sampler.vary.vary_dict.values()]):
-                    # Evaluate dY_hat['param'] at the mean of the parameters
-                    logging.info(f"Using mean value of the parameters to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v.get_mom_parameters()["shift"][0] for v in self.sampler.vary.vary_dict.values()])
-                elif all([type(v) == type(cp.Uniform()) for v in self.sampler.vary.vary_dict.values()]):
-                    logging.info(f"Using mean value of the parameters to evaluate the derivative ")
-                    # Evaluate dY_hat['param'] at the mean of the parameters
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[(v.lower + v.upper)/2.0 for v in self.sampler.vary.vary_dict.values()])
-                else:
-                    # Evaluate dY_hat['param'] at the zero vector
-                    logging.info(f"Using zero vector to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*np.zeros(Ndimensions))
+            try:
+                dY_hat = build_surrogate_der(fit, verbose=False)
+                derivatives_first_dict = {}
+                Ndimensions = len(self.sampler.vary.vary_dict)
+                for i, param_name in enumerate(self.sampler.vary.vary_dict):
+                    if self.sampler.nominal_value:
+                        # Evaluate dY_hat['param'] at the nominal value of the parameters
+                        values = self.sampler.nominal_value
+                        logging.info(f"Using nominal value of the parameters to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v for v in values.values()])
+                    elif all([type(v) == type(cp.Normal()) for v in self.sampler.vary.vary_dict.values()]):
+                        # Evaluate dY_hat['param'] at the mean of the parameters
+                        logging.info(f"Using mean value of the parameters to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v.get_mom_parameters()["shift"][0] for v in self.sampler.vary.vary_dict.values()])
+                    elif all([type(v) == type(cp.Uniform()) for v in self.sampler.vary.vary_dict.values()]):
+                        logging.info(f"Using mean value of the parameters to evaluate the derivative ")
+                        # Evaluate dY_hat['param'] at the mean of the parameters
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[(v.lower + v.upper)/2.0 for v in self.sampler.vary.vary_dict.values()])
+                    else:
+                        # Evaluate dY_hat['param'] at the zero vector
+                        logging.info(f"Using zero vector to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*np.zeros(Ndimensions))
+
+                    results['derivatives_first'][k] = derivatives_first_dict
 
-            results['derivatives_first'][k] = derivatives_first_dict
+            except Exception:
+                traceback.print_exc()
 
             # Transform the relative numbers back to the absolute values
             if self.relative_analysis:

From 12c173784fd295456fe558b2ca5c77454e801417 Mon Sep 17 00:00:00 2001
From: David Coster <David.Coster@ipp.mpg.de>
Date: Tue, 5 Dec 2023 11:19:47 +0100
Subject: [PATCH 42/42] get the path to the campaign.db from an environment
 variable DIR; update uq.campaign to uq.Campaign; add the 'sqlite:///' prefix
 to the db_location

---
 tutorials/easyvvuq_fusion_analysis.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tutorials/easyvvuq_fusion_analysis.py b/tutorials/easyvvuq_fusion_analysis.py
index f753a8a8..70f69bbe 100755
--- a/tutorials/easyvvuq_fusion_analysis.py
+++ b/tutorials/easyvvuq_fusion_analysis.py
@@ -39,8 +39,8 @@ def sobols(P, coefficients):
 
 # Read an old campaign
 time_start = time.time()
-DIR = '[[DIR]]'   ## replace with directory containing campaign.db
-old_campaign = uq.campaign(name="solps_pce.", db_location= f'{os.path.abspath(os.curdir)}/{DIR}/campaign.db')
+DIR = os.getenv('DIR')   ## use the environment variable DIR to store the previous campaign directory path
+old_campaign = uq.Campaign(name="fusion_pce.", db_location= f'sqlite:///{os.path.abspath(os.curdir)}/{DIR}/campaign.db')
 time_end = time.time()
 print('Time for phase 1 = %.3f' % (time_end-time_start))