Merge pull request #406 from UCL-CCS/code_scanning_alerts

Code scanning alerts

.github/workflows/coveralls.yml

@@ -28,8 +28,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python setup.py develop
-        python setup.py build_cannonsim
+        python -m pip install --editable .
+        cd tests/cannonsim/src; make; cd ../../..
         pip install pytest-cov
         pip install coveralls
     - name: Create coverage report

.github/workflows/python-package.yml

@@ -28,8 +28,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python setup.py develop
-        python setup.py build_cannonsim
+        python -m pip install --editable .
+        cd tests/cannonsim/src; make; cd ../../..
         pip install pytest-cov
         pip install coveralls
     - name: Lint with flake8

.gitignore

@@ -21,4 +21,3 @@ pep.sh
 *.csv
 env/
 .ipynb_checkpoints/
-fusion_pce.*/

README.md

@@ -89,16 +89,11 @@ git clone https://github.com/UCL-CCS/EasyVVUQ.git
 
 Note: As above, you need to be sure you are installing for Python 3 - if necessary replace `pip` with `pip3` and `python` with `python3` in the commands below.
 
-We are trying to keep dependencies at a minimum but a few are inevitable, to install them use:
+We are trying to keep dependencies at a minimum but a few are inevitable, to install these, install the EasyVVUQ library itself and build a test case use:
 ```
 cd EasyVVUQ/
 
-pip install --use-feature=2020-resolver -r requirements.txt
-```
-
-Then the library can be installed using:
-```buildoutcfg
-python setup.py install
+bash install_EasyVVUQ.sh
 ```
 
 ## API

easyvvuq/_version.py

@@ -138,23 +138,22 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
+        with open(versionfile_abs, "r") as f:
+            for line in f.readlines():
+                if line.strip().startswith("git_refnames ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["refnames"] = mo.group(1)
+                if line.strip().startswith("git_full ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["full"] = mo.group(1)
+                if line.strip().startswith("git_date ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["date"] = mo.group(1)
     except EnvironmentError:
-        pass
+        if os.getenv("EasyVVUQ_Debug"): print('EnvironmentError raised and ignored')
     return keywords
 
 
@@ -513,7 +512,7 @@ def get_versions():
         if cfg.parentdir_prefix:
             return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
     except NotThisMethod:
-        pass
+        if os.getenv("EasyVVUQ_Debug"): print('NotThisMethod raised and ignored')
 
     return {"version": "0+unknown", "full-revisionid": None,
             "dirty": None,

easyvvuq/actions/execute_local.py

@@ -116,7 +116,7 @@ def start(self, previous=None):
         try:
             previous['encoder_filename'] = self.encoder.target_filename
         except AttributeError:
-            pass
+            if os.getenv("EasyVVUQ_Debug"): print('AttributeError raised and ignored')
         return previous
 
     def finished(self):
@@ -233,6 +233,8 @@ def start(self, previous=None):
         self.ret = subprocess.run(
             self.full_cmd, cwd=target_dir,
             stdout=stdout, stderr=stderr)
+        if isinstance(self.stdout, str): close(stdout)
+        if isinstance(self.stderr, str): close(stderr)
         return previous
 
     def finished(self):
@@ -241,6 +243,8 @@ def finished(self):
     def finalise(self):
         """Performs clean-up if necessary. In this case it isn't. I think.
         """
+        stdout.close()
+        stderr.close()
         pass
 
     def succeeded(self):

easyvvuq/analysis/mcmc.py

@@ -41,9 +41,7 @@ def plot_hist(self, input_parameter, chain=None, skip=0, merge=True):
         input_parameter = (input_parameter, 0)
         if merge:
             chain_keys = list(self.chains.keys())
-            df = self.chains[chain_keys[0]][input_parameter].iloc[skip:]
-            for chain in chain_keys[1:]:
-                df.append(self.chains[chain][input_parameter].iloc[skip:])
+            df = pd.concat([self.chains[ck][input_parameter].iloc[skip:] for ck in chain_keys])
             plt.hist(df, 20)
         else:
             plt.hist(self.chains[chain][input_parameter].iloc[skip:], 20)

easyvvuq/analysis/pce_analysis.py

@@ -10,6 +10,7 @@
 from .base import BaseAnalysisElement
 from .results import AnalysisResults
 from .qmc_analysis import QMCAnalysisResults
+import traceback
 
 __author__ = 'Jalal Lakhlili'
 __license__ = "LGPL"
@@ -475,29 +476,33 @@ def build_surrogate_der(Y_hat, verbose=False):
                 results['sobols_total'][k] = ST
 
             # Sensitivity Analysis: Derivative based
-            dY_hat = build_surrogate_der(fit, verbose=False)
-            derivatives_first_dict = {}
-            Ndimensions = len(self.sampler.vary.vary_dict)
-            for i, param_name in enumerate(self.sampler.vary.vary_dict):
-                if self.sampler.nominal_value:
-                    # Evaluate dY_hat['param'] at the nominal value of the parameters
-                    values = self.sampler.nominal_value
-                    logging.info(f"Using nominal value of the parameters to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v for v in values.values()])
-                elif all([type(v) == type(cp.Normal()) for v in self.sampler.vary.vary_dict.values()]):
-                    # Evaluate dY_hat['param'] at the mean of the parameters
-                    logging.info(f"Using mean value of the parameters to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v.get_mom_parameters()["shift"][0] for v in self.sampler.vary.vary_dict.values()])
-                elif all([type(v) == type(cp.Uniform()) for v in self.sampler.vary.vary_dict.values()]):
-                    logging.info(f"Using mean value of the parameters to evaluate the derivative ")
-                    # Evaluate dY_hat['param'] at the mean of the parameters
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[(v.lower + v.upper)/2.0 for v in self.sampler.vary.vary_dict.values()])
-                else:
-                    # Evaluate dY_hat['param'] at the zero vector
-                    logging.info(f"Using zero vector to evaluate the derivative ")
-                    derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*np.zeros(Ndimensions))
+            try:
+                dY_hat = build_surrogate_der(fit, verbose=False)
+                derivatives_first_dict = {}
+                Ndimensions = len(self.sampler.vary.vary_dict)
+                for i, param_name in enumerate(self.sampler.vary.vary_dict):
+                    if self.sampler.nominal_value:
+                        # Evaluate dY_hat['param'] at the nominal value of the parameters
+                        values = self.sampler.nominal_value
+                        logging.info(f"Using nominal value of the parameters to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v for v in values.values()])
+                    elif all([type(v) == type(cp.Normal()) for v in self.sampler.vary.vary_dict.values()]):
+                        # Evaluate dY_hat['param'] at the mean of the parameters
+                        logging.info(f"Using mean value of the parameters to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[v.get_mom_parameters()["shift"][0] for v in self.sampler.vary.vary_dict.values()])
+                    elif all([type(v) == type(cp.Uniform()) for v in self.sampler.vary.vary_dict.values()]):
+                        logging.info(f"Using mean value of the parameters to evaluate the derivative ")
+                        # Evaluate dY_hat['param'] at the mean of the parameters
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*[(v.lower + v.upper)/2.0 for v in self.sampler.vary.vary_dict.values()])
+                    else:
+                        # Evaluate dY_hat['param'] at the zero vector
+                        logging.info(f"Using zero vector to evaluate the derivative ")
+                        derivatives_first_dict[param_name] = cp.polynomial(dY_hat[param_name])(*np.zeros(Ndimensions))
+
+                    results['derivatives_first'][k] = derivatives_first_dict
 
-            results['derivatives_first'][k] = derivatives_first_dict
+            except Exception:
+                traceback.print_exc()
 
             # Transform the relative numbers back to the absolute values
             if self.relative_analysis:

easyvvuq/db/sql.py

@@ -9,10 +9,8 @@
 import numpy as np
 from sqlalchemy.sql import case
 from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, declarative_base
 from sqlalchemy import MetaData
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import text
 from sqlalchemy.engine import Engine
 from sqlalchemy import event
@@ -321,7 +319,7 @@ def update_sampler(self, sampler_id, sampler_element):
             The sampler that should be used as the new state
         """
 
-        selected = self.session.query(SamplerTable).get(sampler_id)
+        selected = self.session.get(SamplerTable,sampler_id)
         selected.sampler = easyvvuq_serialize(sampler_element)
         self.session.commit()
 
@@ -340,7 +338,7 @@ def resurrect_sampler(self, sampler_id):
             The 'live' sampler object, deserialized from the state in the db
         """
         try:
-            serialized_sampler = self.session.query(SamplerTable).get(sampler_id).sampler
+            serialized_sampler = self.session.get(SamplerTable,sampler_id).sampler 
             sampler = easyvvuq_deserialize(serialized_sampler.encode('utf-8'))
         except AttributeError:
             sampler = None
@@ -581,7 +579,7 @@ def get_sampler_id(self, campaign_id):
         int
             The id of the sampler set for the specified campaign
         """
-        sampler_id = self.session.query(CampaignTable).get(campaign_id).sampler
+        sampler_id = self.session.get(CampaignTable,campaign_id).sampler
         return sampler_id
 
     def set_sampler(self, campaign_id, sampler_id):
@@ -594,7 +592,7 @@ def set_sampler(self, campaign_id, sampler_id):
         sampler_id: int
             ID of the sampler.
         """
-        self.session.query(CampaignTable).get(campaign_id).sampler = sampler_id
+        self.session.get(CampaignTable,campaign_id).sampler = sampler_id
         self.session.commit()
 
     def campaign_dir(self, campaign_name=None):

easyvvuq/encoders/jinja_encoder.py

@@ -64,7 +64,7 @@ def encode(self, params={}, target_dir=''):
         try:
             with open(self.template_fname, 'r') as template_file:
                 template_txt = template_file.read()
-                self.template = Template(template_txt)
+                self.template = Template(template_txt, autoescape=True)
         except FileNotFoundError:
             raise RuntimeError(
                 "the template file specified ({}) does not exist".format(self.template_fname))

easyvvuq/sampling/mc_sampler.py

@@ -49,13 +49,11 @@ def __init__(self, vary, n_mc_samples, **kwargs):
         None.
 
         """
-        super().__init__(vary=vary, max_num=n_mc_samples, **kwargs)
+        super().__init__(vary=vary, count=0, max_num=n_mc_samples, **kwargs)
         # the number of uncertain inputs
         self.n_params = len(vary)
         # the number of MC samples, for each of the n_params + 2 input matrices
         self.n_mc_samples = n_mc_samples
-        self.vary = Vary(vary)
-        self.count = 0
         # joint distribution
         self.joint = cp.J(*list(vary.values()))
         # create the Saltelli sampling plan

easyvvuq/sampling/mcmc.py

@@ -1,6 +1,6 @@
 from .base import BaseSamplingElement
 import numpy as np
-
+import os
 
 class MCMCSampler(BaseSamplingElement, sampler_name='mcmc_sampler'):
     """A Metropolis-Hastings MCMC Sampler.
@@ -142,12 +142,12 @@ def update(self, result, invalid):
                 ignored_runs += list(result.loc[result[('chain_id', 0)]
                                                 == chain_id]['run_id'].values)
             except KeyError:
-                pass
+                if os.getenv("EasyVVUQ_Debug"): print('KeyError raised and ignored')
             try:
                 ignored_runs += list(invalid.loc[invalid[('chain_id', 0)]
                                                  == chain_id]['run_id'].values)
             except KeyError:
-                pass
+                if os.getenv("EasyVVUQ_Debug"): print('KeyError raised and ignored')
         ignored_runs = [run[0] for run in ignored_runs]
         self.iteration += 1
         return ignored_runs

easyvvuq/sampling/qmc.py

@@ -2,7 +2,8 @@
 """
 
 import chaospy as cp
-from SALib.sample import saltelli
+from SALib.sample import sobol
+#from SALib.sample import saltelli
 from .base import BaseSamplingElement, Vary
 import logging
 
@@ -80,7 +81,7 @@ def __init__(self, vary, n_mc_samples, count=0):
             "bounds": [[0, 1]] * self.n_params
         }
 
-        nodes = saltelli.sample(problem, n_mc_samples, calc_second_order=False)
+        nodes = sobol.sample(problem, n_mc_samples, calc_second_order=False,scramble=True)
 
         self._samples = self.distribution.inv(dist_U.fwd(nodes.transpose()))
 

install_EasyVVUQ.sh

@@ -0,0 +1,13 @@
+#1) Install Requirements
+echo 'Install Requirements'
+pip install -r requirements.txt
+
+#2) Install EasyVVUQ
+echo 'Installing EasyVVUQ'
+python -m pip install .
+
+#3) Build cannonsim test
+echo 'Building cannonsim'
+cd tests/cannonsim/src
+make 
+cd ../../..

requirements.txt

@@ -23,4 +23,6 @@ dill
 tqdm
 qcg-pilotjob~=0.13.0
 qcg-pilotjob-executor-api~=0.13.0
-h5py
+h5py
+tomli
+fipy

setup.py

@@ -5,39 +5,6 @@
 import versioneer
 import subprocess
 
-
-class BuildCannonsimCommand(distutils.cmd.Command):
-    description = 'build cannonsim'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        self.announce('Building cannonsim')
-        subprocess.check_call(['make'], cwd=path.abspath('./tests/cannonsim/src'))
-
-
-class BuildNotebooks(distutils.cmd.Command):
-    description = 'build tutorials as Jupyter notebooks'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        self.announce('Building tutorial as Jupyter notebooks')
-        tutorials = ['basic_tutorial.rst', 'cooling_coffee_cup.rst']
-        for tutorial in tutorials:
-            subprocess.check_call(['rst2ipynb', tutorial, '-o', path.splitext(tutorial)[0] + '.ipynb'], cwd=path.abspath('./docs'))
-
-
 class BuildPyCommand(setuptools.command.build_py.build_py):
     def run(self):
         setuptools.command.build_py.build_py.run(self)
@@ -48,8 +15,6 @@ def run(self):
     long_description = f.read()
 
 cmdclass = versioneer.get_cmdclass()
-cmdclass['build_cannonsim'] = BuildCannonsimCommand
-cmdclass['build_notebooks'] = BuildNotebooks
 cmdclass['build_py'] = BuildPyCommand
 
 setup(