Update testing architecture

.coverageac

@@ -0,0 +1,5 @@
+[run]
+omit =
+    *test*
+    *schema2rst.py
+

.github/workflows/CI_WEIS.yml

@@ -1,7 +1,7 @@
 name: CI_WEIS
 
-# We run CI on push commits and pull requests on all branches
-on: [push, pull_request]
+# We run CI on push commits on all branches
+on: [push]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -48,31 +48,27 @@ jobs:
       - name: Install WEIS
         shell: pwsh
         run: |
-          python setup.py develop      
-
-      # Run example scripts
-      - name: Run examples
+          python setup.py develop
+          
+      # List the collected tests for debugging purposes
+      - name: List tests
         shell: pwsh
         run: |
-          cd weis/test
-          python test_examples.py
-
-      # Run scripts within rotor_opt folder with MPI
-      - name: Run parallel examples rotor optimization
+          pytest weis --collect-only
+
+      # Run all tests within WEIS, but not computationally expensive examples
+      - name: Run tests within WEIS
         shell: pwsh
         run: |
-          cd examples/05_IEA-3.4-130-RWT
-          mpirun -np 2 python weis_driver.py
+          pytest weis --cov-config=.coverageac --cov=weis
 
-      # Run scripts within control_opt folder, with and without MPI
-      - name: Run examples control optimization
+      # Run coveralls
+      - name: Run coveralls
+        if: contains( matrix.os, 'ubuntu')
+        # This also works, https://github.com/AndreMiras/coveralls-python-action
+        #uses: AndreMiras/coveralls-python-action@develop
         shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          cd examples/02_control_opt
-          mpirun -np 2 python runOptimization.py
-
-      # Run scripts within dac folder
-      # - name: Run examples distributed aerodynamic control
-      #  run: |
-      #    cd examples/dac_flaps
-      #    python dac_driver.py
+          coveralls

.github/workflows/run_exhaustive_examples.yml

@@ -0,0 +1,79 @@
+name: run_exhaustive_examples
+
+on: [pull_request]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  build:
+    name: Build (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    # if: "contains(github.event.head_commit.message, 'exhaustive_CI')"  # This line could be used to have commit-specific instructions
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ["ubuntu-latest"] #, "macOS-latest"]
+        python-version: ["3.8"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: conda-incubator/setup-miniconda@v2
+        # https://github.com/marketplace/actions/setup-miniconda
+        with:
+          miniconda-version: "latest"
+          channels: conda-forge
+          auto-update-conda: true
+          python-version: 3.8
+          environment-file: environment.yml
+
+      # Install dependencies of WEIS specific to ubuntu
+      - name: Add dependencies ubuntu specific
+        if: false == contains( matrix.os, 'windows')
+        shell: pwsh # putting in a shell command makes for compile linking problems later
+        # (if you use the shell here, cannot use 'compiler' package, but mpi only seems to work with it)
+        run: |
+          conda install -y petsc4py mpi4py 
+          python -c "import platform; print(platform.node())"
+
+      # Install dependencies of WISDEM specific to windows
+      - name: Add dependencies windows specific
+        if: contains( matrix.os, 'windows')
+        run: |
+          conda install -y m2w64-toolchain libpython
+
+      # Debugging session
+      #- name: Setup tmate session
+      #  uses: mxschmitt/action-tmate@v3
+
+      # Install WISDEM
+      - name: Install WEIS
+        shell: pwsh
+        run: |
+          python setup.py develop
+
+      # Run all examples
+      - name: Run tests within WEIS
+        shell: pwsh
+        run: |
+          cd weis/test
+          pytest run_examples.py
+
+      # Run scripts within rotor_opt folder with MPI
+      - name: Run parallel examples rotor optimization
+        shell: pwsh
+        run: |
+          cd examples/05_IEA-3.4-130-RWT
+          mpirun -np 2 python weis_driver.py
+
+      # Run scripts within control_opt folder, with and without MPI
+      - name: Run examples control optimization
+        shell: pwsh
+        run: |
+          cd examples/02_control_opt
+          mpirun -np 2 python runOptimization.py
+
+      # Run scripts within dac folder
+      # - name: Run examples distributed aerodynamic control
+      #  run: |
+      #    cd examples/dac_flaps
+      #    python dac_driver.py
+

README.md

@@ -1,5 +1,10 @@
 # WEIS
 
+[![Coverage Status](https://coveralls.io/repos/github/WISDEM/WEIS/badge.svg?branch=develop)](https://coveralls.io/github/WISDEM/WEIS?branch=develop)
+[![Actions Status](https://github.com/WISDEM/WEIS/workflows/CI_WEIS/badge.svg)](https://github.com/WISDEM/WEIS/actions)
+[![Documentation Status](https://readthedocs.org/projects/weis/badge/?version=develop)](https://weis.readthedocs.io/en/develop/?badge=develop)
+
+
 WEIS, Wind Energy with Integrated Servo-control, performs multifidelity co-design of wind turbines. WEIS is a framework that combines multiple NREL-developed tools to enable design optimization of floating offshore wind turbines.
 
 Author: [NREL WISDEM & OpenFAST & Control Teams](mailto:systems.engineering@nrel.gov) 

docs/how_to_contribute_code.rst

@@ -15,7 +15,7 @@ Documentation
 -------------
 When you add or modify code, make sure to provide relevant documentation that explains the new code.
 This should be done in code via comments and also in the Sphinx documentation if you add a new feature or capability.
-Look at the .rst files in the `docs` section of the repo or click on `view source` on any of the doc pages to see some examples.
+Look at the .rst files in the :code:`docs` section of the repo or click on :code:`view source` on any of the doc pages to see some examples.
 
 There is currently very little documentation for WEIS, so you have a lot of flexibility in terms of where you place your new documentation.
 Do not stress too much about the outline of the information you create, simply that it exists within the repo.
@@ -24,7 +24,7 @@ We will reorganize the documentation content at a later date.
 Using Git subtrees
 ------------------
 
-The WEIS repo contains copies of other codes created by using the `git subtree` commands.
+The WEIS repo contains copies of other codes created by using the :code:`git subtree` commands.
 Below are some details about how to add external codes and update them.
 
 To add an external code, using OpenFAST as an example, type:
@@ -36,7 +36,7 @@ To add an external code, using OpenFAST as an example, type:
   $ git subtree add -P OpenFAST OpenFAST/dev --squash
 
 
-The `--squash` is important so WEIS doesn't get filled up with commits from the subtree repos.
+The :code:`--squash` is important so WEIS doesn't get filled up with commits from the subtree repos.
 
 Once a subtree code exists in this repo, we can update it like this.
 This first two lines are needed only if you don't have the remote for the particular subtree yet.
@@ -49,25 +49,74 @@ If you already have the remote, only the last line is needed.
   $ git subtree pull --prefix OpenFAST https://github.com/OpenFAST/openfast dev --squash --message="Updating to latest OpenFAST develop"
 
 Changes to these subtree codes **should only be made to their original repos**, *not* to this WEIS repo.
-Once those individual repos have been updated, use the previous `git subtree pull` command to pull in those updates to the WEIS repo.
+Once those individual repos have been updated, use the previous :code:`git subtree pull` command to pull in those updates to the WEIS repo.
 Once the upstream repos have your code changes, those changes have been pulled into your branch, you can then submit a PR for WEIS.
 
-If you run into trouble using `git subtree`, specifically if you see `git: 'subtree' is not a git command.`, try using your system git instead of any conda-installed git.
-Specifically, try using `/usr/bin/git subtree` for any subtree commands.
+If you run into trouble using :code:`git subtree`, specifically if you see :code:`git: 'subtree' is not a git command.`, try using your system git instead of any conda-installed git.
+Specifically, try using :code:`/usr/bin/git subtree` for any subtree commands.
 If that doesn't work for you, please open an issue on this repo so we can track it.
 
 Testing
 -------
 When you add code or functionality, add tests that cover the new or modified code.
-These may be units tests for individual components or regression tests for entire models that use the new functionality.
+These may be units tests for individual code blocks or regression tests for entire models that use the new functionality.
+These tests should be a balance between minimizing computational cost and maximizing code coverage.
+This ensures continued functionality of WEIS while keeping development time short.
+
+Any Python file with :code:`test` in its name within the :code:`weis` package directory is tested with each commit to WEIS.
+This is done through GitHub Actions and you can see the automated testing progress on the GitHub repo under the :code:`Actions` tab, `located here <https://github.com/WISDEM/WEIS/actions>`_.
+If any test fails, this information is passed on to GitHub and a red X will be shown next to the commit.
+Otherwise, if all tests pass, a green check mark appears to signify the code changes are valid.
+
+Unit tests
+~~~~~~~~~~ 
+
+Each discipline sub-directory should contain tests in the :code:`test` folder.
+For example, :code:`weis/multifidelity/test` hosts the tests for multifidelity optimization within WEIS.
+Look at :code:`test_simple_models.py` within that folder for a simple unit test that you can mimic when you add new code.
+Another simple unit test is contained in :code:`weis/aeroelasticse/test` called :code:`test_IECWind.py`.
+
+Unit tests should be short and purposeful, test the smallest reasonable block of code, and quickly point to potential problems in the code.
+`This article <https://dzone.com/articles/10-tips-to-writing-good-unit-tests>`_ has some quick tips on how to write good unit tests.
+
+Regression tests
+~~~~~~~~~~~~~~~~
+
+Regression tests examine much larger portions of the code by examining top-level input and output relationships.
+Specifically, these tests check the values that the code produces against "truth" values and returns an error if they do not match.
+As an example, a low-level coding change might alter a default within a subsystem of the model being tested, which might result in a different AEP value for the wind turbine.
+The regression test would report that the AEP value differs, and thus the tests fail.
+Of course, it would be challenging to completely diagnose a coding change based on only regression tests, so well-made unit tests can help narrow down a problem much more quickly.
+
+Within WEIS, regression tests live in the :code:`weis/test` folder.
+Examine :code:`test_aeroelasticse/test_DLC.py` to see an example regression test that checks OpenFAST results obtained through WEIS' wrapper.
+Specifically, that test compares all of the channel outputs against truth values contained in :code:`.pkl` files within the same folder.
+
+Like unit tests, regression tests should run quickly.
+They can have unrealistic simulation parameters (1 second timeseries) as long as they adequately test the code.
+
+
+Coveralls
+~~~~~~~~~
+
+To understand how WEIS is tested, we use a tool called `Coveralls <https://coveralls.io/github/wisdem/WEIS>`_, which reports the lines of code that are used during testing.
+This lets WEIS developers know which functions and methods are tested, as well as where to add tests in the future.
+
+When you push a commit to WEIS, all of the unit and regression tests are ran.
+Then, the coverage from those tests is reported to Coveralls automatically. 
 
-Each discipline sub-directory contains tests in the `test` folder.
-For example, `weis/multifidelity/test` hosts the tests for multifidelity optimization within WEIS.
-Look at `test_simple_models.py` within that folder for a simple unit test that you can mimic when you add new code.
 
 Pull requests
 -------------
 Once you have added or modified code, submit a pull request via the GitHub interface.
 This will automatically go through all of the tests in the repo to make sure everything is functioning properly.
 This also automatically does a coverage test to ensure that any added code is covered in a test.
-The main developers of WEIS will then merge in the request or provide feedback on how to improve the contribution.
+The main developers of WEIS will then merge in the request or provide feedback on how to improve the contribution.
+
+In addition to the full unit and regression test suite, on pull requests additional examples are checked using GitHub Actions using the workflow labeled :code:`run_exhaustive_examples`.
+These examples are useful for users to adapt, but are computationally expensive, so we do not test them on every commit.
+Instead, we test them only when code is about to be added to the main WEIS develop or master branches through pull requests.
+The coverage from these examples are not considered in Coveralls.
+
+The examples that are covered are shown in :code:`weis/test/run_examples.py`.
+If you add an example to WEIS, make sure to add a call to it in the :code:`run_examples.py` script as well.

environment.yml

@@ -5,6 +5,8 @@ channels:
 dependencies:
   - python
   - pytest
+  - pytest-cov
+  - coveralls
   - openmdao=3.4.0
   - jsonschema
   - ruamel_yaml

examples/06_IEA-15-240-RWT/run_control_opt.py

@@ -371,7 +371,7 @@ def gen_linear_model():
 
     TODO: check for the models so we can skip this step if already run
     """
-
+    
 
     linear = LinearFAST(FAST_ver='OpenFAST', dev_branch=True);
 

setup.py

@@ -158,7 +158,7 @@ def build_extension(self, ext):
     long_description_content_type = 'text/markdown',
     author                        = 'NREL',
     url                           = 'https://github.com/WISDEM/WEIS',
-    install_requires              = ['openmdao>=3.2','numpy','scipy','nlopt','dill','smt','control'],
+    install_requires              = ['openmdao>=3.2','numpy','scipy','nlopt','dill','smt','control','jsonmerge'],
     classifiers                   = [_f for _f in CLASSIFIERS.split('\n') if _f],
     packages                      = weis_pkgs,
     package_data                  =  {'':['*.yaml','*.xlsx']},

weis/aeroelasticse/FAST_writer.py

@@ -1713,7 +1713,13 @@ def write_SubDyn(self):
         f.write('{:<22d} {:<11} {:}'.format(self.fst_vt['SubDyn']['NDiv'], 'NDiv', '- Number of sub-elements per member\n'))
         f.write('{!s:<22} {:<11} {:}'.format(self.fst_vt['SubDyn']['CBMod'], 'CBMod', '- [T/F] If True perform C-B reduction, else full FEM dofs will be retained. If True, select Nmodes to retain in C-B reduced system.\n'))
         f.write('{:<22d} {:<11} {:}'.format(self.fst_vt['SubDyn']['Nmodes'], 'Nmodes', '- Number of internal modes to retain (ignored if CBMod=False). If Nmodes=0 --> Guyan Reduction.\n'))
-        f.write('{:<22} {:<11} {:}'.format(", ".join(self.fst_vt['SubDyn']['JDampings']), 'JDampings', '- Damping Ratios for each retained mode (% of critical) If Nmodes>0, list Nmodes structural damping ratios for each retained mode (% of critical), or a single damping ratio to be applied to all retained modes. (last entered value will be used for all remaining modes).\n'))
+
+        JDampings = self.fst_vt['SubDyn']['JDampings']
+        if isinstance(JDampings, int):
+            f.write('{:<22} {:<11} {:}'.format(self.fst_vt['SubDyn']['JDampings'], 'JDampings', '- Damping Ratios for each retained mode (% of critical) If Nmodes>0, list Nmodes structural damping ratios for each retained mode (% of critical), or a single damping ratio to be applied to all retained modes. (last entered value will be used for all remaining modes).\n'))
+        else:
+            f.write('{:<22} {:<11} {:}'.format(", ".join(self.fst_vt['SubDyn']['JDampings']), 'JDampings', '- Damping Ratios for each retained mode (% of critical) If Nmodes>0, list Nmodes structural damping ratios for each retained mode (% of critical), or a single damping ratio to be applied to all retained modes. (last entered value will be used for all remaining modes).\n'))
+
         f.write('---- STRUCTURE JOINTS: joints connect structure members (~Hydrodyn Input File)---\n')
         f.write('{:<22d} {:<11} {:}'.format(self.fst_vt['SubDyn']['NJoints'], 'NJoints', '- Number of joints (-)\n'))
         f.write(" ".join(['{:^11s}'.format(i) for i in ['JointID', 'JointXss', 'JointYss', 'JointZss']])+' [Coordinates of Member joints in SS-Coordinate System]\n')

weis/aeroelasticse/LinearFAST.py

@@ -392,7 +392,7 @@ def runFAST_linear(self):
 
 
 
-def gen_linear_model(wind_speeds):
+def gen_linear_model(wind_speeds, Tmax=600.):
     """ 
     Generate OpenFAST linearizations across wind speeds
 
@@ -407,7 +407,7 @@ def gen_linear_model(wind_speeds):
     linear.weis_dir                 = os.path.dirname( os.path.dirname ( os.path.dirname( __file__ ) ) ) + os.sep
 
     linear.FAST_InputFile           = 'IEA-15-240-RWT-UMaineSemi.fst'   # FAST input file (ext=.fst)
-    linear.FAST_directory           = os.path.join(linear.weis_dir, 'examples/OpenFAST_models/IEA-15-240-RWT/IEA-15-240-RWT-UMaineSemi')   # Path to fst directory files
+    linear.FAST_directory           = os.path.join(linear.weis_dir, 'examples/01_aeroelasticse/OpenFAST_models/IEA-15-240-RWT/IEA-15-240-RWT-UMaineSemi')   # Path to fst directory files
     linear.FAST_steadyDirectory     = os.path.join(linear.weis_dir,'outputs','iea_semi_steady')
     linear.FAST_linearDirectory     = os.path.join(linear.weis_dir,'outputs','iea_semi_lin')
     linear.debug_level              = 2
@@ -426,7 +426,7 @@ def gen_linear_model(wind_speeds):
     linear.GBRatio          = fastRead.fst_vt['ElastoDyn']['GBRatio']
     linear.WindSpeeds       = wind_speeds  #[8.,10.,12.,14.,24.]
     linear.DOFs             = ['GenDOF'] #,'TwFADOF1','PtfmPDOF']  # enable with 
-    linear.TMax             = 600.   # should be 1000-2000 sec or more with hydrodynamic states
+    linear.TMax             = Tmax   # should be 1000-2000 sec or more with hydrodynamic states
     linear.NLinTimes        = 12
 
     #if true, there will be a lot of hydronamic states, equal to num. states in ss_exct and ss_radiation models