Enable CI static check to enforce consistent import order (#1170)

* Enable wrong-import-order Pylint check

* Investigate why enabled check is not enforced

* Edit CLI flags on the command that actually gets run

* Make pytest jobs dependent on pylint

* Run isort 5.12.0

* Run Black

* Move conditional imports after normal imports to allow isort to work

* Run isort --profile=black

* Fix remaining file

* Run isort on recently added or modified files

.github/actions/pylint/action.yml

@@ -15,6 +15,6 @@ runs:
       shell: bash
       run: |
         echo '::group::Output of pylint'
-        pylint --rcfile=.pylint/pylintrc --disable=W,C,R,I foqus_lib/
+        pylint --rcfile=.pylint/pylintrc --disable=all --enable=E --enable=wrong-import-order foqus_lib/
         echo '::endgroup::'
 

.github/workflows/checks.yml

@@ -59,7 +59,7 @@ jobs:
           config: .typos.toml
   pytest:
     name: pytest (py${{ matrix.python-version }}/${{ matrix.os }})
-    needs: [code-formatting, spell-check]
+    needs: [spell-check, code-formatting, pylint]
     runs-on: ${{ matrix.os-version }}
     strategy:
       fail-fast: false
@@ -148,7 +148,7 @@ jobs:
           pip-install-target: -r requirements-dev.txt
       - name: Run pylint
         run: |
-          pylint --rcfile=.pylint/pylintrc --disable=W,C,R,I foqus_lib/
+          pylint --rcfile=.pylint/pylintrc --disable=all --enable=E --enable=wrong-import-order foqus_lib/
 
   docs:
     name: Build docs

foqus_lib/conftest.py

@@ -14,9 +14,10 @@
 #################################################################################
 import contextlib
 import os
-from pathlib import Path
 import shutil
 import zipfile
+from pathlib import Path
+
 import pytest
 
 

foqus_lib/foqus.py

@@ -20,19 +20,20 @@
 Keith Beattie, Lawrence Berkeley National Labs, 2020
 """
 
+import argparse
+import json
+import logging
+
 # Imports
 import signal
-import uuid
 import sys
-import argparse
 import time
-import json
-import logging
+import uuid
 
 # FOQUS imports
 import foqus_lib.version.version as ver  # foqus version and other info
-from foqus_lib.framework.session.session import *
 from foqus_lib.framework.listen.listen import foqusListener2
+from foqus_lib.framework.session.session import *
 from foqus_lib.gui.make_shortcut import makeShortcut
 
 loadGUI = False
@@ -56,17 +57,17 @@ def guiImport(mpl_backend="Qt5Agg"):
     global PyQt5
     # GUI Imports
     try:  # Check if the PySide libraries are available
+        import matplotlib
         import PyQt5
+        import PyQt5.QtCore
+        import PyQt5.QtGui
 
         # QtWidgets, QtGui, and QtCore are used in this module,
         # but they might not be available in PyQt5 without importing them first
         # in most circumstances, they will be already imported
         # from the imports in foqus_lib.framework.session.session
         # to be on the safe side, we run these imports explicitly here, too
         import PyQt5.QtWidgets
-        import PyQt5.QtGui
-        import PyQt5.QtCore
-        import matplotlib
 
         matplotlib.use(mpl_backend)
         matplotlib.rcParams["backend"] = mpl_backend
@@ -362,8 +363,10 @@ def main(args_to_parse=None):
         sys.exit(makeShortcut())
     if args.terminateConsumer:
         try:
-            from foqus_lib.framework.sim.turbineLiteDB import turbineLiteDB
-            from foqus_lib.framework.sim.turbineLiteDB import keepAliveTimer
+            from foqus_lib.framework.sim.turbineLiteDB import (
+                keepAliveTimer,
+                turbineLiteDB,
+            )
 
             fs = generalSettings()  # foqus settings
             fs.load(logging=False)
@@ -379,8 +382,10 @@ def main(args_to_parse=None):
             sys.exit(1)
     elif args.addTurbineApp:
         try:
-            from foqus_lib.framework.sim.turbineLiteDB import turbineLiteDB
-            from foqus_lib.framework.sim.turbineLiteDB import keepAliveTimer
+            from foqus_lib.framework.sim.turbineLiteDB import (
+                keepAliveTimer,
+                turbineLiteDB,
+            )
 
             fs = generalSettings()  # foqus settings
             fs.load(logging=False)
@@ -596,8 +601,7 @@ def main(args_to_parse=None):
         listener.start()
         listener.join()
     elif args.consumer:
-        from foqus_lib.framework.sim.turbineLiteDB import turbineLiteDB
-        from foqus_lib.framework.sim.turbineLiteDB import keepAliveTimer
+        from foqus_lib.framework.sim.turbineLiteDB import keepAliveTimer, turbineLiteDB
 
         load_gui = False
         # Make ctrl-c do nothing but and SIGINT donothing but interrupt

foqus_lib/framework/graph/OptGraphOptim.py

@@ -12,11 +12,13 @@
 # respectively. This file is also available online at the URL
 # "https://github.com/CCSI-Toolset/FOQUS".
 #################################################################################
-import numpy
-import time
 import math
-from foqus_lib.framework.graph.node import *
+import time
+
+import numpy
+
 from foqus_lib.framework.graph.edge import *
+from foqus_lib.framework.graph.node import *
 
 
 class optimObj:

foqus_lib/framework/graph/graph.py

@@ -21,25 +21,27 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 
-import queue
-import foqus_lib.framework.sampleResults.results as resultList
-import multiprocessing.dummy as multiprocessing
-import numpy
-import math
-import time
-import csv
 import copy
-import threading
+import csv
 import logging
+import math
+import multiprocessing.dummy as multiprocessing
+import queue
 import sys
+import threading
+import time
 from collections import OrderedDict
+
+import numpy
+import pandas
+
+import foqus_lib.framework.sampleResults.results as resultList
+from foqus_lib.framework.graph.edge import *  # Edge and variable connection classes
 from foqus_lib.framework.graph.node import *  # Node, input var and output var classes
 from foqus_lib.framework.graph.nodeModelTypes import nodeModelTypes
-from foqus_lib.framework.graph.edge import *  # Edge and variable connection classes
+from foqus_lib.framework.graph.nodeVars import *
 from foqus_lib.framework.graph.OptGraphOptim import *  # Objective function calculation class
 from foqus_lib.framework.sim.turbineConfiguration import *
-from foqus_lib.framework.graph.nodeVars import *
-import pandas
 
 _log = logging.getLogger("foqus." + __name__)
 

foqus_lib/framework/graph/heatIntegration.py

@@ -12,9 +12,10 @@
 # respectively. This file is also available online at the URL
 # "https://github.com/CCSI-Toolset/FOQUS".
 #################################################################################
-from . import node as gn
 import subprocess
 
+from . import node as gn
+
 
 def makeHeatIntegrationNode(node):
     # setup whatever i need for heat integration

foqus_lib/framework/graph/node.py

@@ -19,19 +19,21 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 
-import os
 import json
-import math
-import numpy as np
 import logging
-from foqus_lib.framework.pymodel.pymodel import *
-from foqus_lib.framework.graph.nodeVars import *
-from foqus_lib.framework.graph.nodeModelTypes import nodeModelTypes
+import math
+import os
 from collections import OrderedDict
+from importlib import import_module
+
+import numpy as np
+
+from foqus_lib.framework.at_dict.at_dict import AtDict
 from foqus_lib.framework.foqusOptions.optionList import optionList
+from foqus_lib.framework.graph.nodeModelTypes import nodeModelTypes
+from foqus_lib.framework.graph.nodeVars import *
+from foqus_lib.framework.pymodel.pymodel import *
 from foqus_lib.framework.sim.turbineConfiguration import TurbineInterfaceEx
-from foqus_lib.framework.at_dict.at_dict import AtDict
-from importlib import import_module
 
 _logger = logging.getLogger("foqus." + __name__)
 
@@ -146,7 +148,6 @@ def attempt_load_sklearn(try_imports=True):
     try:
         assert try_imports  # if False will auto-trigger exceptions
         # sklearn should be installed, but not required for non ML/AI models
-        import sklearn
         import pickle
 
         skl_pickle_load = pickle.load
@@ -168,9 +169,10 @@ def attempt_load_smt(try_imports=True):
     try:
         assert try_imports  # if False will auto-trigger exceptions
         # smt should be installed, but not required for non ML/AI models
-        import smt
         import pickle
 
+        import smt
+
         smt_pickle_load = pickle.load
 
     # throw warning if manually failed for test or if package actually not available

foqus_lib/framework/graph/nodeVars.py

@@ -20,12 +20,14 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 
-from collections import OrderedDict
-import numpy as np
+import copy
 import json
-import math
 import logging
-import copy
+import math
+from collections import OrderedDict
+
+import numpy as np
+
 from foqus_lib.framework.foqusException.foqusException import *
 from foqus_lib.framework.uq.Distribution import Distribution
 

foqus_lib/framework/listen/listen.py

@@ -23,12 +23,13 @@
 """
 
 
+import copy
 import logging
-from multiprocessing.connection import Listener
 import threading
 import time
+from multiprocessing.connection import Listener
+
 import numpy as np
-import copy
 
 
 class foqusListener2(threading.Thread):

foqus_lib/framework/ml_ai_models/mlaiSearch.py

@@ -20,11 +20,11 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 
-import sys
-import os
+import imp
 import importlib
 import logging
-import imp
+import os
+import sys
 
 _log = logging.getLogger("foqus." + __name__)
 

foqus_lib/framework/optimizer/BFGS.py

@@ -23,17 +23,19 @@
 
 John Eslick, Carnegie Mellon University, 2014
 """
-import time
 import copy
 import csv
-import pickle
-import queue
-import sys
 import logging
 import math
-import numpy
 import os
+import pickle
+import queue
+import sys
+import time
 import traceback
+
+import numpy
+
 from foqus_lib.framework.optimizer.optimization import optimization
 
 # Check that the BFGS python script is available and import it if

foqus_lib/framework/optimizer/NLopt.py

@@ -25,17 +25,19 @@
 
 John Eslick, Carnegie Mellon University, 2014
 """
-import time  # Some of these things are left over from CMA-ES
 import copy  # too lazy to sort out which I really need in here
 import csv
-import pickle
-import queue
-import sys
 import logging
 import math
-import numpy
 import os
+import pickle
+import queue
+import sys
+import time  # Some of these things are left over from CMA-ES
 import traceback
+
+import numpy
+
 from foqus_lib.framework.optimizer.optimization import optimization
 
 # Check that the NLopt module is available and import it if possible.

foqus_lib/framework/optimizer/OptCMA.py

@@ -32,17 +32,19 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 
-import time
 import copy
 import csv
-import pickle
-import queue
-import sys
 import logging
 import math
-import numpy
 import os
+import pickle
+import queue
+import sys
+import time
 import traceback
+
+import numpy
+
 from foqus_lib.framework.optimizer.optimization import optimization
 
 # Check that the CMA-ES python script is available and import it if

foqus_lib/framework/optimizer/OptSNOBFIT.py

@@ -12,7 +12,9 @@
 # respectively. This file is also available online at the URL
 # "https://github.com/CCSI-Toolset/FOQUS".
 #################################################################################
-import time, copy
+import copy
+import time
+
 import numpy
 
 snobfit_available = True