From 22265d2a972f85c787dfb35480ec531613f40307 Mon Sep 17 00:00:00 2001 From: dghoshal-lbl Date: Thu, 24 Mar 2022 11:42:52 -0400 Subject: [PATCH 1/7] Running OUU tutorial-1 example: exception doing Run OUU --- foqus_lib/gui/tests/test_ouu.py | 175 ++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 foqus_lib/gui/tests/test_ouu.py diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py new file mode 100644 index 000000000..7ec0b595d --- /dev/null +++ b/foqus_lib/gui/tests/test_ouu.py @@ -0,0 +1,175 @@ +import time +import typing +import os + +from PyQt5 import QtWidgets, QtCore +import pytest + +from foqus_lib.gui.ouu.ouuSetupFrame import ouuSetupFrame +from foqus_lib.gui.ouu import foqusPSUADEClient +from foqus_lib.gui.ouu.OUUInputsTable import OUUInputsTable +from foqus_lib.gui.ouu import nodeToUQModel + +from foqus_lib.framework.uq.LocalExecutionModule import LocalExecutionModule +from foqus_lib.gui.common.InputPriorTable import InputPriorTable + + +@pytest.fixture(scope="class") +def setup_frame_blank(main_window, request): + main_window.ouuSetupAction.trigger() + setup_frame: ouuSetupFrame = main_window.ouuSetupFrame + request.cls.frame = setup_frame + return setup_frame + +@pytest.mark.usefixtures("setup_frame_blank") +class TestOUU(): + frame: ouuSetupFrame = ... + ############### + ''' + simple test to check the very basic test case + ''' + @pytest.fixture(scope="class") + def launchWindow(self, qtbot): + qtbot.focused = self.frame + + @pytest.mark.usefixtures("launchWindow") + def test_window(self): + assert True + + ################ + ''' + Fixtures for comprehensive tests + ''' + @pytest.fixture(scope="class") + def selectModel(self, qtbot): + ouu_frame = self.frame + qtbot.focused = ouu_frame + fname = os.path.join( + os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ouu_optdriver.in", + ) + fname = os.path.abspath(fname) + ouu_frame.filesDir, _ = os.path.split(fname) + ouu_frame.modelFile_edit.setText(fname) + ouu_frame.model = LocalExecutionModule.readSampleFromPsuadeFile(fname).model + ouu_frame.input_table.init(ouu_frame.model, InputPriorTable.OUU) + ouu_frame.setFixed_button.setEnabled(True) + ouu_frame.setX1_button.setEnabled(True) + ouu_frame.setX2_button.setEnabled(True) + ouu_frame.setX3_button.setEnabled(True) + ouu_frame.setX4_button.setEnabled(True) + ouu_frame.initTabs() + ouu_frame.setCounts() + + + @pytest.fixture(scope="class") + def setVariables(self, qtbot): + vars = ["Opt: Primary Continuous (Z1)", "Opt: Recourse (Z2)", "UQ: Discrete (Z3)"] + # "Opt: Recourse (Z2)", "UQ: Discrete (Z3)", "UQ: Continuous (Z4)" + with qtbot.focusing_on(table=any): + rownum = 0 + for i in range(len(vars)): + for _ in range(4): + qtbot.select_row(rownum) + qtbot.using(column="Type").set_option(vars[i]) + rownum += 1 + + @pytest.fixture(scope="class") + def selectOptimizer(self, qtbot): + qtbot.select_tab("Optimization Setup") + # with qtbot.focusing_on(group_box="Optimization Solver"): + # combo = qtbot.locate(combo_box=any) + # # optimizer = "Use model as optimizer: min_Z2 G(Z1,Z2,Z3,Z4)" + # # qtbot.using(combo).set_option(vars[i]) + + + + @pytest.fixture(scope="class") + def selectDiscreteRandomVars(self, qtbot): + ouu_frame = self.frame + qtbot.select_tab("UQ Setup") + + fname = os.path.join( + os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ex1_x3sample.smp", + ) + + ouu_frame.filesDir, _ = os.path.split(fname) + + data = LocalExecutionModule.readDataFromSimpleFile( + fname, hasColumnNumbers=False) + data = data[0] + + numInputs = data.shape[1] + M3 = len(ouu_frame.input_table.getUQDiscreteVariables()[0]) + print(data) + + # self.data, self.numInputs, self.M3 = data, numInputs, M3 + + # ouu_frame.loadTable(ouu_frame.z3_table, data) + + if numInputs != M3: + # have to be an assertion + pass + else: + # ouu_frame.compressSamples_chk.setEnabled(True) + with qtbot.focusing_on(group_box=" Discrete Random Variables (Z3)"): + ouu_frame.loadTable(ouu_frame.z3_table, data) + + + @pytest.fixture(scope="class") + def loadRandomVariables(self): + ouu_frame = self.frame + with qtbot.focusing_on(table=ouu_frame.z3_table): + ouu_frame.loadTable(ouu_frame.z3_table, self.data) + + + @pytest.fixture(scope="class") + def launchTest(self, qtbot): + qtbot.select_tab("Launch/Progress") + + def run_and_wait(): + qtbot.click(button="Run OUU") + qtbot.wait_until_called(self.frame.unfreeze) + + run_and_wait() + # run_button = qtbot.locate(button="Run OUU") + # qtbot.click(run_button) + + # with qtbot.focusing_on( + # group_box="Progress" + # ), qtbot.taking_screenshots(): + # # ouu_frame = self.frame + + # with qtbot.select_tab("Launch/Progress"), qtbot.taking_screenshots(): + # qtbot.click(button="Run OUU") + + + ################### + ''' + Comprehensive tests for OUU tutorial 1 + ''' + def testModel(self, qtbot, selectModel, setVariables, selectOptimizer): + assert True + + @pytest.mark.usefixtures("selectDiscreteRandomVars") + def testRandomVarSetting(self): + assert True + + # def testRandomVarSetting(self, qtbot, selectDiscreteRandomVars): + # assert self.numInputs == self.M3 + # with qtbot.searching_within(group_box="Discrete Random Variables (Z3)"): + # var_table = qtbot.locate_widget(table=True) + # assert var_table.rowCount() > 0 + + # @pytest.mark.usefixtures("loadRandomVariables") + # def testRandomVarTable(self, qtbot): + # with qtbot.searching_within(group_box="Discrete Random Variables (Z3)"): + # var_table = qtbot.locate_widget(table=True) + # assert var_table.rowCount() > 0 + + @pytest.mark.usefixtures("launchTest") + def testRun(self): + with qtbot.searching_within(group_box="Launch/Progress"): + sol_table = qtbot.locate_widget(table=True) + assert sol_table.rowCount() > 0 \ No newline at end of file From 0747ea4441f11fcc24262a9dbae8ca1b081cb27c Mon Sep 17 00:00:00 2001 From: dghoshal-lbl Date: Thu, 24 Mar 2022 12:39:07 -0400 Subject: [PATCH 2/7] try2: combining the workflow steps together --- foqus_lib/gui/tests/test_ouu.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py index 7ec0b595d..acb4dd565 100644 --- a/foqus_lib/gui/tests/test_ouu.py +++ b/foqus_lib/gui/tests/test_ouu.py @@ -40,6 +40,11 @@ def test_window(self): ''' Fixtures for comprehensive tests ''' + # def __init__(self): + # self.data = None + # self.numInputs = 0 + # self.M3 = None + @pytest.fixture(scope="class") def selectModel(self, qtbot): ouu_frame = self.frame @@ -102,7 +107,7 @@ def selectDiscreteRandomVars(self, qtbot): numInputs = data.shape[1] M3 = len(ouu_frame.input_table.getUQDiscreteVariables()[0]) - print(data) + # print(data) # self.data, self.numInputs, self.M3 = data, numInputs, M3 @@ -133,6 +138,7 @@ def run_and_wait(): qtbot.wait_until_called(self.frame.unfreeze) run_and_wait() + # qtbot.wait_until(has_dialog, timeout=10_000) # run_button = qtbot.locate(button="Run OUU") # qtbot.click(run_button) @@ -149,12 +155,16 @@ def run_and_wait(): ''' Comprehensive tests for OUU tutorial 1 ''' - def testModel(self, qtbot, selectModel, setVariables, selectOptimizer): + def testModel(self, qtbot, selectModel, setVariables, selectOptimizer, selectDiscreteRandomVars, launchTest): assert True - @pytest.mark.usefixtures("selectDiscreteRandomVars") - def testRandomVarSetting(self): - assert True + # @pytest.mark.usefixtures("selectDiscreteRandomVars") + # def testRandomVarSetting(self): + # assert True + # # assert self.numInputs == self.M3 + # # with qtbot.searching_within(group_box=" Discrete Random Variables (Z3)"): + # # var_table = qtbot.locate_widget(table=True) + # # assert var_table.rowCount() > 0 # def testRandomVarSetting(self, qtbot, selectDiscreteRandomVars): # assert self.numInputs == self.M3 @@ -168,8 +178,9 @@ def testRandomVarSetting(self): # var_table = qtbot.locate_widget(table=True) # assert var_table.rowCount() > 0 - @pytest.mark.usefixtures("launchTest") - def testRun(self): - with qtbot.searching_within(group_box="Launch/Progress"): - sol_table = qtbot.locate_widget(table=True) - assert sol_table.rowCount() > 0 \ No newline at end of file + # @pytest.mark.usefixtures("launchTest") + # def testRun(self, qtbot): + # assert True + # # with qtbot.searching_within(group_box="Best So Far"): + # # sol_table = qtbot.locate_widget(table=True) + # # assert sol_table.rowCount() > 0 \ No newline at end of file From 25340ae06736711ed5f9579d4d1b156f79759e7c Mon Sep 17 00:00:00 2001 From: dghoshal-lbl Date: Sat, 26 Mar 2022 17:22:00 -0400 Subject: [PATCH 3/7] test module for OUU Example 1 workflow --- foqus_lib/gui/tests/test_ouu.py | 254 ++++++++++++++++++++------------ 1 file changed, 157 insertions(+), 97 deletions(-) diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py index acb4dd565..6d761ad38 100644 --- a/foqus_lib/gui/tests/test_ouu.py +++ b/foqus_lib/gui/tests/test_ouu.py @@ -1,3 +1,13 @@ +''' +Test module for FOQUS OUU Tutorial Example 1: `OUU with Discrete Uncertain Parameters Only` +URL: https://foqus.readthedocs.io/en/stable/chapt_ouu/tutorial.html#example-1-ouu-with-discrete-uncertain-parameters-only + +To run the test: +pytest -k test_ouu [--slowdown-wait=] + +Author: Devarshi Ghoshal +''' + import time import typing import os @@ -13,20 +23,33 @@ from foqus_lib.framework.uq.LocalExecutionModule import LocalExecutionModule from foqus_lib.gui.common.InputPriorTable import InputPriorTable +from PyQt5.QtWidgets import QMessageBox + @pytest.fixture(scope="class") def setup_frame_blank(main_window, request): + """ + Sets up a blank OUU Frame. + + Args: + main_window: FOQUS main GUI window. Instance of foqus_lib.gui.main.mainWindow.mainWindow. + request: pytest request. + + Returns: + ouuSetupFrame: FOQUS OUU UI. + """ main_window.ouuSetupAction.trigger() setup_frame: ouuSetupFrame = main_window.ouuSetupFrame request.cls.frame = setup_frame return setup_frame + @pytest.mark.usefixtures("setup_frame_blank") class TestOUU(): frame: ouuSetupFrame = ... ############### ''' - simple test to check the very basic test case + Simple test to check the very basic test- launching the FOQUS main window. ''' @pytest.fixture(scope="class") def launchWindow(self, qtbot): @@ -38,22 +61,49 @@ def test_window(self): ################ ''' - Fixtures for comprehensive tests + Fixtures for comprehensive tests for OUU example 1. ''' - # def __init__(self): - # self.data = None - # self.numInputs = 0 - # self.M3 = None + @pytest.fixture(scope="class") + def model_file(self): + model_file_name = os.path.join(os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ouu_optdriver.in") + return model_file_name + + @pytest.fixture(scope="class") + def model_file_button_label(self): + label = "Load Model From File" + return label @pytest.fixture(scope="class") - def selectModel(self, qtbot): + def ouu_variables(self): + ouu_vars = ["Opt: Primary Continuous (Z1)", "Opt: Recourse (Z2)", "UQ: Discrete (Z3)"] + # "Opt: Recourse (Z2)", "UQ: Discrete (Z3)", "UQ: Continuous (Z4)" + return ouu_vars + + @pytest.fixture(scope="class") + def sample_file(self): + sample_file_name = os.path.join(os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ex1_x3sample.smp") + return sample_file_name + + @pytest.fixture(scope="class") + def exec_timeout(self): + timeout = 90_000 + return timeout + + @pytest.fixture(scope="class") + def selectModel(self, qtbot, model_file, model_file_button_label): + """ + [Step-1] Select the model from an example file. + + Args: + qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. + model_file: (from fixture) location of the example model file. + model_file_button_label: (from fixture) label for the radio button for selecting model file. + """ ouu_frame = self.frame qtbot.focused = ouu_frame - fname = os.path.join( - os.path.dirname(__file__), - "../../../examples/tutorial_files/OUU/ouu_optdriver.in", - ) - fname = os.path.abspath(fname) + fname = os.path.abspath(model_file) ouu_frame.filesDir, _ = os.path.split(fname) ouu_frame.modelFile_edit.setText(fname) ouu_frame.model = LocalExecutionModule.readSampleFromPsuadeFile(fname).model @@ -65,122 +115,132 @@ def selectModel(self, qtbot): ouu_frame.setX4_button.setEnabled(True) ouu_frame.initTabs() ouu_frame.setCounts() - + qtbot.click(radio_button=model_file_button_label) + @pytest.fixture(scope="class") - def setVariables(self, qtbot): - vars = ["Opt: Primary Continuous (Z1)", "Opt: Recourse (Z2)", "UQ: Discrete (Z3)"] - # "Opt: Recourse (Z2)", "UQ: Discrete (Z3)", "UQ: Continuous (Z4)" + def setVariables(self, qtbot, ouu_variables): + """ + [Step-2] Set OUU variable types. + + Args: + qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. + ouu_variables: (from fixture) variable types from a dropdown list. + """ with qtbot.focusing_on(table=any): rownum = 0 - for i in range(len(vars)): + for i in range(len(ouu_variables)): for _ in range(4): qtbot.select_row(rownum) - qtbot.using(column="Type").set_option(vars[i]) + qtbot.using(column="Type").set_option(ouu_variables[i]) rownum += 1 @pytest.fixture(scope="class") def selectOptimizer(self, qtbot): + """ + [Step-3] Select the optimizer for the test. It should be the default + optimizer BOBYQA for this test, will all the default settings. + + Args: + qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. + """ qtbot.select_tab("Optimization Setup") - # with qtbot.focusing_on(group_box="Optimization Solver"): - # combo = qtbot.locate(combo_box=any) - # # optimizer = "Use model as optimizer: min_Z2 G(Z1,Z2,Z3,Z4)" - # # qtbot.using(combo).set_option(vars[i]) + with qtbot.focusing_on(group_box="Objective Function for Optimization Under Uncertainty (OUU)"): + qtbot.click(radio_button="Mean of G(Z1,Z2,Z3,Z4) with respect to Z3 and Z4") + @pytest.fixture(scope="class") + def discreteVars(self, qtbot, sample_file): + """ + [Step-4] Set up the discrete variables from a simple example file. + Args: + qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. + sample_file: simple input file from examples. - @pytest.fixture(scope="class") - def selectDiscreteRandomVars(self, qtbot): + Returns: + [type]: Discrete random variables (Z3). + """ ouu_frame = self.frame qtbot.select_tab("UQ Setup") - fname = os.path.join( - os.path.dirname(__file__), - "../../../examples/tutorial_files/OUU/ex1_x3sample.smp", - ) - - ouu_frame.filesDir, _ = os.path.split(fname) + ouu_frame.filesDir, _ = os.path.split(sample_file) data = LocalExecutionModule.readDataFromSimpleFile( - fname, hasColumnNumbers=False) - data = data[0] - - numInputs = data.shape[1] - M3 = len(ouu_frame.input_table.getUQDiscreteVariables()[0]) - # print(data) + sample_file, hasColumnNumbers=False) - # self.data, self.numInputs, self.M3 = data, numInputs, M3 - - # ouu_frame.loadTable(ouu_frame.z3_table, data) + data = data[0] - if numInputs != M3: - # have to be an assertion - pass - else: - # ouu_frame.compressSamples_chk.setEnabled(True) - with qtbot.focusing_on(group_box=" Discrete Random Variables (Z3)"): - ouu_frame.loadTable(ouu_frame.z3_table, data) + with qtbot.focusing_on(group_box=" Discrete Random Variables (Z3)"): + ouu_frame.compressSamples_chk.setEnabled(True) + ouu_frame.loadTable(ouu_frame.z3_table, data) + return data @pytest.fixture(scope="class") - def loadRandomVariables(self): - ouu_frame = self.frame - with qtbot.focusing_on(table=ouu_frame.z3_table): - ouu_frame.loadTable(ouu_frame.z3_table, self.data) - - - @pytest.fixture(scope="class") - def launchTest(self, qtbot): + def launchTest(self, qtbot, exec_timeout): + """ + [Step-5] Final step to run the optimizer and plot the graph. + + Args: + qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. + exec_timeout: timeout to run the optimizer and finish executing the workflow. + """ qtbot.select_tab("Launch/Progress") - def run_and_wait(): + with qtbot.waiting_for_modal(timeout=exec_timeout): qtbot.click(button="Run OUU") - qtbot.wait_until_called(self.frame.unfreeze) - - run_and_wait() - # qtbot.wait_until(has_dialog, timeout=10_000) - # run_button = qtbot.locate(button="Run OUU") - # qtbot.click(run_button) - - # with qtbot.focusing_on( - # group_box="Progress" - # ), qtbot.taking_screenshots(): - # # ouu_frame = self.frame - - # with qtbot.select_tab("Launch/Progress"), qtbot.taking_screenshots(): - # qtbot.click(button="Run OUU") ################### ''' Comprehensive tests for OUU tutorial 1 ''' - def testModel(self, qtbot, selectModel, setVariables, selectOptimizer, selectDiscreteRandomVars, launchTest): + @pytest.mark.usefixtures("selectModel") + def testModelSelection(self): + """ + [Test-1] Test that the correct model input file is selected and + the radio button is selected, else the test fails. + """ + model_file = self.frame.modelFile_edit.text() + assert os.path.basename(model_file) == "ouu_optdriver.in" + assert self.frame.modelFile_radio.isChecked() + + @pytest.mark.usefixtures("setVariables") + def testVariables(self): + """ + [Test-2] Test that the correct variables - Z1, Z2, Z3 - are set. + """ + fixed_text = self.frame.fixedCount_static.text() + x1_text = self.frame.x1Count_static.text() + x2_text = self.frame.x2Count_static.text() + x3_text = self.frame.x3Count_static.text() + x4_text = self.frame.x4Count_static.text() + assert (fixed_text == "# Fixed: 0" and + x1_text == "# Primary Opt Vars: 4" and + x2_text == "# Recourse Opt Vars: 4" and + x3_text == "# Discrete RVs: 4" and + x4_text == "# Continuous RVs: 0") + + @pytest.mark.usefixtures("selectOptimizer") + def testOptimizer(self): + """ + [Test-3] Test that BOBYQA is selected as the optimizer. + """ + assert self.frame.mean_radio.isChecked() + assert self.frame.primarySolver_combo.currentText() == "BOBYQA" + assert self.frame.secondarySolver_combo.currentText() == "Use model as optimizer: min_Z2 G(Z1,Z2,Z3,Z4)" + + def testRandomVars(self, discreteVars): + """ + [Test-4] Test that the discrete variables are selected appropriately. + """ + n_inps = discreteVars.shape[1] + n_vars = len(self.frame.input_table.getUQDiscreteVariables()[0]) + assert n_inps == n_vars + + @pytest.mark.usefixtures("launchTest") + def testRunOUU(self): + """ + [Test-5] Test that the optimizer launches and finishes. + """ assert True - - # @pytest.mark.usefixtures("selectDiscreteRandomVars") - # def testRandomVarSetting(self): - # assert True - # # assert self.numInputs == self.M3 - # # with qtbot.searching_within(group_box=" Discrete Random Variables (Z3)"): - # # var_table = qtbot.locate_widget(table=True) - # # assert var_table.rowCount() > 0 - - # def testRandomVarSetting(self, qtbot, selectDiscreteRandomVars): - # assert self.numInputs == self.M3 - # with qtbot.searching_within(group_box="Discrete Random Variables (Z3)"): - # var_table = qtbot.locate_widget(table=True) - # assert var_table.rowCount() > 0 - - # @pytest.mark.usefixtures("loadRandomVariables") - # def testRandomVarTable(self, qtbot): - # with qtbot.searching_within(group_box="Discrete Random Variables (Z3)"): - # var_table = qtbot.locate_widget(table=True) - # assert var_table.rowCount() > 0 - - # @pytest.mark.usefixtures("launchTest") - # def testRun(self, qtbot): - # assert True - # # with qtbot.searching_within(group_box="Best So Far"): - # # sol_table = qtbot.locate_widget(table=True) - # # assert sol_table.rowCount() > 0 \ No newline at end of file From ed346d560620e43feda19c1bcc16bcc9c267537c Mon Sep 17 00:00:00 2001 From: dghoshal-lbl Date: Sat, 26 Mar 2022 17:32:06 -0400 Subject: [PATCH 4/7] test module for OUU Example 1 workflow- edit comment --- foqus_lib/gui/tests/test_ouu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py index 6d761ad38..59d63dd73 100644 --- a/foqus_lib/gui/tests/test_ouu.py +++ b/foqus_lib/gui/tests/test_ouu.py @@ -94,7 +94,10 @@ def exec_timeout(self): @pytest.fixture(scope="class") def selectModel(self, qtbot, model_file, model_file_button_label): """ - [Step-1] Select the model from an example file. + [Step-1] Select the model from an example file. + TODO: The code below needs to be called through a function in ouuSetupFrame.py. + Currently, this is too tightly integrated and can not be reused correctly + for the test. Hence, copied and used here directly. Args: qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. From 3b7ca97ef32771b1aeaafa192e3ff8a2ef995a09 Mon Sep 17 00:00:00 2001 From: dghoshal-lbl Date: Sat, 26 Mar 2022 23:16:56 -0400 Subject: [PATCH 5/7] fix formatting --- foqus_lib/gui/tests/test_ouu.py | 84 ++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py index 59d63dd73..efca2bab3 100644 --- a/foqus_lib/gui/tests/test_ouu.py +++ b/foqus_lib/gui/tests/test_ouu.py @@ -1,4 +1,4 @@ -''' +""" Test module for FOQUS OUU Tutorial Example 1: `OUU with Discrete Uncertain Parameters Only` URL: https://foqus.readthedocs.io/en/stable/chapt_ouu/tutorial.html#example-1-ouu-with-discrete-uncertain-parameters-only @@ -6,7 +6,7 @@ pytest -k test_ouu [--slowdown-wait=] Author: Devarshi Ghoshal -''' +""" import time import typing @@ -45,28 +45,32 @@ def setup_frame_blank(main_window, request): @pytest.mark.usefixtures("setup_frame_blank") -class TestOUU(): +class TestOUU: frame: ouuSetupFrame = ... ############### - ''' + """ Simple test to check the very basic test- launching the FOQUS main window. - ''' + """ + @pytest.fixture(scope="class") def launchWindow(self, qtbot): qtbot.focused = self.frame - + @pytest.mark.usefixtures("launchWindow") def test_window(self): assert True ################ - ''' + """ Fixtures for comprehensive tests for OUU example 1. - ''' + """ + @pytest.fixture(scope="class") def model_file(self): - model_file_name = os.path.join(os.path.dirname(__file__), - "../../../examples/tutorial_files/OUU/ouu_optdriver.in") + model_file_name = os.path.join( + os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ouu_optdriver.in", + ) return model_file_name @pytest.fixture(scope="class") @@ -76,14 +80,20 @@ def model_file_button_label(self): @pytest.fixture(scope="class") def ouu_variables(self): - ouu_vars = ["Opt: Primary Continuous (Z1)", "Opt: Recourse (Z2)", "UQ: Discrete (Z3)"] + ouu_vars = [ + "Opt: Primary Continuous (Z1)", + "Opt: Recourse (Z2)", + "UQ: Discrete (Z3)", + ] # "Opt: Recourse (Z2)", "UQ: Discrete (Z3)", "UQ: Continuous (Z4)" return ouu_vars @pytest.fixture(scope="class") def sample_file(self): - sample_file_name = os.path.join(os.path.dirname(__file__), - "../../../examples/tutorial_files/OUU/ex1_x3sample.smp") + sample_file_name = os.path.join( + os.path.dirname(__file__), + "../../../examples/tutorial_files/OUU/ex1_x3sample.smp", + ) return sample_file_name @pytest.fixture(scope="class") @@ -94,10 +104,10 @@ def exec_timeout(self): @pytest.fixture(scope="class") def selectModel(self, qtbot, model_file, model_file_button_label): """ - [Step-1] Select the model from an example file. + [Step-1] Select the model from an example file. TODO: The code below needs to be called through a function in ouuSetupFrame.py. Currently, this is too tightly integrated and can not be reused correctly - for the test. Hence, copied and used here directly. + for the test. Hence, copied and used here directly. Args: qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. @@ -120,7 +130,7 @@ def selectModel(self, qtbot, model_file, model_file_button_label): ouu_frame.setCounts() qtbot.click(radio_button=model_file_button_label) - + @pytest.fixture(scope="class") def setVariables(self, qtbot, ouu_variables): """ @@ -148,13 +158,15 @@ def selectOptimizer(self, qtbot): qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. """ qtbot.select_tab("Optimization Setup") - with qtbot.focusing_on(group_box="Objective Function for Optimization Under Uncertainty (OUU)"): + with qtbot.focusing_on( + group_box="Objective Function for Optimization Under Uncertainty (OUU)" + ): qtbot.click(radio_button="Mean of G(Z1,Z2,Z3,Z4) with respect to Z3 and Z4") @pytest.fixture(scope="class") def discreteVars(self, qtbot, sample_file): """ - [Step-4] Set up the discrete variables from a simple example file. + [Step-4] Set up the discrete variables from a simple example file. Args: qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. @@ -169,7 +181,8 @@ def discreteVars(self, qtbot, sample_file): ouu_frame.filesDir, _ = os.path.split(sample_file) data = LocalExecutionModule.readDataFromSimpleFile( - sample_file, hasColumnNumbers=False) + sample_file, hasColumnNumbers=False + ) data = data[0] @@ -182,7 +195,7 @@ def discreteVars(self, qtbot, sample_file): @pytest.fixture(scope="class") def launchTest(self, qtbot, exec_timeout): """ - [Step-5] Final step to run the optimizer and plot the graph. + [Step-5] Final step to run the optimizer and plot the graph. Args: qtbot: pytest_qt_extras QtBot to test/interact with FOQUS GUI. @@ -191,18 +204,18 @@ def launchTest(self, qtbot, exec_timeout): qtbot.select_tab("Launch/Progress") with qtbot.waiting_for_modal(timeout=exec_timeout): - qtbot.click(button="Run OUU") - + qtbot.click(button="Run OUU") ################### - ''' + """ Comprehensive tests for OUU tutorial 1 - ''' + """ + @pytest.mark.usefixtures("selectModel") def testModelSelection(self): """ [Test-1] Test that the correct model input file is selected and - the radio button is selected, else the test fails. + the radio button is selected, else the test fails. """ model_file = self.frame.modelFile_edit.text() assert os.path.basename(model_file) == "ouu_optdriver.in" @@ -211,27 +224,32 @@ def testModelSelection(self): @pytest.mark.usefixtures("setVariables") def testVariables(self): """ - [Test-2] Test that the correct variables - Z1, Z2, Z3 - are set. + [Test-2] Test that the correct variables - Z1, Z2, Z3 - are set. """ fixed_text = self.frame.fixedCount_static.text() x1_text = self.frame.x1Count_static.text() x2_text = self.frame.x2Count_static.text() x3_text = self.frame.x3Count_static.text() x4_text = self.frame.x4Count_static.text() - assert (fixed_text == "# Fixed: 0" and - x1_text == "# Primary Opt Vars: 4" and - x2_text == "# Recourse Opt Vars: 4" and - x3_text == "# Discrete RVs: 4" and - x4_text == "# Continuous RVs: 0") + assert ( + fixed_text == "# Fixed: 0" + and x1_text == "# Primary Opt Vars: 4" + and x2_text == "# Recourse Opt Vars: 4" + and x3_text == "# Discrete RVs: 4" + and x4_text == "# Continuous RVs: 0" + ) @pytest.mark.usefixtures("selectOptimizer") def testOptimizer(self): """ - [Test-3] Test that BOBYQA is selected as the optimizer. + [Test-3] Test that BOBYQA is selected as the optimizer. """ assert self.frame.mean_radio.isChecked() assert self.frame.primarySolver_combo.currentText() == "BOBYQA" - assert self.frame.secondarySolver_combo.currentText() == "Use model as optimizer: min_Z2 G(Z1,Z2,Z3,Z4)" + assert ( + self.frame.secondarySolver_combo.currentText() + == "Use model as optimizer: min_Z2 G(Z1,Z2,Z3,Z4)" + ) def testRandomVars(self, discreteVars): """ From c6dbcedca94320259ed38bb9fc6d8a5c761f2ecb Mon Sep 17 00:00:00 2001 From: Ludovico Bianchi Date: Wed, 6 Apr 2022 19:00:59 -0500 Subject: [PATCH 6/7] Try using another handler for modal dialog Try alternative strategy to handle modal dialogs Run Black Try with alternative strategy to handle main window close event Run Black Try if using --pyargs makes a difference Try adding explicit cleanup for qtbot instance --- .github/workflows/checks.yml | 2 +- foqus_lib/gui/tests/conftest.py | 15 +++- foqus_lib/gui/tests/test_main_window.py | 2 +- foqus_lib/gui/tests/test_ouu.py | 22 +++-- pytest_qt_extras.py | 115 +++++++++++++++++------- 5 files changed, 111 insertions(+), 45 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e61c03b74..8ee1184c6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -92,7 +92,7 @@ jobs: pip-install-target: -r requirements-dev.txt ${{ matrix.optional-dependencies }} - name: Set common pytest flags run: - echo 'PYTEST_ADDOPTS=--verbose --artifacts-path "$PYTEST_ARTIFACTS_PATH"' >> $GITHUB_ENV + echo 'PYTEST_ADDOPTS=--pyargs foqus_lib --verbose --artifacts-path "$PYTEST_ARTIFACTS_PATH"' >> $GITHUB_ENV - name: Set up GUI test environment (Linux) if: contains(matrix.os, 'linux') run: | diff --git a/foqus_lib/gui/tests/conftest.py b/foqus_lib/gui/tests/conftest.py index 749f08aa8..4e32d9004 100644 --- a/foqus_lib/gui/tests/conftest.py +++ b/foqus_lib/gui/tests/conftest.py @@ -46,7 +46,7 @@ def qtbot(request, qapp, qtbot_params) -> pytest_qt_extras.QtBot: yield _qtbot if exceptions: pytest.fail(format_captured_exceptions(exceptions)) - _qtbot.describe() + # _qtbot.cleanup() @pytest.fixture(scope="session") @@ -81,15 +81,22 @@ def main_window(foqus_session, qtbot, main_window_params): showSDOE=False, ts=False, ) - main_win.closeEvent = lambda *args, **kwargs: None print(f"main_win={main_win}") main_win.app = QtWidgets.QApplication.instance() print(f"main_win.app={main_win.app}") - # qtbot.addWidget(main_win) + # qtbot.add_widget(main_win) qtbot.waitForWindowShown(main_win) print(f"main_win.app.activeWindow()={main_win.app.activeWindow()}") yield main_win - main_win.close() + + def handle_closing_prompt(w: QtWidgets.QMessageBox): + return QtWidgets.QMessageBox.No + + with pytest_qt_extras._ModalPatcher(QtWidgets.QMessageBox).patching( + handle_closing_prompt + ): + main_win.close() + qtbot.cleanup() @pytest.fixture(scope="class") diff --git a/foqus_lib/gui/tests/test_main_window.py b/foqus_lib/gui/tests/test_main_window.py index a39a35519..f7bbe3a6b 100644 --- a/foqus_lib/gui/tests/test_main_window.py +++ b/foqus_lib/gui/tests/test_main_window.py @@ -8,4 +8,4 @@ def test_main_window_opens(qtbot, main_window): qtbot.slow_down() - main_window.close() + assert main_window is not None diff --git a/foqus_lib/gui/tests/test_ouu.py b/foqus_lib/gui/tests/test_ouu.py index efca2bab3..4f51f0976 100644 --- a/foqus_lib/gui/tests/test_ouu.py +++ b/foqus_lib/gui/tests/test_ouu.py @@ -44,6 +44,13 @@ def setup_frame_blank(main_window, request): return setup_frame +def _accept_dialog(w): + import time + + time.sleep(1) + w.buttonBox.accepted.emit() + + @pytest.mark.usefixtures("setup_frame_blank") class TestOUU: frame: ouuSetupFrame = ... @@ -193,7 +200,7 @@ def discreteVars(self, qtbot, sample_file): return data @pytest.fixture(scope="class") - def launchTest(self, qtbot, exec_timeout): + def runUntilConfirmationDialog(self, qtbot, exec_timeout): """ [Step-5] Final step to run the optimizer and plot the graph. @@ -203,8 +210,11 @@ def launchTest(self, qtbot, exec_timeout): """ qtbot.select_tab("Launch/Progress") - with qtbot.waiting_for_modal(timeout=exec_timeout): - qtbot.click(button="Run OUU") + qtbot.click(button="Run OUU") + with qtbot.waiting_for_dialog( + dialog_cls=QtWidgets.QMessageBox, timeout=exec_timeout + ) as dialog: + yield dialog ################### """ @@ -259,9 +269,9 @@ def testRandomVars(self, discreteVars): n_vars = len(self.frame.input_table.getUQDiscreteVariables()[0]) assert n_inps == n_vars - @pytest.mark.usefixtures("launchTest") - def testRunOUU(self): + def testRunOUU(self, runUntilConfirmationDialog): """ [Test-5] Test that the optimizer launches and finishes. """ - assert True + dialog = runUntilConfirmationDialog + assert "finished" in dialog.text diff --git a/pytest_qt_extras.py b/pytest_qt_extras.py index 7ec98de5f..96079b1e6 100644 --- a/pytest_qt_extras.py +++ b/pytest_qt_extras.py @@ -5,6 +5,7 @@ import logging from pathlib import Path import typing as t +from types import ModuleType import time try: # available starting with Python 3.8 @@ -214,46 +215,74 @@ def _do_nothing(*args, **kwargs): _logger.debug("monkeypatching done") -@contextlib.contextmanager -def replace_with_signal(target, signal, retval=None): - mp = MonkeyPatch() +@dataclass +class _DialogProxy: + window_title: str = "" + text: str = "" - _logger.info(f"replacing target {target} with signal {signal}") - if isinstance(target, tuple) and len(target) == 2: - owner, name = target - instance = None - # TODO check if methodtype? - else: - instance = getattr(target, "__self__", None) - owner = instance.__class__ - name = target.__name__ - func = getattr(owner, name) - assert callable(func), f"{func} must be callable" - _logger.debug(dict(target=target, name=name, owner=owner, func=func)) - def _proxy_call(*args, **kwargs): - call_info = CallInfo( - callee=func, args=args, kwargs=kwargs, name=name, instance=instance +@dataclass +class _ModalPatcher: + + dialog_cls: type + scope: t.Optional[ModuleType] = None + + def __post_init__(self): + if self.scope: + assert isinstance(self.scope, ModuleType), ( + "If given, 'scope' should be a module object, " + f"but it is {type(self.scope)} instead" + ) + self._patch = MonkeyPatch() + + def _apply_patch(self, owner: object, name: str, replacement: object): + self._patch.setattr( + owner, + name, + replacement, ) - signal.emit(call_info) + def _replace_entire_class(self, replacement: type): + self._apply_patch( + owner=self.scope, name=self.dialog_cls.__name__, replacement=replacement + ) - return retval + def _replace_individual_methods(self, replacement: type, method_names: t.List[str]): + for meth_name in method_names: + self._apply_patch( + owner=self.dialog_cls, + name=meth_name, + replacement=getattr(replacement, meth_name), + ) - mp.setattr(owner, name, _proxy_call) + @contextlib.contextmanager + def patching(self, dispatch_func: t.Callable): - patched_info = _WrappedCallable( - wrapped=func, - name=name, - instance=instance, - wrapper=_proxy_call, - ) + # TODO: consider if using type(...) to build the class object would make things clearer + class tmp(self.dialog_cls): + def exec_(inst): + return dispatch_func(inst) - _logger.debug("returning patched object") - yield patched_info - _logger.debug("start undoing monkeypatching") - mp.undo() - _logger.debug("monkeypatching done") + def exec(inst): + return dispatch_func(inst) + + def show(inst): + dispatch_func(inst) + + def open(inst): + dispatch_func(inst) + + if self.scope is not None: + self._replace_entire_class(tmp) + else: + self._replace_individual_methods( + tmp, method_names=["exec_", "exec", "show", "open"] + ) + + try: + yield self + finally: + self._patch.undo() class _Signals(QtCore.QObject): @@ -264,7 +293,7 @@ class _Signals(QtCore.QObject): actionEnd = QtCore.pyqtSignal(Action) locateBegin = QtCore.pyqtSignal(object) locateEnd = QtCore.pyqtSignal(object) - callProxy = QtCore.pyqtSignal(CallInfo) + dialogDisplay = QtCore.pyqtSignal(_DialogProxy) @property def by_type_and_when(self): @@ -1579,3 +1608,23 @@ def _to_be_called_async(): signal.disconnect(_modal_slot) waiting_for_modal = intercepting_modal + + @contextlib.contextmanager + def waiting_for_dialog(self, timeout=0, dialog_cls=W.QMessageBox) -> _DialogProxy: + signal = self._signals.dialogDisplay + + def dispatch(modal: dialog_cls): + proxy = _DialogProxy(text=modal.text()) + print(proxy) + signal.emit(proxy) + return True + # return modal.defaultButton() + + with _ModalPatcher(dialog_cls).patching(dispatch): + with self.wait_signal(signal, timeout=timeout) as blocker: + pass + dialog_info_for_checking: _DialogProxy = blocker.args[0] + yield dialog_info_for_checking + + def cleanup(self): + self._focus_stack.clear() From b893e5dd717aebdaa0a32338285be455fc5f42e5 Mon Sep 17 00:00:00 2001 From: Ludovico Bianchi Date: Thu, 14 Apr 2022 13:45:45 -0500 Subject: [PATCH 7/7] Restore signal replacement functionality used by test_ml_ai.py --- pytest_qt_extras.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pytest_qt_extras.py b/pytest_qt_extras.py index 96079b1e6..0b5dcca4a 100644 --- a/pytest_qt_extras.py +++ b/pytest_qt_extras.py @@ -285,6 +285,48 @@ def open(inst): self._patch.undo() +@contextlib.contextmanager +def replace_with_signal(target, signal, retval=None): + mp = MonkeyPatch() + + _logger.info(f"replacing target {target} with signal {signal}") + if isinstance(target, tuple) and len(target) == 2: + owner, name = target + instance = None + # TODO check if methodtype? + else: + instance = getattr(target, "__self__", None) + owner = instance.__class__ + name = target.__name__ + func = getattr(owner, name) + assert callable(func), f"{func} must be callable" + _logger.debug(dict(target=target, name=name, owner=owner, func=func)) + + def _proxy_call(*args, **kwargs): + call_info = CallInfo( + callee=func, args=args, kwargs=kwargs, name=name, instance=instance + ) + + signal.emit(call_info) + + return retval + + mp.setattr(owner, name, _proxy_call) + + patched_info = _WrappedCallable( + wrapped=func, + name=name, + instance=instance, + wrapper=_proxy_call, + ) + + _logger.debug("returning patched object") + yield patched_info + _logger.debug("start undoing monkeypatching") + mp.undo() + _logger.debug("monkeypatching done") + + class _Signals(QtCore.QObject): __instance = None callBegin = QtCore.pyqtSignal(CallInfo) @@ -293,6 +335,7 @@ class _Signals(QtCore.QObject): actionEnd = QtCore.pyqtSignal(Action) locateBegin = QtCore.pyqtSignal(object) locateEnd = QtCore.pyqtSignal(object) + callProxy = QtCore.pyqtSignal(CallInfo) dialogDisplay = QtCore.pyqtSignal(_DialogProxy) @property