From 4fb3df1c37fadd78ddcfdddd783fb61cd997c051 Mon Sep 17 00:00:00 2001 From: Sebastian Uerlich Date: Wed, 18 May 2022 12:28:03 +0200 Subject: [PATCH 1/4] Add support for versions of SCIP >= 8 --- pyomo/opt/solver/shellcmd.py | 1 + pyomo/solvers/plugins/solvers/SCIPAMPL.py | 54 +++++++++++++++++------ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pyomo/opt/solver/shellcmd.py b/pyomo/opt/solver/shellcmd.py index f21959c083f..84b74b16344 100644 --- a/pyomo/opt/solver/shellcmd.py +++ b/pyomo/opt/solver/shellcmd.py @@ -328,6 +328,7 @@ def _execute_command(self,command): stderr=t.STDERR, timeout=timeout, universal_newlines=True, + cwd=command.cwd if "cwd" in command else None, ) t.STDOUT.flush() t.STDERR.flush() diff --git a/pyomo/solvers/plugins/solvers/SCIPAMPL.py b/pyomo/solvers/plugins/solvers/SCIPAMPL.py index 3da2b74730e..0a692ddfb62 100644 --- a/pyomo/solvers/plugins/solvers/SCIPAMPL.py +++ b/pyomo/solvers/plugins/solvers/SCIPAMPL.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ import os +import os.path import subprocess from pyomo.common import Executable @@ -30,6 +31,10 @@ class SCIPAMPL(SystemCallSolver): """A generic optimizer that uses the AMPL Solver Library to interface with applications. """ + # Cache default executable, so we do not need to repeatedly query the + # versions every time. + _known_versions = {} + def __init__(self, **kwds): # # Call base constructor @@ -58,22 +63,36 @@ def _default_results_format(self, prob_format): return ResultsFormat.sol def _default_executable(self): + + executable = Executable("scip") + + if executable: + executable_path = executable.path() + if executable_path not in self._known_versions: + self._known_versions[executable_path] = self._get_version(executable_path) + _ver = self._known_versions[executable_path] + if _ver and _ver >= (8,): + return executable_path + + # revert to scipampl for older versions executable = Executable("scipampl") if not executable: - logger.warning("Could not locate the 'scipampl' executable, " - "which is required for solver %s" % self.name) + logger.warning("Could not locate the 'scipampl' executable or" + " the 'scip' executable since 8.0.0, which is" + "required for solver %s" % self.name) self.enable = False return None return executable.path() - def _get_version(self): + def _get_version(self, solver_exec=None): """ Returns a tuple describing the solver executable version. """ - solver_exec = self.executable() if solver_exec is None: - return _extract_version('') - results = subprocess.run( [solver_exec], timeout=1, + solver_exec = self.executable() + if solver_exec is None: + return _extract_version('') + results = subprocess.run([solver_exec, "--version"], timeout=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) @@ -120,7 +139,17 @@ def create_command_line(self, executable, problem_files): else: env['AMPLFUNC'] = env['PYOMO_AMPLFUNC'] - cmd = [executable, problem_files[0], '-AMPL'] + # Since Version 8.0.0 .nl problem file paths should be provided without the .nl + # extension + if executable not in self._known_versions: + self._known_versions[executable] = self._get_version(executable) + _ver = self._known_versions[executable] + if _ver and _ver > (8, 0, 0): + problem_file = os.path.splitext(problem_files[0])[0] + else: + problem_file = problem_files[0] + + cmd = [executable, problem_file, '-AMPL'] if self._timer: cmd.insert(0, self._timer) @@ -156,16 +185,15 @@ def create_command_line(self, executable, problem_files): "file '%s' will be ignored." % (default_of_name, default_of_name)) + options_dir = TempfileManager.create_tempdir() # Now write the new options file - options_filename = TempfileManager.\ - create_tempfile(suffix="_scip.set") - with open(options_filename, "w") as f: + with open(os.path.join(options_dir, 'scip.set'), 'w') as f: for line in of_opt: f.write(line+"\n") + else: + options_dir = None - cmd.append(options_filename) - - return Bunch(cmd=cmd, log_file=self._log_file, env=env) + return Bunch(cmd=cmd, log_file=self._log_file, env=env, cwd=options_dir) def _postsolve(self): results = super(SCIPAMPL, self)._postsolve() From de3bab6263f55cce3e930f5234388e1e3bff769c Mon Sep 17 00:00:00 2001 From: Sebastian Uerlich Date: Wed, 18 May 2022 12:36:47 +0200 Subject: [PATCH 2/4] Add timelimit to SCIP options --- pyomo/solvers/plugins/solvers/SCIPAMPL.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/SCIPAMPL.py b/pyomo/solvers/plugins/solvers/SCIPAMPL.py index 0a692ddfb62..9cb69001e8b 100644 --- a/pyomo/solvers/plugins/solvers/SCIPAMPL.py +++ b/pyomo/solvers/plugins/solvers/SCIPAMPL.py @@ -168,6 +168,9 @@ def create_command_line(self, executable, problem_files): env_opt.append(key+"="+str(self.options[key])) of_opt.append(str(key)+" = "+str(self.options[key])) + if self._timelimit is not None and self._timelimit > 0.0 and 'limits/time' not in self.options: + of_opt.append("limits/time = " + str(self._timelimit)) + envstr = "%s_options" % self.options.solver # Merge with any options coming in through the environment env[envstr] = " ".join(env_opt) From 001ae2b7fea6f9093e7d505160e1462fab8a9771 Mon Sep 17 00:00:00 2001 From: Sebastian Uerlich Date: Thu, 26 May 2022 15:47:22 +0200 Subject: [PATCH 3/4] Improve SCIPAMPL readability --- pyomo/solvers/plugins/solvers/SCIPAMPL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SCIPAMPL.py b/pyomo/solvers/plugins/solvers/SCIPAMPL.py index 9cb69001e8b..0de81311048 100644 --- a/pyomo/solvers/plugins/solvers/SCIPAMPL.py +++ b/pyomo/solvers/plugins/solvers/SCIPAMPL.py @@ -78,7 +78,7 @@ def _default_executable(self): executable = Executable("scipampl") if not executable: logger.warning("Could not locate the 'scipampl' executable or" - " the 'scip' executable since 8.0.0, which is" + " the 'scip' executable since 8.0.0, which is " "required for solver %s" % self.name) self.enable = False return None @@ -144,7 +144,7 @@ def create_command_line(self, executable, problem_files): if executable not in self._known_versions: self._known_versions[executable] = self._get_version(executable) _ver = self._known_versions[executable] - if _ver and _ver > (8, 0, 0): + if _ver and _ver >= (8, 0, 0): problem_file = os.path.splitext(problem_files[0])[0] else: problem_file = problem_files[0] From 10b003829291e4c7e34c429ba7a6c1e805d3d071 Mon Sep 17 00:00:00 2001 From: Sebastian Uerlich Date: Thu, 26 May 2022 15:48:13 +0200 Subject: [PATCH 4/4] Add tests for SCIP versions --- pyomo/solvers/tests/mip/test_scip_version.py | 269 +++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 pyomo/solvers/tests/mip/test_scip_version.py diff --git a/pyomo/solvers/tests/mip/test_scip_version.py b/pyomo/solvers/tests/mip/test_scip_version.py new file mode 100644 index 00000000000..74f27a6337b --- /dev/null +++ b/pyomo/solvers/tests/mip/test_scip_version.py @@ -0,0 +1,269 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import subprocess +from os.path import join, exists, splitext + +import pyomo.common.unittest as unittest + +from pyomo.common.fileutils import this_file_dir, ExecutableData +from pyomo.common.tempfiles import TempfileManager + +import pyomo.environ +from pyomo.opt import SolverFactory +from pyomo.core import ConcreteModel, Var, Objective, Constraint + +import pyomo.solvers.plugins.solvers.SCIPAMPL + +currdir = this_file_dir() +deleteFiles = True + + +class Test(unittest.TestCase): + + def setUp(self): + scip = SolverFactory('scip', solver_io='nl') + type(scip)._known_versions = {} + TempfileManager.push() + + self.patch_run = unittest.mock.patch('pyomo.solvers.plugins.solvers.SCIPAMPL.subprocess.run') + # Executable cannot be partially mocked since it creates a PathData object. + self.patch_path = unittest.mock.patch.object(pyomo.common.fileutils.PathData, 'path', autospec=True) + self.patch_available = unittest.mock.patch.object(pyomo.common.fileutils.PathData, 'available', autospec=True) + + self.run = self.patch_run.start() + self.path = self.patch_path.start() + self.available = self.patch_available.start() + + self.executable_paths = {"scip": join(currdir, "scip"), "scipampl": join(currdir, "scipampl")} + + def tearDown(self): + self.patch_run.stop() + self.patch_path.stop() + self.patch_available.stop() + + TempfileManager.pop(remove=deleteFiles or self.currentTestPassed()) + + def generate_stdout(self, solver, version): + if solver == "scip": + # Template from SCIP 8.0.0 + stdout = "SCIP version {} [precision: 8 byte] [memory: block] [mode: optimized] [LP solver: SoPlex 6.0.0] [GitHash: d9b84b0709]\n"\ + "Copyright (C) 2002-2021 Konrad-Zuse-Zentrum fuer Informationstechnik Berlin (ZIB)\n"\ + "\n"\ + "External libraries:\n" \ + " SoPlex 6.0.0 Linear Programming Solver developed at Zuse Institute Berlin (soplex.zib.de) [GitHash: f5cfa86b]" + + # Template from SCIPAMPL 7.0.3 + elif solver == "scipampl": + stdout = "SCIP version {} [precision: 8 byte] [memory: block] [mode: optimized] [LP solver: SoPlex 5.0.2] [GitHash: 74c11e60cd]\n"\ + "Copyright (C) 2002-2021 Konrad-Zuse-Zentrum fuer Informationstechnik Berlin (ZIB)\n"\ + "\n"\ + "External libraries:\n"\ + " Readline 8.0 GNU library for command line editing (gnu.org/s/readline)" + else: + raise ValueError("Unsupported solver for stdout generation.") + + version = ".".join(str(e) for e in version[:3]) + return stdout.format(version) + + def set_solvers(self, scip=(8, 0, 0, 0), scipampl=(7, 0, 3, 0), fail=True): + + executables = {"scip": scip, "scipampl": scipampl} + + def get_executable(*args, **kwargs): + name = args[0]._registered_name + if name in executables: + if executables[name]: + return self.executable_paths[name] + else: + return None + elif fail: + self.fail("Solver creation looked up a non scip executable.") + else: + return False + + def get_available(*args, **kwargs): + name = args[0]._registered_name + if name in executables: + return executables[name] is not None + elif fail: + self.fail("Solver creation looked up a non scip executable.") + else: + return False + + def run(args, **kwargs): + for solver_name, solver_version in executables.items(): + if not args[0].endswith(solver_name): + continue + if solver_version is None: + raise FileNotFoundError() + else: + return subprocess.CompletedProcess(args, 0, self.generate_stdout(solver_name, solver_version), None) + if fail: + self.fail("Solver creation looked up a non scip executable.") + + self.path.side_effect = get_executable + self.available.side_effect = get_available + self.run.side_effect = run + + def test_scip_available(self): + self.set_solvers() + scip = SolverFactory('scip', solver_io='nl') + scip_executable = scip.executable() + self.assertIs(scip_executable, self.executable_paths["scip"]) + # only one call to path expected, since no check for SCIPAMPL is needed + self.assertEqual(1, self.path.call_count) + self.assertEqual(1, self.run.call_count) + self.available.assert_called() + + # version should now be cached + scip.executable() + self.assertEqual(1, self.run.call_count) + + self.assertTrue(scip.available()) + + def test_scipampl_fallback(self): + self.set_solvers(scip=(7, 0, 3, 0)) + scip = SolverFactory('scip', solver_io='nl') + scip_executable = scip.executable() + self.assertIs(scip_executable, self.executable_paths["scipampl"]) + + # get SCIP and SCIPAMPL paths + self.assertEqual(2, self.path.call_count) + # only check SCIP version + self.assertEqual(1, self.run.call_count) + self.available.assert_called() + + self.assertEqual((7, 0, 3, 0), scip._get_version()) + # also check SCIPAMPL version + self.assertEqual(2, self.run.call_count) + + # versions should now be cached + scip.executable() + self.assertEqual(2, self.run.call_count) + + self.assertTrue(scip.available()) + + def test_no_scip(self): + self.set_solvers(scip=None) + scip = SolverFactory('scip', solver_io='nl') + scip_executable = scip.executable() + self.assertIs(scip_executable, self.executable_paths["scipampl"]) + + # get scipampl path + self.assertEqual(1, self.path.call_count) + # cannot check SCIP version + self.assertEqual(0, self.run.call_count) + self.available.assert_called() + + self.assertEqual((7, 0, 3, 0), scip._get_version()) + # also check SCIPAMPL version + self.assertEqual(1, self.run.call_count) + + # versions should now be cached + scip.executable() + self.assertEqual(1, self.run.call_count) + + self.assertTrue(scip.available()) + + def test_no_fallback(self): + self.set_solvers(scip=None, scipampl=None) + scip = SolverFactory('scip', solver_io='nl') + self.assertFalse(scip.available()) + self.assertIsNone(scip.executable()) + + # cannot check SCIP versions + self.assertEqual(0, self.run.call_count) + self.available.assert_called() + + def test_scip_solver_options(self): + self.set_solvers(fail=False) + scip = SolverFactory('scip', solver_io='nl') + m = self.model = ConcreteModel() + m.v = Var() + m.o = Objective(expr=m.v) + m.c = Constraint(expr=m.v >= 1) + + # cache version and reset mock + scip._get_version() + self.run.reset_mock() + + # SCIP is not actually called + with self.assertRaises(FileNotFoundError) as cm: + scip.solve(m, timelimit=10) + + # SCIP calls should have 3 options + args = self.run.call_args[0][0] + self.assertEqual(3, len(args)) + # check scip call + self.assertEqual(self.executable_paths["scip"], args[0]) + # check for nl file existence + self.assertTrue(exists(args[1] + ".nl")) + # check proper sol filename + self.assertEqual(args[1] + ".sol", cm.exception.filename) + # check -AMPL option + self.assertEqual("-AMPL", args[2]) + # check options file + options_dir = self.run.call_args[1]['cwd'] + self.assertTrue(exists(options_dir + "/scip.set")) + with open(options_dir + "/scip.set", 'r') as options: + self.assertEqual(["limits/time = 10\n"], options.readlines()) + + def test_scipampl_solver_options(self): + self.set_solvers(scip=None, fail=False) + scip = SolverFactory('scip', solver_io='nl') + m = self.model = ConcreteModel() + m.v = Var() + m.o = Objective(expr=m.v) + m.c = Constraint(expr=m.v >= 1) + + # cache version and reset mock + scip._get_version() + self.run.reset_mock() + + # SCIP is not actually called + with self.assertRaises(FileNotFoundError) as cm: + scip.solve(m, timelimit=10, options={'numerics/feastol': 1e-09}) + + # check scip call + args = self.run.call_args[0][0] + self.assertEqual(self.executable_paths["scipampl"], args[0]) + # check for nl file existence + self.assertTrue(exists(args[1])) + (root, ext) = splitext(args[1]) + self.assertEqual(".nl", ext) + # check proper sol filename + self.assertEqual(root + ".sol", cm.exception.filename) + # check -AMPL option + self.assertEqual("-AMPL", args[2]) + # check options file + options_dir = self.run.call_args[1].get('cwd', None) + + if options_dir is not None and exists(options_dir + "/scip.set"): + # SCIPAMPL call should have 3 options + self.assertEqual(3, len(args)) + options_file = options_dir + "/scip.set" + else: + # SCIPAMPL call should have 4 options + self.assertEqual(4, len(args)) + # SCIPAMPL can also receive the options file as the fourth command line argument + options_file = args[3] + self.assertTrue(exists(options_file)) + + with open(options_file, 'r') as options: + lines = options.readlines() + self.assertIn("numerics/feastol = 1e-09\n", lines) + self.assertIn("limits/time = 10\n", lines) + + +if __name__ == "__main__": + deleteFiles = False + unittest.main()