Merge pull request #52 from venaturum/GH51_pretree_solutions

[#GH51] Parsing heuristic solutions found prior to nodelog (but not by NoRel)

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         - id: trailing-whitespace
 
     - repo: https://github.com/myint/autoflake
-      rev: v1.4
+      rev: v2.3.1
       hooks:
         - id: autoflake
           files: (^(src|tests)/)|(^[^/]*$)

CHANGELOG.md

@@ -2,6 +2,7 @@
 ## Unreleased
 ### Fixed
 ### Changed
+- Parsing of heuristic solutions found prior to nodelog (but not found by NoRel) (#51)
 ### Removed
 
 ## 3.0.0 - 2023-10-11

gurobi-logtools.ipynb

@@ -285,7 +285,13 @@
     }
    },
    "source": [
-    "This simply creates two more dataframes, `timelines[\"nodelog\"]` and `timelines[\"rootlp\"]`, which contain the information about the LP relaxation and node log. Let's have a look at them:"
+    "This simply creates four more dataframes:\n",
+    "- `timelines[\"nodelog\"]` : information related to the node log, i.e. branch and bound tree\n",
+    "- `timelines[\"norel\"]` : information related to solutions produced by the No Relaxation heuristic (\"NoRel\")\n",
+    "- `timelines[\"pretreesols\"]` : information related to heuristic solutions found prior to the branch and bound tree (but not including those found by NoRel)\n",
+    "- `timelines[\"rootlp\"]` : information related to the root linear relaxation solve\n",
+    "\n",
+    "Let's have a look at them:"
    ]
   },
   {
@@ -302,6 +308,27 @@
     "results.progress(\"nodelog\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ae623641",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# empty dataframe, since the No Relaxation heuristic was not used\n",
+    "results.progress(\"norel\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab3c3af4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results.progress(\"pretreesols\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

src/gurobi_logtools/api.py

@@ -53,6 +53,8 @@ def progress(self, section="nodelog") -> dict:
                 log = parser.continuous_parser.get_progress()
             elif section == "norel":
                 log = parser.norel_parser.get_progress()
+            elif section == "pretreesols":
+                log = parser.pretree_solution_parser.get_progress()
             else:
                 raise ValueError(f"Unknown section '{section}'")
 
@@ -194,4 +196,5 @@ def get_dataframe(logfiles: List[str], timelines=False, prettyparams=False):
         norel=result.progress("norel"),
         rootlp=result.progress("rootlp"),
         nodelog=result.progress("nodelog"),
+        pretreesols=result.progress("pretreesols"),
     )

src/gurobi_logtools/parsers/continuous.py

@@ -1,6 +1,7 @@
 import re
 
 from gurobi_logtools.parsers.barrier import BarrierParser
+from gurobi_logtools.parsers.pretree_solutions import PretreeSolutionParser
 from gurobi_logtools.parsers.simplex import SimplexParser
 from gurobi_logtools.parsers.util import typeconvert_groupdict
 
@@ -24,7 +25,7 @@ class ContinuousParser:
         re.compile(r"(?P<OPTIMAL>Optimal objective\s+(?P<ObjVal>.*))$"),
     ]
 
-    def __init__(self):
+    def __init__(self, pre_tree_solution_parser: PretreeSolutionParser):
         """Initialize the Continuous parser."""
         self._barrier_parser = BarrierParser()
         self._simplex_parser = SimplexParser()
@@ -33,6 +34,8 @@ def __init__(self):
 
         self._current_pattern = None
 
+        self._pre_tree_solution_parser = pre_tree_solution_parser
+
     def parse(self, line: str) -> bool:
         """Parse the given log line to populate summary and progress data.
 
@@ -44,6 +47,10 @@ def parse(self, line: str) -> bool:
         Returns:
             bool: Return True if the given line is matched by some pattern.
         """
+
+        if self._pre_tree_solution_parser.parse(line):
+            return True
+
         mip_relaxation_match = ContinuousParser.mip_relaxation_pattern.match(line)
         if mip_relaxation_match:
             self._current_pattern = "relaxation"

src/gurobi_logtools/parsers/presolve.py

@@ -1,5 +1,6 @@
 import re
 
+from gurobi_logtools.parsers.pretree_solutions import PretreeSolutionParser
 from gurobi_logtools.parsers.util import typeconvert_groupdict
 
 
@@ -66,7 +67,7 @@ class PresolveParser:
     # Special case: model solved by presolve
     presolve_all_removed = re.compile(r"Presolve: All rows and columns removed")
 
-    def __init__(self):
+    def __init__(self, pre_tree_solution_parser: PretreeSolutionParser):
         """Initialize the Presolve parser.
 
         The PresolveParser extends beyond the lines associated with the presolved
@@ -75,6 +76,7 @@ def __init__(self):
         """
         self._summary = {}
         self._started = False
+        self._pre_tree_solution_parser = pre_tree_solution_parser
 
     def parse(self, line: str) -> bool:
         """Parse the given log line to populate summary data.
@@ -94,6 +96,9 @@ def parse(self, line: str) -> bool:
                 return True
             return False
 
+        if self._pre_tree_solution_parser.parse(line):
+            return True
+
         for pattern in PresolveParser.presolve_intermediate_patterns:
             match = pattern.match(line)
             if match:

src/gurobi_logtools/parsers/pretree_solutions.py

@@ -0,0 +1,43 @@
+import re
+
+from gurobi_logtools.parsers.util import typeconvert_groupdict
+
+
+class PretreeSolutionParser:
+    pretree_solution_regex = re.compile(
+        r"Found heuristic solution:\sobjective\s(?P<Incumbent>[^\s]+)"
+    )
+
+    def __init__(self):
+        """Initialize the pre-tree solutions parser (does not include NoRel solutions).
+
+        The PresolveParser extends beyond the lines associated with the presolved
+        model. Specifically, it includes information for all lines appearing between
+        the HeaderParser and the NoRelParser or the RelaxationParser.
+        """
+        self._progress = []
+        self._summary = {}
+        # self._started = False
+
+    def parse(self, line: str) -> bool:
+        """Parse the given log line to populate summary data.
+
+        Args:
+            line (str): A line in the log file.
+
+        Returns:
+            bool: Return True if the given line is matched by some pattern.
+        """
+        match = self.pretree_solution_regex.match(line)
+        if match:
+            self._progress.append(typeconvert_groupdict(match))
+            return True
+        return False
+
+    def get_summary(self) -> dict:
+        """Return the current parsed summary."""
+        return {"PreTreeSolutions": len(self._progress)}
+
+    def get_progress(self) -> list:
+        """Return the progress of the search tree."""
+        return self._progress

src/gurobi_logtools/parsers/single_log.py

@@ -5,6 +5,7 @@
 from gurobi_logtools.parsers.nodelog import NodeLogParser
 from gurobi_logtools.parsers.norel import NoRelParser
 from gurobi_logtools.parsers.presolve import PresolveParser
+from gurobi_logtools.parsers.pretree_solutions import PretreeSolutionParser
 from gurobi_logtools.parsers.termination import TerminationParser
 from gurobi_logtools.parsers.util import model_type
 
@@ -16,11 +17,13 @@ class SingleLogParser:
     """
 
     def __init__(self, write_to_dir=None):
+        self.pretree_solution_parser = self.make_pretree_solution_parser()
+
         # Parsers in sequence
         self.header_parser = HeaderParser()
-        self.presolve_parser = PresolveParser()
+        self.presolve_parser = PresolveParser(self.pretree_solution_parser)
         self.norel_parser = NoRelParser()
-        self.continuous_parser = ContinuousParser()
+        self.continuous_parser = ContinuousParser(self.pretree_solution_parser)
         self.nodelog_parser = NodeLogParser()
         self.termination_parser = TerminationParser()
 
@@ -39,6 +42,9 @@ def __init__(self, write_to_dir=None):
         self.write_to_dir = pathlib.Path(write_to_dir) if write_to_dir else None
         self.lines = [] if self.write_to_dir else None
 
+    def make_pretree_solution_parser(self):
+        return PretreeSolutionParser()
+
     def close(self):
         if self.write_to_dir:
             paramstr = "-".join(
@@ -66,6 +72,7 @@ def get_summary(self):
         summary.update(self.presolve_parser.get_summary())
         summary.update(self.norel_parser.get_summary())
         summary.update(self.continuous_parser.get_summary())
+        summary.update(self.pretree_solution_parser.get_summary())
         summary.update(self.nodelog_parser.get_summary())
         summary.update(self.termination_parser.get_summary())
         summary["ModelType"] = model_type(

tests/assets/multiknapsack.log

@@ -0,0 +1,71 @@
+
+Gurobi 9.5.0 (mac64[x86], gurobi_cl) logging started Fri Jul  5 19:18:47 2024
+
+Set parameter LogFile to value "multiknapsack.log"
+Using license file /Users/riley.clement/gurobi.lic
+
+Gurobi Optimizer version 9.5.0 build v11.0.2rc0 (mac64[x86] - Darwin 23.5.0 23F79)
+Copyright (c) 2021, Gurobi Optimization, LLC
+
+Read MPS format model from file multiknapsack.mps.bz2
+Reading time = 2.31 seconds
+: 1000 rows, 1000 columns, 1000000 nonzeros
+
+CPU model: Intel(R) Core(TM) i5-1038NG7 CPU @ 2.00GHz
+Thread count: 4 physical cores, 8 logical processors, using up to 8 threads
+
+Optimize a model with 1000 rows, 1000 columns and 1000000 nonzeros
+Model fingerprint: 0x1dcb478a
+Variable types: 0 continuous, 1000 integer (0 binary)
+Coefficient statistics:
+  Matrix range     [8e-07, 1e+00]
+  Objective range  [3e-04, 1e+00]
+  Bounds range     [0e+00, 0e+00]
+  RHS range        [2e+00, 1e+04]
+Found heuristic solution: objective 2.6508070
+Presolve removed 751 rows and 0 columns
+Presolve time: 0.63s
+Presolved: 249 rows, 1000 columns, 249000 nonzeros
+Variable types: 0 continuous, 1000 integer (0 binary)
+
+Starting NoRel heuristic
+Found heuristic solution: objective 15.6561779
+Found heuristic solution: objective 20.2661725
+Found heuristic solution: objective 21.6309598
+Found heuristic solution: objective 23.7055101
+Found heuristic solution: objective 24.1081356
+Elapsed time for NoRel heuristic: 8s (best bound 24.6734)
+Elapsed time for NoRel heuristic: 15s (best bound 24.6734)
+Elapsed time for NoRel heuristic: 21s (best bound 24.6734)
+Elapsed time for NoRel heuristic: 29s (best bound 24.6734)
+
+Root relaxation: objective 2.467344e+01, 32 iterations, 0.02 seconds (0.01 work units)
+
+Found heuristic solution: objective 15.5735142
+
+    Nodes    |    Current Node    |     Objective Bounds      |     Work
+ Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time
+
+     0     0   24.67344    0    5   15.57351   24.67344  58.4%     -    0s
+H    0     0                      23.8510977   24.67344  3.45%     -    0s
+H    0     0                      24.1081356   24.67344  2.34%     -    0s
+     0     0   24.54405    0    7   24.10814   24.54405  1.81%     -    1s
+     0     0   24.54405    0    5   24.10814   24.54405  1.81%     -    1s
+     0     0   24.54405    0    7   24.10814   24.54405  1.81%     -    1s
+     0     0   24.49025    0    6   24.10814   24.49025  1.59%     -    1s
+     0     0   24.48961    0    7   24.10814   24.48961  1.58%     -    1s
+     0     0   24.45227    0    8   24.10814   24.45227  1.43%     -    1s
+     0     0   24.40551    0    8   24.10814   24.40551  1.23%     -    1s
+     0     2   24.40551    0    8   24.10814   24.40551  1.23%     -    1s
+
+Cutting planes:
+  Gomory: 8
+  Lift-and-project: 1
+
+Explored 838 nodes (2566 simplex iterations) in 1.27 seconds (0.53 work units)
+Thread count was 8 (of 8 available processors)
+
+Solution count 5: 24.1081 23.8511 15.5735 ... 2.65081
+
+Optimal solution found (tolerance 1.00e-04)
+Best objective 2.410813557868e+01, best bound 2.410813557868e+01, gap 0.0000%

tests/parsers/test_continuous.py

@@ -1,6 +1,7 @@
 from unittest import TestCase, main
 
 from gurobi_logtools.parsers.continuous import ContinuousParser
+from gurobi_logtools.parsers.pretree_solutions import PretreeSolutionParser
 from gurobi_logtools.parsers.util import parse_block
 
 example_log_barrier_with_simplex = """
@@ -128,7 +129,7 @@
 
 class TestContinuous(TestCase):
     def test_last_progress_entry_barrier_with_simplex(self):
-        continuous_parser = ContinuousParser()
+        continuous_parser = ContinuousParser(PretreeSolutionParser())
         parse_block(continuous_parser, example_log_barrier_with_simplex)
         self.assertEqual(
             continuous_parser.get_progress()[-1], expected_progress_last_entry
@@ -153,7 +154,7 @@ def test_get_summary_progress(self):
             ],
         ):
             with self.subTest(example_log=example_log):
-                continuous_parser = ContinuousParser()
+                continuous_parser = ContinuousParser(PretreeSolutionParser())
                 parse_block(continuous_parser, example_log)
                 self.assertEqual(continuous_parser.get_summary(), expected_summary)
                 self.assertEqual(continuous_parser.get_progress(), expected_progress)

tests/parsers/test_presolve.py

@@ -1,6 +1,7 @@
 from unittest import TestCase, main
 
 from gurobi_logtools.parsers.presolve import PresolveParser
+from gurobi_logtools.parsers.pretree_solutions import PretreeSolutionParser
 from gurobi_logtools.parsers.util import parse_lines
 
 example_log_0 = """
@@ -139,7 +140,7 @@ def test_first_line_matched(self):
 
         for i, example_log in enumerate([example_log_0, example_log_1, example_log_2]):
             with self.subTest(example_log=example_log):
-                presolve_parser = PresolveParser()
+                presolve_parser = PresolveParser(PretreeSolutionParser())
                 for line in example_log.strip().split("\n"):
                     if presolve_parser.parse(line):
                         self.assertEqual(line, expected_start_lines[i])
@@ -155,7 +156,7 @@ def test_get_summary(self):
         ]
         for i, example_log in enumerate([example_log_0, example_log_1, example_log_2]):
             with self.subTest(example_log=example_log):
-                presolve_parser = PresolveParser()
+                presolve_parser = PresolveParser(PretreeSolutionParser())
                 lines = example_log.strip().split("\n")
                 parse_lines(presolve_parser, lines)
                 self.assertEqual(presolve_parser.get_summary(), expected_summaries[i])