First attempt at comparison comment

.github/workflows/ci.yml

@@ -248,7 +248,44 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: metrics
+          path: ${{ matrix.design.pdk }}-${{ matrix.design.scl }}-${{ matrix.design.name }}.metrics.json
+  manage_metrics:
+    runs-on: ubuntu-22.04
+    needs: [test]
+    if: ${{ always() }}
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install OpenLane
+        run: |
+          python3 -m pip install -e .
+      - name: Get Main Metrics
+        run: |
+          sudo apt install -y jq
+          MAIN_SHA=$(gh api /repos/${{ github.repository }}/branches/main | jq -r .commit.sha )
+          mkdir -p gold
+          curl -L https://github.com/efabless/openlane-metrics/tarball/commit-$MAIN_SHA | tar -xzvC gold
+      - name: Download Metrics
+        uses: actions/download-artifact@v3
+        with:
+          name: current
           path: ${{ matrix.design.name }}-${{ matrix.design.pdk }}-${{ matrix.design.scl }}.metrics.json
+      - uses: actions/github-script@v6
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          script: |
+            const { execSync } = require("child_process");
+            let table = spawnSync("openlane.common.metrics", ["compare-multiple", "gold", "current", "--plain", "--include-tables"], { encoding: "utf8" });
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: table
+            })
+
   publish:
     runs-on: ubuntu-22.04
     needs: [

openlane/common/metrics/__main__.py

@@ -11,11 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import json
 from decimal import Decimal
+from typing import Set, Tuple
 
 import rich
 import cloup
+from rich.markdown import Markdown
 
 from .util import MetricDiff
 from ..misc import Filter
@@ -31,7 +34,7 @@
     "ir__*",
     "power__*",
     "timing__*_vio__*",
-    "*errors*",
+    "*error*",
     "!*__iter:*",
 ]
 
@@ -46,12 +49,11 @@ def cli():
 
 @cloup.command()
 @cloup.option("-f", "--filter", "filters", multiple=True, default=("DEFAULT",))
-@cloup.option("--rich-table/--markdown-table", default=True)
 @cloup.argument("metric_files", nargs=2)
-def compare(metric_files, rich_table, filters):
-    a, b = metric_files
-    a = json.load(open(a, encoding="utf8"), parse_float=Decimal)
-    b = json.load(open(b, encoding="utf8"), parse_float=Decimal)
+def compare(metric_files: Tuple[str, str], filters: Tuple[str, ...]):
+    a_path, b_path = metric_files
+    a = json.load(open(a_path, encoding="utf8"), parse_float=Decimal)
+    b = json.load(open(b_path, encoding="utf8"), parse_float=Decimal)
 
     final_filters = []
     for filter in filters:
@@ -61,14 +63,132 @@ def compare(metric_files, rich_table, filters):
             final_filters.append(filter)
 
     diff = MetricDiff.from_metrics(a, b, Filter(final_filters))
-    if rich_table:
-        rich.print(diff.render_rich(sort_by=("corner", "")))
-    else:
-        rich.print(diff.render_md(sort_by=("corner", "")))
+    md_str = diff.render_md(sort_by=("corner", ""))
+    rich.print(md_str)
 
 
 cli.add_command(compare)
 
 
+@cloup.command(hidden=True)
+@cloup.option("--include-tables/--no-tables", default=False)
+@cloup.option("--render/--plain", default=True)
+@cloup.option("-f", "--filter", "filter_wildcards", multiple=True, default=("DEFAULT",))
+@cloup.argument("metric_folders", nargs=2)
+def compare_multiple(
+    filter_wildcards: Tuple[str, ...],
+    render: bool,
+    include_tables: bool,
+    metric_folders: Tuple[str, str],
+):
+    path_a, path_b = metric_folders
+
+    a: Set[Tuple[str, str, str]] = set()
+    b: Set[Tuple[str, str, str]] = set()
+
+    def add_designs(in_dir: str, to_set: Set[Tuple[str, str, str]]):
+        for file in os.listdir(in_dir):
+            basename = os.path.basename(file)
+            pdk, scl, design = basename.split("-", maxsplit=2)
+            if ".metrics.json" in design:
+                design = design[: -len(".metrics.json")]
+            to_set.add((pdk, scl, design))
+
+    add_designs(path_a, a)
+    add_designs(path_b, b)
+
+    not_in_a = b - a
+    not_in_b = a - b
+    common = a.intersection(b)
+
+    difference_report = ""
+    for tup in not_in_a:
+        pdk, scl, design = tup
+        difference_report += f"* Results for a new test, `{'/'.join(tup)}`, detected.\n"
+    for tup in not_in_b:
+        pdk, scl, design = tup
+        difference_report += (
+            f"* ‼️ Results for `{'/'.join(tup)}` appear to be missing!\n"
+        )
+
+    final_filters = []
+    for wildcard in filter_wildcards:
+        if wildcard == "DEFAULT":
+            final_filters += default_filter_set
+        else:
+            final_filters.append(wildcard)
+
+    filter = Filter(final_filters)
+    critical_change_report = ""
+    tables = ""
+    total_critical = 0
+    if include_tables:
+        tables += "## Per-design breakdown\n\n"
+    for pdk, scl, design in sorted(common):
+        metrics_a = json.load(
+            open(
+                os.path.join(path_a, f"{pdk}-{scl}-{design}.metrics.json"),
+                encoding="utf8",
+            ),
+            parse_float=Decimal,
+        )
+
+        metrics_b = json.load(
+            open(
+                os.path.join(path_b, f"{pdk}-{scl}-{design}.metrics.json"),
+                encoding="utf8",
+            ),
+            parse_float=Decimal,
+        )
+
+        diff = MetricDiff.from_metrics(
+            metrics_a,
+            metrics_b,
+            filter,
+        )
+
+        stats = diff.stats()
+
+        total_critical += stats.critical
+        if stats.critical > 0:
+            critical_change_report += f"  * `{pdk}/{scl}/{design}`: \n"
+            for row in diff.differences:
+                if not row.critical:
+                    continue
+                critical_change_report += (
+                    f"    * `{row.metric_name}` ({row.before} -> {row.after}) \n"
+                )
+
+        if include_tables:
+            tables += (
+                f"<details><summary><code>{pdk}/{scl}/{design}</code></summary>\n\n"
+            )
+            tables += diff.render_md(("corner", ""))
+            tables += "\n</details>\n\n"
+
+    if total_critical == 0:
+        critical_change_report = (
+            "* No critical regressions were detected.\n" + critical_change_report
+        )
+    else:
+        critical_change_report = (
+            "* **A number of critical regressions were detected:**\n"
+            + critical_change_report
+        )
+
+    report = "# CI Report\n\n"
+    report += difference_report
+    report += critical_change_report
+    report += tables
+
+    if render:
+        rich.print(Markdown(report))
+    else:
+        rich.print(report)
+
+
+cli.add_command(compare_multiple)
+
+
 if __name__ == "__main__":
     cli()

openlane/common/metrics/util.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import re
 import textwrap
+from dataclasses import dataclass
 from typing import (
     List,
     Mapping,
@@ -108,6 +109,14 @@ def _key_from_metrics(fields: Iterable[str], metric: str) -> List[str]:
     return result
 
 
+@dataclass
+class MetricStatistics:
+    better: int = 0
+    worse: int = 0
+    critical: int = 0
+    unchanged: int = 0
+
+
 class MetricDiff(object):
     differences: List[MetricComparisonResult]
 
@@ -156,9 +165,10 @@ def render_rich(self, sort_by: Optional[Iterable[str]] = None) -> rich.table.Tab
 
     def render_md(self, sort_by: Optional[Iterable[str]] = None) -> str:
         table = textwrap.dedent(
-            """
-            | Metric | Before | After | Delta |
-            | - | - | - | - |
+            f"""
+            
+            | {'Metric':<70} | {'Before':<10} | {'After':<10} | {'Delta':<20} |
+            | {'-':<70} | {'-':<10} | {'-':<10} | {'-':<20} |
             """
         )
 
@@ -174,14 +184,31 @@ def render_md(self, sort_by: Optional[Iterable[str]] = None) -> str:
             emoji = ""
             if row.better is not None:
                 if row.better:
-                    emoji = "⭕"
+                    emoji = " ⭕"
                 else:
-                    emoji = "❗"
+                    emoji = " ❗"
             if row.critical:
-                emoji = "‼️"
-            table += f"| {row.metric_name} | {before} | {after} | {delta} {emoji} |\n"
+                emoji = " ‼️"
+            table += f"| {row.metric_name:<70} | {before:<10} | {after:<10} | {f'{delta}{emoji}':<20} |\n"
+
+        table += "\n"
         return table
 
+    def stats(self) -> MetricStatistics:
+        stats = MetricStatistics()
+        for row in self.differences:
+            if row.delta == 0 or row.before == row.after:
+                stats.unchanged += 1
+            elif row.critical:
+                stats.critical += 1
+                stats.worse += 1
+            elif row.better is not None:
+                if row.better:
+                    stats.better += 1
+                else:
+                    stats.worse += 1
+        return stats
+
     @classmethod
     def from_metrics(
         Self,

openlane/state/__main__.py

@@ -36,7 +36,6 @@ def cli():
 )
 @cloup.argument("run_dir")
 def latest(extract_metrics_to: Optional[str], run_dir: str):
-    metrics = {}
     exit_code = 0
 
     if latest_state := get_latest_file(run_dir, "state_*.json"):
@@ -47,13 +46,12 @@ def latest(extract_metrics_to: Optional[str], run_dir: str):
             exit(1)
         metrics = state["metrics"]
         print(latest_state, end="")
+        if output := extract_metrics_to:
+            json.dump(metrics, open(output, "w", encoding="utf8"))
     else:
         print("No state_*.json files found", file=sys.stderr)
         exit_code = 1
 
-    if output := extract_metrics_to:
-        json.dump(metrics, open(output, "w", encoding="utf8"))
-
     exit(exit_code)