diff --git a/.github/scripts/get_latest_metrics.py b/.github/scripts/get_latest_metrics.py deleted file mode 100644 index 8da83c5d..00000000 --- a/.github/scripts/get_latest_metrics.py +++ /dev/null @@ -1,32 +0,0 @@ -from math import inf -import os -import sys -import glob -import json -import argparse - -cmd = argparse.ArgumentParser() -cmd.add_argument("-o", "--output", required=True) -cmd.add_argument("run_dir") -args = cmd.parse_args() - -state_out_jsons = sorted(glob.glob(os.path.join(args.run_dir, "*", "state_out.json"))) -latest_time = -inf -latest_json = None -for state_out_json in state_out_jsons: - time = os.path.getmtime(state_out_json) - if time > latest_time: - latest_time = time - latest_json = state_out_json - -if latest_json is None: - print("Flow did not produce any output states", file=sys.stderr) - json.dump({}, open(args.output, "w", encoding="utf8")) - exit(1) - -print(f"Using '{latest_json}'…") -state_out = json.load(open(latest_json, encoding="utf8")) - -metrics = state_out["metrics"] - -json.dump(metrics, open(args.output, "w", encoding="utf8")) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6112cffa..79305257 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,10 +225,10 @@ jobs: run: | nix-shell --run "\ python3 -m openlane\ - --run-tag ${{ matrix.design.pdk }}-${{ matrix.design.scl }}\ - --pdk ${{ matrix.design.pdk }}\ - --scl ${{ matrix.design.scl }}\ - ${{ matrix.design.config }}\ + --run-tag ${{ matrix.design.pdk }}-${{ matrix.design.scl }}\ + --pdk ${{ matrix.design.pdk }}\ + --scl ${{ matrix.design.scl }}\ + ${{ matrix.design.config }}\ " - name: Upload Run Folder if: ${{ always() }} @@ -239,7 +239,11 @@ jobs: - name: Fetch Metrics if: ${{ always() }} run: | - python3 ./.github/scripts/get_latest_metrics.py ${{ matrix.design.run_folder }} -o ${{ matrix.design.name }}-${{ matrix.design.pdk }}-${{ matrix.design.scl }}.metrics.json + nix-shell --run "\ + python3 -m openlane.state latest\ + ${{ matrix.design.run_folder }}\ + --extract-metrics-to ${{ matrix.design.pdk }}-${{ matrix.design.scl }}-${{ matrix.design.name }}.metrics.json\ + " - name: Upload Metrics uses: actions/upload-artifact@v3 with: diff --git a/openlane/__main__.py b/openlane/__main__.py index 2ce06955..6e6c463b 100644 --- a/openlane/__main__.py +++ b/openlane/__main__.py @@ -68,7 +68,6 @@ def run( _force_run_dir: Optional[str], _force_design_dir: Optional[str], ) -> int: - config_file = config_files[0] # Enforce Mutual Exclusion diff --git a/openlane/common/__init__.py b/openlane/common/__init__.py index fdc5c5bf..933fae9a 100644 --- a/openlane/common/__init__.py +++ b/openlane/common/__init__.py @@ -43,6 +43,7 @@ format_size, format_elapsed_time, Filter, + get_latest_file, ) from .types import ( is_number, @@ -51,7 +52,7 @@ ) from .toolbox import Toolbox from .drc import DRC, Violation - +from . import cli ## TPE diff --git a/openlane/common/metrics/__main__.py b/openlane/common/metrics/__main__.py index 84735d1c..c6e13198 100644 --- a/openlane/common/metrics/__main__.py +++ b/openlane/common/metrics/__main__.py @@ -19,6 +19,7 @@ from .util import MetricDiff from ..misc import Filter +from ..cli import formatter_settings default_filter_set = [ @@ -35,6 +36,14 @@ ] +@cloup.group( + no_args_is_help=True, + formatter_settings=formatter_settings, +) +def cli(): + pass + + @cloup.command() @cloup.option("-f", "--filter", "filters", multiple=True, default=("DEFAULT",)) @cloup.option("--rich-table/--markdown-table", default=True) @@ -58,11 +67,6 @@ def compare(metric_files, rich_table, filters): rich.print(diff.render_md(sort_by=("corner", ""))) -@cloup.group() -def cli(): - pass - - cli.add_command(compare) diff --git a/openlane/common/misc.py b/openlane/common/misc.py index 200be297..f3d01463 100644 --- a/openlane/common/misc.py +++ b/openlane/common/misc.py @@ -13,22 +13,27 @@ # limitations under the License. import os import re +import glob import typing import fnmatch import pathlib import unicodedata +from math import inf from enum import Enum from typing import ( Any, Generator, Iterable, + Optional, Sequence, SupportsFloat, TypeVar, + Union, ) from deprecated.sphinx import deprecated +from .types import Path T = TypeVar("T") @@ -280,3 +285,21 @@ def filter( break if allowed: yield input + + +def get_latest_file(in_path: Union[str, os.PathLike], filename: str) -> Optional[Path]: + """ + :param in_path: A directory to search in + :param filename: The final filename + :returns: The latest file matching the parameters, by modification time + """ + glob_results = glob.glob(os.path.join(in_path, "**", filename), recursive=True) + latest_time = -inf + latest_json = None + for result in glob_results: + time = os.path.getmtime(result) + if time > latest_time: + latest_time = time + latest_json = Path(result) + + return latest_json diff --git a/openlane/config/flow.py b/openlane/config/flow.py index a9419395..98f11368 100644 --- a/openlane/config/flow.py +++ b/openlane/config/flow.py @@ -25,7 +25,7 @@ Variable( "STD_CELL_LIBRARY", str, - "Specifies the default standard cell library to be used under the specified PDK.", + "Specifies the default standard cell library to be used under the specified PDK. Must be a valid C identifier.", pdk=True, ), Variable( @@ -418,12 +418,12 @@ Variable( "DESIGN_NAME", str, - "The name of the top level module of the design. This is the only variable that MUST be set in every single OpenLane configuration file or dictionary.", + "The name of the top level module of the design. Must be a valid C identifier.", ), Variable( "PDK", str, - "Specifies the process design kit (PDK).", + "Specifies the process design kit (PDK). Must be a valid C identifier.", default="sky130A", ), Variable( diff --git a/openlane/flows/flow.py b/openlane/flows/flow.py index 88e3a38a..5274566f 100644 --- a/openlane/flows/flow.py +++ b/openlane/flows/flow.py @@ -58,7 +58,15 @@ register_additional_handler, deregister_additional_handler, ) -from ..common import get_tpe, mkdirp, protected, final, slugify, Toolbox +from ..common import ( + get_tpe, + mkdirp, + protected, + final, + slugify, + Toolbox, + get_latest_file, +) class FlowError(RuntimeError): @@ -503,18 +511,7 @@ def start( # Extract Maximum State if with_initial_state is None: - latest_time = 0 - latest_json: Optional[str] = None - state_out_jsons = sorted( - glob.glob(os.path.join(self.run_dir, "*", "state_out.json")) - ) - for state_out_json in state_out_jsons: - time = os.path.getmtime(state_out_json) - if time > latest_time: - latest_time = time - latest_json = state_out_json - - if latest_json is not None: + if latest_json := get_latest_file(self.run_dir, "state_out.json"): verbose(f"Using state at '{latest_json}'.") initial_state = State.loads( diff --git a/openlane/state/__main__.py b/openlane/state/__main__.py new file mode 100644 index 00000000..74463357 --- /dev/null +++ b/openlane/state/__main__.py @@ -0,0 +1,63 @@ +# Copyright 2023 Efabless Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import json +from typing import Optional + +import cloup + +from ..common import get_latest_file +from ..common.cli import formatter_settings + + +@cloup.group( + no_args_is_help=True, + formatter_settings=formatter_settings, +) +def cli(): + pass + + +@cloup.command() +@cloup.option( + "--extract-metrics-to", + default=None, +) +@cloup.argument("run_dir") +def latest(extract_metrics_to: Optional[str], run_dir: str): + metrics = {} + exit_code = 0 + + if latest_state := get_latest_file(run_dir, "state_*.json"): + try: + state = json.load(open(latest_state, encoding="utf8")) + except json.JSONDecodeError as e: + print(f"Latest state at {latest_state} is invalid: {e}", file=sys.stderr) + exit(1) + metrics = state["metrics"] + print(latest_state, end="") + else: + print("No state_*.json files found", file=sys.stderr) + exit_code = 1 + + if output := extract_metrics_to: + json.dump(metrics, open(output, "w", encoding="utf8")) + + exit(exit_code) + + +cli.add_command(latest) + +if __name__ == "__main__": + cli()