[leaderboard] Move leaderboard utility into compiler_gym namespace.

This adds a compiler_gym.leaderboard module that contains the LLVM
codesize leaderboard helper code. New API docs provide improved
explanation of how to use it.

Issue #158.

BUILD.bazel

@@ -22,6 +22,7 @@ py_library(
         "//compiler_gym/bin",
         "//compiler_gym/datasets",
         "//compiler_gym/envs",
+        "//compiler_gym/leaderboard",
         "//compiler_gym/service",
         "//compiler_gym/spaces",
         "//compiler_gym/views",

compiler_gym/BUILD

@@ -22,7 +22,7 @@ py_library(
 py_library(
     name = "compiler_env_state",
     srcs = ["compiler_env_state.py"],
-    visibility = ["//compiler_gym/envs:__subpackages__"],
+    visibility = ["//compiler_gym:__subpackages__"],
 )
 
 py_library(

compiler_gym/leaderboard/BUILD

@@ -0,0 +1,26 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "leaderboard",
+    srcs = ["__init__.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":llvm_codesize",
+    ],
+)
+
+py_library(
+    name = "llvm_codesize",
+    srcs = ["llvm_codesize.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//compiler_gym:compiler_env_state",
+        "//compiler_gym/bin:validate",
+        "//compiler_gym/envs",
+        "//compiler_gym/util",
+    ],
+)

compiler_gym/leaderboard/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+"""This module contains helper submodules for creating CompilerGym leaderboard submissions."""

leaderboard/llvm_codesize/eval_policy.py → compiler_gym/leaderboard/llvm_codesize.py

@@ -2,20 +2,27 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
-"""This module defines a helper function for evaluating LLVM codesize reduction
-policies.
-
-Usage:
-
-    from compiler_gym.envs import LlvmEnv
-    from eval_policy import eval_policy
-
-    class MyLlvmCodesizePolicy:
-        def __call__(env: LlvmEnv) -> None:
-            pass # ...
-
-    if __name__ == "__main__":
-        eval_policy(MyLlvmCodesizePolicy())
+"""LLVM is a popular open source compiler used widely in industry and research.
+This environment exposes the optimization pipeline as a set of actions that can
+be applied to a particular program. The goal of the agent is to select the
+sequence of optimizations that lead to the greatest reduction in instruction
+count in the program being compiled. Reward is the reduction in codesize
+achieved scaled to the reduction achieved by LLVM's builtin -Oz pipeline. Users
+who wish to create a submission for this leaderboard should consider using the
+:func:`eval_llvm_codesize_policy()
+<compiler_gym.leaderboard.llvm_codesize.eval_llvm_codesize_policy>` helper.
+
++--------------------+------------------------------------------------------+
+| Property           | Value                                                |
++====================+======================================================+
+| Environment        | :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>`.        |
++--------------------+------------------------------------------------------+
+| Observation Space  | Any.                                                 |
++--------------------+------------------------------------------------------+
+| Reward Space       | Instruction count reduction relative to :code:`-Oz`. |
++--------------------+------------------------------------------------------+
+| Test Dataset       | The 23 cBench benchmarks.                            |
++--------------------+------------------------------------------------------+
 """
 import platform
 import sys
@@ -33,9 +40,9 @@ def __call__(env: LlvmEnv) -> None:
 from cpuinfo import get_cpu_info
 from tqdm import tqdm
 
-import compiler_gym  # noqa Register environments.
-from compiler_gym import CompilerEnvState
+import compiler_gym.envs  # noqa Register environments.
 from compiler_gym.bin.validate import main as validate
+from compiler_gym.compiler_env_state import CompilerEnvState
 from compiler_gym.envs import LlvmEnv
 from compiler_gym.util.tabulate import tabulate
 from compiler_gym.util.timer import Timer
@@ -163,11 +170,71 @@ def run(self):
                 self.n += 1
 
 
-def eval_policy(policy: Policy) -> None:
-    """Evaluate a policy on a target dataset.
+def eval_llvm_codesize_policy(policy: Policy) -> None:
+    """Evaluate an LLVM codesize policy and generate results for a leaderboard
+    submission.
+
+    To use it, you define your policy as a function that takes an
+    :class:`LlvmEnv <compiler_gym.envs.LlvmEnv>` instance as input and modifies
+    it in place. For example, for a trivial random policy:
+
+        >>> from compiler_gym.envs import LlvmEnv
+        >>> def my_policy(env: LlvmEnv) -> None:
+        ....   # Defines a policy that takes 10 random steps.
+        ...    for _ in range(10):
+        ...        _, _, done, _ = env.step(env.action_space.sample())
+        ...        if done: break
+
+    If you would like a stateful policy, you could use a class and override the
+    :code:`__call__()` method:
+
+        >>> class MyPolicy:
+        ...     def __call__(env: LlvmEnv) -> None:
+        ...         pass # ... do fun stuff!
+        >>> my_policy = MyPolicy()
+
+    You then call the :func:`eval_llvm_codesize_policy()
+    <compiler_gym.leaderboard.llvm_codesize.eval_llvm_codesize_policy>` helper
+    function, passing it your policy as its only argument:
+
+    >>> eval_llvm_codesize_policy(my_policy)
+
+    Put together as a complete example this is what an example leaderboard
+    submission script looks like:
+
+    .. code-block:: python
+
+        # my_policy.py
+        from compiler_gym.leaderboard.llvm_codesize import eval_llvm_codesize_policy
+        from compiler_gym.envs import LlvmEnv
+
+        def my_policy(env: LlvmEnv) -> None:
+            pass # ... do fun stuff!
+
+        if __name__ == "__main__":
+            eval_llvm_codesize_policy(my_policy)
+
+    The :func:`eval_llvm_codesize_policy()
+    <compiler_gym.leaderboard.llvm_codesize.eval_llvm_codesize_policy>` helper
+    defines a number of commandline flags that can be overriden to control the
+    behavior of the evaluation. For example the flag :code:`--n` determines the
+    number of times the policy is run on each benchmark (default is 10), and
+    :code:`--logfile` determines the path of the generated results file:
+
+    .. code-block::
+
+        $ python my_policy.py --n=5 --logfile=my_policy_results.csv
+
+    You can use :code:`--helpfull` flag to list all of the flags that are
+    defined:
+
+    .. code-block::
+
+        $ python my_policy.py --helpfull
 
-    A policy is a function that takes as input an LlvmEnv environment and
-    performs a set of actions on it.
+    Once you are happy with your approach, see the `contributing guide
+    <https://github.com/facebookresearch/CompilerGym/blob/development/CONTRIBUTING.md#leaderboard-submissions>`_
+    for instructions on preparing a submission to the leaderboard.
     """
 
     def main(argv):

docs/source/compiler_gym/leaderboard.rst

@@ -0,0 +1,16 @@
+compiler_gym.leaderboard
+========================
+
+We provide `leaderboards
+<https://github.com/facebookresearch/CompilerGym#leaderboards>`_ to track the
+performance of user-submitted algorithms on CompilerGym tasks. The goal of the
+leaderboards is to provide a venue for researchers to promote their work, and to
+provide a common benchmark for evaluating approaches. The
+:code:`compiler_gym.leaderboard` module contains helper submodules that can be
+used for preparing leaderboard submissions.
+
+LLVM Codesize
+-------------
+
+.. automodule:: compiler_gym.leaderboard.llvm_codesize
+    :members:

docs/source/index.rst

@@ -31,6 +31,7 @@ for applying reinforcement learning to compiler optimizations.
    compiler_gym/datasets
    compiler_gym/envs
    llvm/api
+   compiler_gym/leaderboard
    compiler_gym/service
    compiler_gym/spaces
    compiler_gym/views

leaderboard/llvm_codesize/e_greedy/BUILD

@@ -8,7 +8,7 @@ py_library(
     name = "e_greedy",
     srcs = ["e_greedy.py"],
     deps = [
-        "//leaderboard/llvm_codesize:eval_policy",
+        "//compiler_gym/leaderboard:llvm_codesize",
     ],
 )
 
@@ -18,6 +18,7 @@ py_test(
     srcs = ["e_greedy_test.py"],
     deps = [
         ":e_greedy",
+        "//compiler_gym/leaderboard:llvm_codesize",
         "//tests:test_main",
         "//tests/pytest_plugins:llvm",
     ],

leaderboard/llvm_codesize/e_greedy/e_greedy.py

@@ -4,19 +4,14 @@
 # LICENSE file in the root directory of this source tree.
 """ϵ-greedy policy for LLVM codesize."""
 import logging
-import os
 import random
-import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import NamedTuple
 
 from absl import flags
 
 from compiler_gym.envs import CompilerEnv, LlvmEnv
-
-# Import the ../eval_policy.py helper.
-sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/..")
-from eval_policy import eval_policy  # noqa pylint: disable=wrong-import-position
+from compiler_gym.leaderboard.llvm_codesize import eval_llvm_codesize_policy
 
 flags.DEFINE_float(
     "epsilon", 0, "The ratio of patience to the size of the action space. "
@@ -113,4 +108,4 @@ def e_greedy_search(env: LlvmEnv) -> None:
 
 
 if __name__ == "__main__":
-    eval_policy(e_greedy_search)
+    eval_llvm_codesize_policy(e_greedy_search)

leaderboard/llvm_codesize/e_greedy/e_greedy_test.py

@@ -2,16 +2,16 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
-"""Tests for //leaderboard/llvm_codesize:eval_policy."""
+"""Tests for //leaderboard/llvm_codesize/e_greedy."""
 from concurrent.futures import ThreadPoolExecutor
 
 import pytest
 from absl import flags
 
 from compiler_gym.envs import LlvmEnv
+from compiler_gym.leaderboard.llvm_codesize import eval_llvm_codesize_policy
 from leaderboard.llvm_codesize.e_greedy.e_greedy import (
     e_greedy_search,
-    eval_policy,
     select_best_action,
 )
 from tests.test_main import main as _test_main
@@ -33,7 +33,7 @@ def test_random_search():
         ]
     )
     with pytest.raises(SystemExit):
-        eval_policy(e_greedy_search)
+        eval_llvm_codesize_policy(e_greedy_search)
 
 
 def test_select_best_action_closed_environment(env: LlvmEnv):

setup.py

@@ -48,23 +48,24 @@ def get_tag(self):
     url="https://github.com/facebookresearch/CompilerGym",
     license="MIT",
     packages=[
-        "compiler_gym",
         "compiler_gym.bin",
         "compiler_gym.datasets",
-        "compiler_gym.envs",
-        "compiler_gym.envs.llvm",
-        "compiler_gym.envs.llvm.service",
         "compiler_gym.envs.llvm.service.passes",
-        "compiler_gym.service",
+        "compiler_gym.envs.llvm.service",
+        "compiler_gym.envs.llvm",
+        "compiler_gym.envs",
+        "compiler_gym.leaderboard",
         "compiler_gym.service.proto",
+        "compiler_gym.service",
         "compiler_gym.spaces",
-        "compiler_gym.third_party",
         "compiler_gym.third_party.autophase",
-        "compiler_gym.third_party.llvm",
         "compiler_gym.third_party.inst2vec",
-        "compiler_gym.util",
+        "compiler_gym.third_party.llvm",
+        "compiler_gym.third_party",
         "compiler_gym.util.flags",
+        "compiler_gym.util",
         "compiler_gym.views",
+        "compiler_gym",
     ],
     package_dir={
         "": "bazel-bin/package.runfiles/CompilerGym",

tests/leaderboard/BUILD

@@ -0,0 +1,15 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "llvm_codesize_test",
+    srcs = ["llvm_codesize_test.py"],
+    deps = [
+        "//compiler_gym/leaderboard:llvm_codesize",
+        "//tests:test_main",
+        "//tests/pytest_plugins:common",
+    ],
+)