First step towards using bzlmod

* Move projects that are available already in https://registry.bazel.build/ from from load_external to MODULE.bazel. * This is not complete: there are probably more steps to simplify and clean-up after the first submit. * Switching to bzlmod broke making the compilation-db as the action hooks don't seem to work anymore ( xls/dev_tools//make-compilation-db.sh ). The used com_grail_bazel_compdb is not maintained anymore, so this needs to be updated to something more modern (hedronvision?). Since this is not actively needed in daily development (and I am the only one really using it), postponed for later. Issues: google#931 Also, needed to make Python rules to work well with --incompatible_default_to_explicit_init_py (bazelbuild/bazel#10076)
hzeller · Aug 28, 2024 · a934a25 · a934a25
1 parent 02f5fd9
commit a934a25
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 212 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -1,10 +1,14 @@
-# We use bazel >= 6, but no bzlmod yet.
-common --noenable_bzlmod
+common --enable_bzlmod
 
 # Disable rules_python Starlark rules for Bazel 7+.
 # See https://github.com/bazelbuild/rules_python/issues/1069#issuecomment-1942053014.
 build --action_env=RULES_PYTHON_ENABLE_PYSTAR=0
 
+# Disable automatic generation of __init__.py files. This allows
+# namespace packages (such as `google`) to work correctly.
+# https://github.com/bazelbuild/bazel/issues/10076
+build --incompatible_default_to_explicit_init_py=true
+
 # Minimium c++ standard used.
 build --cxxopt "-std=c++20"            --host_cxxopt "-std=c++20"
 build --action_env=BAZEL_CXXOPTS=-std=c++20

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ xls_clang-tidy.out
 user.bazelrc
 .vscode
 .cache
+MODULE.bazel.lock
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -0,0 +1,40 @@
+module(
+    name = "xls",
+    repo_name = "com_google_xls",
+)
+
+# Compiler toolchain
+bazel_dep(name = "toolchains_llvm", version = "1.1.2")
+
+# Configure and register the toolchain.
+llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
+llvm.toolchain(
+   llvm_version = "17.0.6",
+)
+
+use_repo(llvm, "llvm_toolchain")
+register_toolchains("@llvm_toolchain//:all")
+
+# Adding grpc first, it messes with python dependencies, so get this done first.
+bazel_dep(name = "grpc", version = "1.66.0", repo_name="com_github_grpc_grpc")
+
+#
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "rules_cc", version="0.0.9")
+bazel_dep(name = "rules_license", version="0.0.8")
+bazel_dep(name = "rules_pkg", version = "1.0.1")
+bazel_dep(name = "abseil-cpp", version = "20240116.2", repo_name = "com_google_absl")
+bazel_dep(name = "abseil-py", version = "2.1.0", repo_name = "com_google_absl_py")
+bazel_dep(name = "google_benchmark", version = "1.8.5", repo_name = "com_google_benchmark")
+bazel_dep(name = "googleapis", version = "0.0.0-20240326-1c8d509c5", repo_name = "com_google_googleapis")
+bazel_dep(name = "protobuf", version = "28.0-rc2", repo_name = "com_google_protobuf")
+bazel_dep(name = "riegeli", version = "0.0.0-20240606-973b6f0", repo_name = "com_google_riegeli")
+bazel_dep(name = "re2", version = "2024-07-02", repo_name = "com_googlesource_code_re2")
+bazel_dep(name = "googletest", version = "1.15.2", repo_name = "com_google_googletest")
+bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "jsonhpp")
+bazel_dep(name = "boringssl", version = "0.0.0-20240530-2db0eb3")
+
+# Not all targets we directly need are available yet. This how to find
+# them:
+# find . -name BUILD -o -name "*.bzl" | xargs sed 's/.*"\(@[_a-z0-9]*\).*/\1/p;d' | sort -u
+# Once added here, remove from the WORKSPACE http archives.
diff --git a/WORKSPACE b/WORKSPACE
@@ -12,46 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# TODO: Everything here should go away and migrate to MODULE.bzlmod
+
 workspace(name = "com_google_xls")
 
-# Load and configure a hermetic LLVM based C/C++ toolchain. This is done here
-# and not in load_external.bzl because it requires several sequential steps of
-# declaring archives and using things in them, which is awkward to do in .bzl
-# files because it's not allowed to use `load` inside of a function.
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-# Released 2023-09-20, current as of 2024-06-26 (but there is already a 0.0.10rc1)
-# Needs to be loaded first, as llvm toolchain has an ancient version of this.
-http_archive(
-    name = "rules_cc",
-    urls = ["https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz"],
-    sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf",
-    strip_prefix = "rules_cc-0.0.9",
-)
-
-# Commit on  2024-07-19, current as of 2024-07-21.
-http_archive(
-    name = "toolchains_llvm",
-    integrity = "sha256-RVp0bZsDrelQAtxswWOLB4j8zCXQy/cSe1m3wL/E2PU=",
-    strip_prefix = "toolchains_llvm-01132cfdae7d7187a885cf79d5a3ac1ed8a02e5a",
-    url = "https://github.com/bazel-contrib/toolchains_llvm/archive/01132cfdae7d7187a885cf79d5a3ac1ed8a02e5a.tar.gz",
-)
-
-load("@toolchains_llvm//toolchain:deps.bzl", "bazel_toolchain_dependencies")
-
-bazel_toolchain_dependencies()
-
-load("@toolchains_llvm//toolchain:rules.bzl", "llvm_toolchain")
-
-llvm_toolchain(
-    name = "llvm_toolchain",
-    llvm_version = "17.0.6",
-)
-
-load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
-
-llvm_register_toolchains()
-
 load("//dependency_support:load_external.bzl", "load_external_repositories")
 
 load_external_repositories()
@@ -75,22 +41,10 @@ python_register_toolchains(
     ignore_root_user_error = True,
 )
 
-# gRPC deps should be loaded before initializing other repos. Otherwise, various
-# errors occur during repo loading and initialization.
-load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
-
-grpc_deps()
-
 load("//dependency_support:initialize_external.bzl", "initialize_external_repositories")
 
 initialize_external_repositories()
 
 load("@xls_pip_deps//:requirements.bzl", xls_pip_install_deps = "install_deps")
 
 xls_pip_install_deps()
-
-# Loading the extra deps must be called after initialize_eternal_repositories or
-# the call to pip_parse fails.
-load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
-
-grpc_extra_deps()
diff --git a/dependency_support/initialize_external.bzl b/dependency_support/initialize_external.bzl
@@ -14,10 +14,12 @@
 
 """Provides helper that initializes external repositories with third-party code."""
 
+# TODO: with MODULE.bzlmod, probably some of these can be removed now, with the
+# eventual goal that not of this is needed anymore.
+
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-load("@com_grail_bazel_compdb//:deps.bzl", "bazel_compdb_deps")
 load("@io_bazel_rules_closure//closure:repositories.bzl", "rules_closure_dependencies", "rules_closure_toolchains")
 load("@project_python//:defs.bzl", python_interpreter_target = "interpreter")
 load("@rules_7zip//:setup.bzl", "setup_7zip")  # needed by rules_hdl
@@ -48,6 +50,5 @@ def initialize_external_repositories():
     )
     initialize_boost()
     initialize_llvm()
-    bazel_compdb_deps()
     benchmark_deps()
     rules_pkg_dependencies()
diff --git a/dependency_support/load_external.bzl b/dependency_support/load_external.bzl
@@ -14,6 +14,9 @@
 
 """Provides helper that loads external repositories with third-party code."""
 
+# This file should go away over time: goal is to convert all these
+# dependencies and put into MODULE.bzlmod
+
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//dependency_support/boost:workspace.bzl", repo_boost = "repo")
 load("//dependency_support/llvm:workspace.bzl", repo_llvm = "repo")
@@ -38,60 +41,6 @@ def load_external_repositories():
     repo_llvm()
     repo_rules_hdl()
 
-    # Release 2024-01-22, current as of 2024-06-26
-    # zlib is added automatically by gRPC, but the zlib BUILD file used by gRPC
-    # does not include all the source code (e.g., gzread is missing) which
-    # breaks other users of zlib like iverilog. So add zlib explicitly here with
-    # a working BUILD file.
-    # Needs to be early in this file to make sure this is the version
-    # picked -- Version 1.3.x fixes function prototype warnings in c++20.
-    http_archive(
-        name = "zlib",
-        sha256 = "50b24b47bf19e1f35d2a21ff36d2a366638cdf958219a66f30ce0861201760e6",
-        strip_prefix = "zlib-1.3.1",
-        urls = [
-            "https://github.com/madler/zlib/archive/v1.3.1.zip",
-        ],
-        build_file = "//dependency_support/zlib:bundled.BUILD.bazel",
-    )
-
-    # V 1.14.0 (released 2023-08-02, current as of 2024-06-26)
-    http_archive(
-        name = "com_google_googletest",
-        urls = ["https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip"],
-        strip_prefix = "googletest-1.14.0",
-        sha256 = "1f357c27ca988c3f7c6b4bf68a9395005ac6761f034046e9dde0896e3aba00e4",
-    )
-
-    # LTS 20240116.2 (released 2024-04-08, current as of 2024-06-26)
-    http_archive(
-        name = "com_google_absl",
-        urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.2.tar.gz"],
-        strip_prefix = "abseil-cpp-20240116.2",
-        sha256 = "733726b8c3a6d39a4120d7e45ea8b41a434cdacde401cba500f14236c49b39dc",
-    )
-
-    # Released 2024-06-03, current as of 2024-06-26
-    # Protobuf depends on Skylib
-    # Load bazel skylib as per
-    # https://github.com/bazelbuild/bazel-skylib/releases
-    http_archive(
-        name = "bazel_skylib",
-        urls = [
-            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.1/bazel-skylib-1.7.1.tar.gz",
-        ],
-        sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f",
-    )
-
-    http_archive(
-        name = "boringssl",
-        # Commit date: 2024-06-24
-        # Note for updating: we need to use a commit from the main-with-bazel branch.
-        strip_prefix = "boringssl-e6b03733628149a89a1d18b3ef8f39aa1055aba8",
-        sha256 = "006596f84d9cc142d9d6c48600cf6208f9d24426943b05e8bcda06e523f69dc8",
-        urls = ["https://github.com/google/boringssl/archive/e6b03733628149a89a1d18b3ef8f39aa1055aba8.tar.gz"],
-    )
-
     # Commit on 2023-02-09
     http_archive(
         name = "pybind11_bazel",
@@ -115,27 +64,6 @@ def load_external_repositories():
         ],
     )
 
-    # Version release tag 2023-01-11
-    http_archive(
-        name = "com_google_absl_py",
-        strip_prefix = "abseil-py-1.4.0",
-        urls = ["https://github.com/abseil/abseil-py/archive/refs/tags/v1.4.0.tar.gz"],
-        sha256 = "0fb3a4916a157eb48124ef309231cecdfdd96ff54adf1660b39c0d4a9790a2c0",
-    )
-
-    # Released on 2024-06-01, current as of 2024-06-26
-    http_archive(
-        name = "com_googlesource_code_re2",
-        strip_prefix = "re2-2024-06-01",
-        sha256 = "7326c74cddaa90b12090fcfc915fe7b4655723893c960ee3c2c66e85c5504b6c",
-        urls = [
-            "https://github.com/google/re2/archive/refs/tags/2024-06-01.tar.gz",
-        ],
-        repo_mapping = {
-            "@abseil-cpp": "@com_google_absl",
-        },
-    )
-
     # Released on 2022-12-27.
     # Current as of 2024-06-26 would be 6.0.2, but that does not work yet
     # with rules_hdl (it assumes rules_proto_toolchains is in repositories.bzl)
@@ -184,14 +112,6 @@ def load_external_repositories():
         build_file = "//dependency_support/linenoise:bundled.BUILD.bazel",
     )
 
-    # Commit from 2024-06-26
-    http_archive(
-        name = "com_google_riegeli",
-        sha256 = "38fd4b6bc24958ae51e1a5a0eb57ce9c3dbbaf5034a78453a4d133597fbf31e4",
-        strip_prefix = "riegeli-cb68d579f108c96831b6a7815da43ff24b4e5242",
-        url = "https://github.com/google/riegeli/archive/cb68d579f108c96831b6a7815da43ff24b4e5242.tar.gz",
-    )
-
     # Needed by fuzztest. Release 2024-05-21, current as of 2024-06-26
     http_archive(
         name = "snappy",
@@ -218,19 +138,6 @@ def load_external_repositories():
         urls = ["https://github.com/google/highwayhash/archive/f8381f3331d9c56a9792f9b4a35f61c41108c39e.tar.gz"],
     )
 
-    # Released 2024-06-07, current as of 2024-06-26.
-    http_archive(
-        name = "com_github_grpc_grpc",
-        urls = ["https://github.com/grpc/grpc/archive/v1.64.2.tar.gz"],
-        patches = ["//dependency_support/com_github_grpc_grpc:0001-Add-absl-status-to-deps.patch"],
-        sha256 = "c682fc39baefc6e804d735e6b48141157b7213602cc66dbe0bf375b904d8b5f9",
-        strip_prefix = "grpc-1.64.2",
-        repo_mapping = {
-            "@local_config_python": "@project_python",
-            "@system_python": "@project_python",
-        },
-    )
-
     # Used by xlscc. Tagged 2024-02-16 (note: release is lagging tag), current as of 2024-06-26
     http_archive(
         name = "com_github_hlslibs_ac_types",
@@ -259,14 +166,6 @@ def load_external_repositories():
         strip_prefix = "or-tools-" + ORTOOLS_VERSION,
     )
 
-    # Released 2024-05-23, current as of 2024-06-26.
-    http_archive(
-        name = "com_google_benchmark",
-        urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz"],
-        sha256 = "3e7059b6b11fb1bbe28e33e02519398ca94c1818874ebed18e504dc6f709be45",
-        strip_prefix = "benchmark-1.8.4",
-    )
-
     # Updated to head on 2024-03-14
     FUZZTEST_COMMIT = "393ae75c0fca5f9892e73969da5d6bce453ad318"
     http_archive(
@@ -278,23 +177,6 @@ def load_external_repositories():
         patches = ["//dependency_support/com_google_fuzztest:e317d5277e34948ae7048cb5e48309e0288e8df3.patch"],
     )
 
-    # Released 2024-01-24, current as of 2024-06-26
-    http_archive(
-        name = "rules_license",
-        urls = [
-            "https://github.com/bazelbuild/rules_license/releases/download/0.0.8/rules_license-0.0.8.tar.gz",
-        ],
-        sha256 = "241b06f3097fd186ff468832150d6cc142247dc42a32aaefb56d0099895fd229",
-    )
-
-    # 2022-09-19
-    http_archive(
-        name = "com_grail_bazel_compdb",
-        sha256 = "a3ff6fe238eec8202270dff75580cba3d604edafb8c3408711e82633c153efa8",
-        strip_prefix = "bazel-compilation-database-940cedacdb8a1acbce42093bf67f3a5ca8b265f7",
-        urls = ["https://github.com/grailbio/bazel-compilation-database/archive/940cedacdb8a1acbce42093bf67f3a5ca8b265f7.tar.gz"],
-    )
-
     # Tagged 2024-08-23, current as of 2024-08-24
     VERIBLE_TAG = "v0.0-3756-gda9a0f8c"
     http_archive(
@@ -306,24 +188,6 @@ def load_external_repositories():
         patches = ["//dependency_support/verible:visibility.patch"],
     )
 
-    # Used by Verible; current as of 2024-06-26
-    http_archive(
-        name = "jsonhpp",
-        build_file = "@verible//bazel:jsonhpp.BUILD",
-        sha256 = "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406",
-        strip_prefix = "json-3.11.3",
-        urls = [
-            "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz",
-        ],
-    )
-
-    # Released 2024-06-03, current as of 2024-06-26
-    http_archive(
-        name = "rules_pkg",
-        urls = ["https://github.com/bazelbuild/rules_pkg/releases/download/1.0.0/rules_pkg-1.0.0.tar.gz"],
-        sha256 = "cad05f864a32799f6f9022891de91ac78f30e0fa07dc68abac92a628121b5b11",
-    )
-
     # Used in C++ tests of the ZSTD Module
     # Transitive dependency of fuzztest (required by riegeli in fuzztest workspace)
     # Version fdfb2aff released on 2024-07-31

diff --git a/xls/dev_tools/make-compilation-db.sh b/xls/dev_tools/make-compilation-db.sh
@@ -13,24 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -u
-set -e
-
-readonly OUTPUT_BASE="$(bazel info output_base)"
-
-readonly COMPDB_SCRIPT="${OUTPUT_BASE}/external/com_grail_bazel_compdb/generate.py"
-[ -r "${COMPDB_SCRIPT}" ] || bazel fetch @com_grail_bazel_compdb//...
-
-python3 "${COMPDB_SCRIPT}"
-
-# Massage the output so that clang-tidy fully undestands the compile commands:
-# remove flags where it gets confused.
-# Also, make sure that the command also contains -xc++ (bazel sometimes does
-# not add that in libraries that don't have a *.cc file but only *.h or *.inc)
-sed -i compile_commands.json -f - <<EOF
-s/"command": "\([^ ]*\) /"command": "\1 -x c++ /  # -xc++ as first argument.
-s/ -f[^ ]*/ /g             # remove all -fxyz options
-s/ --target[^ ]*/ /g       # target platform not needed, might confuse
-s/ -stdlib=libc++/ /       # otherwise, clang-tidy does not find c++ headers
-EOF
-
+echo "Sorry, currently no compilation DB with bazel 7 and bzlmod. Stay tuned."
+exit 1