go_repository: add 'clean' build_file_generation

Before bzlmod, repositories were global, and BUILD files inside upstream
repositories could load from the repositories defined at the root-level
WORKSPACE file.

With the introduction of visibility rules, these result in errors that
cannot trivially be worked around. One pattern that has emerged is
ignoring existing build files by specifying `gazelle:build_file_name
BUILD.bazel`, however this does not work for anything already using
BUILD.bazel filenames.

This does affect real-world users: an example is
github.com/google/go-jsonnet which has a BUILD.bazel file to `genrule`
its AST. The module is pure-go, but the build step contains cpp code,
and when imported under bzlmod-gazelle will not work because the cpp
code is invoked. As a workaround, there is now a 'jsonnet' BCR module.

This patch introduces a `build_file_generation = clean` option, which
removes existing build files before generating new ones. It allows
loading the jsonnet code once again, and also makes it possible to
load some of the complex protobuf repositories.

README.rst

@@ -674,7 +674,7 @@ The following flags are accepted:
 +----------------------------------------------------------------------------------------------------------+----------------------------------------------+
 | Sets the ``build_extra_args attribute`` for the generated `go_repository`_ rule(s).                                                                     |
 +----------------------------------------------------------------------------------------------------------+----------------------------------------------+
-| :flag:`-build_file_generation auto|on|off`                                                               |                                              |
+| :flag:`-build_file_generation auto|on|off|clean`                                                         |                                              |
 +----------------------------------------------------------------------------------------------------------+----------------------------------------------+
 | Sets the ``build_file_generation`` attribute for the generated `go_repository`_ rule(s).                                                                |
 +----------------------------------------------------------------------------------------------------------+----------------------------------------------+

internal/bzlmod/go_deps.bzl

@@ -48,7 +48,7 @@ https://github.com/bazelbuild/bazel-gazelle/tree/master/internal/bzlmod/default_
 _GAZELLE_ATTRS = {
     "build_file_generation": attr.string(
         default = "on",
-        doc = """One of `"auto"`, `"on"` (default), `"off"`.
+        doc = """One of `"auto"`, `"on"` (default), `"off"`, `"clean"`.
 
         Whether Gazelle should generate build files for the Go module.
 
@@ -60,11 +60,14 @@ _GAZELLE_ATTRS = {
         In `"auto"` mode, Gazelle will run if there is no build file in the Go
         module's root directory.
 
+        In `"clean"` mode, Gazelle will not reuse any existing build files.
+
         """,
         values = [
             "auto",
             "off",
             "on",
+            "clean",
         ],
     ),
     "build_extra_args": attr.string_list(

internal/go_repository.bzl

@@ -128,7 +128,7 @@ def _go_repository_impl(ctx):
     go_env_cache = str(ctx.path(Label("@bazel_gazelle_go_repository_cache//:go.env")))
     if not ctx.attr.urls:
         fetch_repo = str(ctx.path(Label("@bazel_gazelle_go_repository_tools//:bin/fetch_repo{}".format(executable_extension(ctx)))))
-    generate = ctx.attr.build_file_generation == "on"
+    generate = ctx.attr.build_file_generation in ["on", "clean"]
     _gazelle = "@bazel_gazelle_go_repository_tools//:bin/gazelle{}".format(executable_extension(ctx))
     if generate:
         gazelle_path = ctx.path(Label(_gazelle))
@@ -296,8 +296,26 @@ def _go_repository_impl(ctx):
         if result.return_code:
             fail("%s: %s" % (ctx.name, result.stderr))
 
-    # Repositories are fetched. Determine if build file generation is needed.
     build_file_names = ctx.attr.build_file_name.split(",")
+
+    # Clean existing build files if requested
+    if ctx.attr.build_file_generation == "clean":
+        # Pure-starlark way of doing the equivalent of "find . -type f -name BUILD -delete".
+        # Since Windows doesn't have the "find" command we have to do it here, and since
+        # Starlark doesn't let us write a while loop, we have to hope the user has less
+        # than a billion files.
+        paths = [ctx.path("")]
+        for i in range(1000000000):
+            if len(paths) == 0:
+                break
+            path = paths.pop()
+            if path.is_dir:
+                paths += path.readdir(watch = 'no')
+            else:
+                if path.basename in build_file_names:
+                    ctx.delete(path)
+
+    # Repositories are fetched. Determine if build file generation is needed.
     existing_build_file = ""
     for name in build_file_names:
         path = ctx.path(name)
@@ -515,15 +533,17 @@ go_repository = repository_rule(
         ),
         "build_file_generation": attr.string(
             default = "auto",
-            doc = """One of `"auto"`, `"on"`, `"off"`.
+            doc = """One of `"auto"`, `"on"`, `"off"`, `"clean"`.
 
             Whether Gazelle should generate build files in the repository. In `"auto"`
             mode, Gazelle will run if there is no build file in the repository root
-            directory.""",
+            directory. In `"clean"` mode, Gazelle will not reuse any existing build
+            files.""",
             values = [
                 "on",
                 "auto",
                 "off",
+                "clean",
             ],
         ),
         "build_naming_convention": attr.string(

repository.md

@@ -178,7 +178,7 @@ go_repository(
 | <a id="go_repository-build_directives"></a>build_directives |  A list of directives to be written to the root level build file before Calling Gazelle to generate build files. Each string in the list will be prefixed with `#` automatically. A common use case is to pass a list of Gazelle directives.   | List of strings | optional |  `[]`  |
 | <a id="go_repository-build_external"></a>build_external |  One of `"external"`, `"static"` or `"vendored"`.<br><br>This sets Gazelle's `-external` command line flag. In `"static"` mode, Gazelle will not call out to the network to resolve imports.<br><br>**NOTE:** This cannot be used to ignore the `vendor` directory in a repository. The `-external` flag only controls how Gazelle resolves imports which are not present in the repository. Use `build_extra_args = ["-exclude=vendor"]` instead.   | String | optional |  `"static"`  |
 | <a id="go_repository-build_extra_args"></a>build_extra_args |  A list of additional command line arguments to pass to Gazelle when generating build files.   | List of strings | optional |  `[]`  |
-| <a id="go_repository-build_file_generation"></a>build_file_generation |  One of `"auto"`, `"on"`, `"off"`.<br><br>Whether Gazelle should generate build files in the repository. In `"auto"` mode, Gazelle will run if there is no build file in the repository root directory.   | String | optional |  `"auto"`  |
+| <a id="go_repository-build_file_generation"></a>build_file_generation |  One of `"auto"`, `"on"`, `"off"`, `"clean"`.<br><br>Whether Gazelle should generate build files in the repository. In `"auto"` mode, Gazelle will run if there is no build file in the repository root directory. In `"clean"` mode, Gazelle will not reuse any existing build files.   | String | optional |  `"auto"`  |
 | <a id="go_repository-build_file_name"></a>build_file_name |  Comma-separated list of names Gazelle will consider to be build files. If a repository contains files named `build` that aren't related to Bazel, it may help to set this to `"BUILD.bazel"`, especially on case-insensitive file systems.   | String | optional |  `"BUILD.bazel,BUILD"`  |
 | <a id="go_repository-build_file_proto_mode"></a>build_file_proto_mode |  One of `"default"`, `"legacy"`, `"disable"`, `"disable_global"` or `"package"`.<br><br>This sets Gazelle's `-proto` command line flag. See [Directives] for more information on each mode.   | String | optional |  `""`  |
 | <a id="go_repository-build_naming_convention"></a>build_naming_convention |  Sets the library naming convention to use when resolving dependencies against this external repository. If unset, the convention from the external workspace is used. Legal values are `go_default_library`, `import`, and `import_alias`.<br><br>See the `gazelle:go_naming_convention` directive in [Directives] for more information.   | String | optional |  `"import_alias"`  |

tests/bcr/go_mod/MODULE.bazel

@@ -102,6 +102,12 @@ go_deps.archive_override(
     ],
 )
 
+# Test a module that ships with its own BUILD files and first needs rewriting
+go_deps.gazelle_override(
+    build_file_generation = "clean",
+    path = "github.com/google/go-jsonnet",
+)
+
 # Transitive dependencies have to be listed here explicitly.
 go_deps.module(
     indirect = True,
@@ -130,6 +136,7 @@ use_repo(
     "com_github_datadog_sketches_go",
     "com_github_envoyproxy_protoc_gen_validate",
     "com_github_fmeum_dep_on_gazelle",
+    "com_github_google_go_jsonnet",
     "com_github_google_safetext",
     "com_github_stretchr_testify",
     "org_golang_google_protobuf",

tests/bcr/go_mod/go.mod

@@ -23,13 +23,16 @@ require (
 require (
 	github.com/bazelbuild/bazel-gazelle v0.30.0 // indirect
 	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/google/go-jsonnet v0.20.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/yaml.v2 v2.2.7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	rsc.io/quote v1.5.2 // indirect
 	rsc.io/sampler v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.1.0 // indirect
 )
 
 replace example.org/hello => ../../fixtures/hello

tests/bcr/go_mod/go.sum

@@ -49,6 +49,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g=
+github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI=
@@ -117,6 +119,8 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -126,3 +130,5 @@ rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
 rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
 rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

tests/bcr/go_mod/pkg/BUILD.bazel

@@ -58,6 +58,7 @@ go_test(
         "@com_github_datadog_sketches_go//ddsketch",
         "@com_github_envoyproxy_protoc_gen_validate//validate",
         "@com_github_fmeum_dep_on_gazelle//:dep_on_gazelle",
+        "@com_github_google_go_jsonnet//:go-jsonnet",
         "@com_github_google_safetext//yamltemplate",
         "@com_github_stretchr_testify//require:go_default_library",
         "@my_rules_go//go/runfiles",

tests/bcr/go_mod/pkg/pkg_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/cloudflare/circl/dh/x25519"
 	"github.com/fmeum/dep_on_gazelle"
+	"github.com/google/go-jsonnet"
 	"github.com/google/safetext/yamltemplate"
 	"github.com/stretchr/testify/require"
 
@@ -38,6 +39,11 @@ func TestBuildFileGeneration(t *testing.T) {
 	yamltemplate.HTMLEscapeString("<b>foo</b>")
 }
 
+func TestCleanBuildFileGeneration(t *testing.T) {
+	// github.com/google/go-jsonnet@v0.20.0 requires fully replacing the BUILD files it provides
+	jsonnet.Version()
+}
+
 func TestGeneratedFilesPreferredOverProtos(t *testing.T) {
 	_, _ = ddsketch.NewDefaultDDSketch(0.01)
 }