feat(gazelle): Add "python_test_file_pattern" directive

CHANGELOG.md

@@ -39,12 +39,18 @@ A brief description of the categories of changes:
 * (gazelle) Added a new `python_default_visibility` directive to control the
   _default_ visibility of generated targets. See the [docs][python_default_visibility]
   for details.
+* (gazelle) Added a new `python_test_file_pattern` directive. This directive tells
+  gazelle which python files should be mapped to the `py_test` rule. See the
+  [original issue][test_file_pattern_issue] and the [docs][test_file_pattern_docs]
+  for details.
 
 * (wheel) Add support for `data_files` attributes in py_wheel rule
   ([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
 
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 [python_default_visibility]: gazelle/README.md#directive-python_default_visibility
+[test_file_pattern_issue]: https://github.com/bazelbuild/rules_python/issues/1816
+[test_file_pattern_docs]: gazelle/README.md#directive-python_test_file_pattern
 
 ### Changed
 

gazelle/MODULE.bazel

@@ -14,7 +14,7 @@ go_deps.from_file(go_mod = "//:go.mod")
 use_repo(
     go_deps,
     "com_github_bazelbuild_buildtools",
-    "com_github_bmatcuk_doublestar",
+    "com_github_bmatcuk_doublestar_v4",
     "com_github_emirpasic_gods",
     "com_github_ghodss_yaml",
     "in_gopkg_yaml_v2",

gazelle/README.md

@@ -202,6 +202,8 @@ Python-specific directives are as follows:
 | Instructs gazelle to use these visibility labels on all python targets. `labels` is a comma-separated list of labels (without spaces). | `//$python_root:__subpackages__` |
 | [`# gazelle:python_visibility label`](#directive-python_visibility) | |
 | Appends additional visibility labels to each generated target. This directive can be set multiple times. | |
+| [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` |
+| Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. |
 
 
 #### Directive: `python_root`:
@@ -359,6 +361,70 @@ py_library(
 ```
 
 
+#### Directive: `python_test_file_pattern`:
+
+This directive adjusts which python files will be mapped to the `py_test` rule.
+
++ The default is `*_test.py,test_*.py`: both `test_*.py` and `*_test.py` files
+  will generate `py_test` targets.
++ This directive must have a value. If no value is given, an error will be raised.
++ It is recommended, though not necessary, to include the `.py` extension in
+  the `glob`s: `foo*.py,?at.py`.
++ Like most directives, it applies to the current Bazel package and all subpackages
+  until the directive is set again.
++ This directive accepts multiple `glob` patterns, separated by commas without spaces:
+
+```starlark
+# gazelle:python_test_file_pattern foo*.py,?at
+
+py_library(
+    name = "mylib",
+    srcs = ["mylib.py"],
+)
+
+py_test(
+    name = "foo_bar",
+    srcs = ["foo_bar.py"],
+)
+
+py_test(
+    name = "cat",
+    srcs = ["cat.py"],
+)
+
+py_test(
+    name = "hat",
+    srcs = ["hat.py"],
+)
+```
+
+
+##### Notes
+
+Resetting to the default value (such as in a subpackage) is manual. Set:
+
+```starlark
+# gazelle:python_test_file_pattern *_test.py,test_*.py
+```
+
+There currently is no way to tell gazelle that _no_ files in a package should
+be mapped to `py_test` targets (see [Issue #1826][issue-1826]). The workaround
+is to set this directive to a pattern that will never match a `.py` file, such
+as `foo.bar`:
+
+```starlark
+# No files in this package should be mapped to py_test targets.
+# gazelle:python_test_file_pattern foo.bar
+
+py_library(
+    name = "my_test",
+    srcs = ["my_test.py"],
+)
+```
+
+[issue-1826]: https://github.com/bazelbuild/rules_python/issues/1826
+
+
 ### Libraries
 
 Python source files are those ending in `.py` but not ending in `_test.py`.
@@ -438,7 +504,7 @@ for more information on extending Gazelle.
 
 If you add new Go dependencies to the plugin source code, you need to "tidy" the go.mod file.
 After changing that file, run `go mod tidy` or `bazel run @go_sdk//:bin/go -- mod tidy`
-to update the go.mod and go.sum files. Then run `bazel run //:update_go_deps` to have gazelle
+to update the go.mod and go.sum files. Then run `bazel run //:gazelle_update_repos` to have gazelle
 add the new dependenies to the deps.bzl file. The deps.bzl file is used as defined in our /WORKSPACE
 to include the external repos Bazel loads Go dependencies from.
 

gazelle/deps.bzl

@@ -38,10 +38,10 @@ def gazelle_deps():
     )
 
     go_repository(
-        name = "com_github_bmatcuk_doublestar",
-        importpath = "github.com/bmatcuk/doublestar",
-        sum = "h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=",
-        version = "v1.3.4",
+        name = "com_github_bmatcuk_doublestar_v4",
+        importpath = "github.com/bmatcuk/doublestar/v4",
+        sum = "h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=",
+        version = "v4.6.1",
     )
 
     go_repository(

gazelle/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/bazelbuild/bazel-gazelle v0.31.1
 	github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d
 	github.com/bazelbuild/rules_go v0.41.0
-	github.com/bmatcuk/doublestar v1.3.4
+	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0
 	gopkg.in/yaml.v2 v2.4.0

gazelle/go.sum

@@ -6,8 +6,8 @@ github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d h1:Fl1FfItZp
 github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
 github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k=
 github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
+github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

gazelle/python/BUILD.bazel

@@ -38,7 +38,7 @@ go_library(
         "@bazel_gazelle//resolve:go_default_library",
         "@bazel_gazelle//rule:go_default_library",
         "@com_github_bazelbuild_buildtools//build:go_default_library",
-        "@com_github_bmatcuk_doublestar//:doublestar",
+        "@com_github_bmatcuk_doublestar_v4//:doublestar",
         "@com_github_emirpasic_gods//lists/singlylinkedlist",
         "@com_github_emirpasic_gods//sets/treeset",
         "@com_github_emirpasic_gods//utils",

gazelle/python/configure.go

@@ -65,6 +65,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.TestNamingConvention,
 		pythonconfig.DefaultVisibilty,
 		pythonconfig.Visibility,
+		pythonconfig.TestFilePattern,
 	}
 }
 
@@ -181,6 +182,13 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 			}
 		case pythonconfig.Visibility:
 			config.AppendVisibility(strings.TrimSpace(d.Value))
+		case pythonconfig.TestFilePattern:
+			value := strings.TrimSpace(d.Value)
+			if value == "" {
+				log.Fatal("ERROR: Directive 'python_test_file_pattern' requires a value.")
+			}
+			globStrings := strings.Split(value, ",")
+			config.SetTestFilePattern(globStrings)
 		}
 	}
 

gazelle/python/generate.go

@@ -28,7 +28,7 @@ import (
 	"github.com/bazelbuild/bazel-gazelle/language"
 	"github.com/bazelbuild/bazel-gazelle/rule"
 	"github.com/bazelbuild/rules_python/gazelle/pythonconfig"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/emirpasic/gods/lists/singlylinkedlist"
 	"github.com/emirpasic/gods/sets/treeset"
 	godsutils "github.com/emirpasic/gods/utils"
@@ -54,6 +54,19 @@ func GetActualKindName(kind string, args language.GenerateArgs) string {
 	return kind
 }
 
+func matchesAnyGlob(s string, globs []string) bool {
+	for _, g := range globs {
+		ok, err := doublestar.Match(g, s)
+		if err != nil {
+			log.Fatalf("ERROR: Failed to compile glob '%v'. Error: %v\n", g, err)
+		}
+		if ok {
+			return true
+		}
+	}
+	return false
+}
+
 // GenerateRules extracts build metadata from source files in a directory.
 // GenerateRules is called in each directory where an update is requested
 // in depth-first post-order.
@@ -100,6 +113,8 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 	hasPyTestEntryPointTarget := false
 	hasConftestFile := false
 
+	testFileGlobs := cfg.TestFilePattern()
+
 	for _, f := range args.RegularFiles {
 		if cfg.IgnoresFile(filepath.Base(f)) {
 			continue
@@ -113,7 +128,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 				hasPyTestEntryPointFile = true
 			} else if f == conftestFilename {
 				hasConftestFile = true
-			} else if strings.HasSuffix(f, "_test.py") || strings.HasPrefix(f, "test_") {
+			} else if matchesAnyGlob(f, testFileGlobs) {
 				pyTestFilenames.Add(f)
 			} else {
 				pyLibraryFilenames.Add(f)
@@ -195,7 +210,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 							}
 						}
 						baseName := filepath.Base(path)
-						if strings.HasSuffix(baseName, "_test.py") || strings.HasPrefix(baseName, "test_") {
+						if matchesAnyGlob(baseName, testFileGlobs) {
 							pyTestFilenames.Add(srcPath)
 						} else {
 							pyLibraryFilenames.Add(srcPath)

gazelle/python/testdata/directive_python_test_file_pattern/README.md

@@ -0,0 +1,19 @@
+# Directive: `python_test_file_pattern`
+
+This test case asserts that the `# gazelle:python_test_file_pattern` directive
+works as intended.
+
+It consists of 6 cases:
+
+1.  When not set, both `*_test.py` and `test_*.py` files are mapped to the `py_test`
+    rule.
+2.  When set to a single value `*_test.py`, `test_*.py` files are mapped to the
+    `py_library` rule.
+3.  When set to a single value `test_*.py`, `*_test.py` files are mapped to the
+    `py_library` rule (ie: the inverse of case 2, but also with "file" generation
+    mode).
+4.  Arbitrary `glob` patterns are supported.
+5.  Multiple `glob` patterns are supported and that patterns don't technically
+    need to end in `.py` if they end in a wildcard (eg: we won't make a `py_test`
+    target for the extensionless file `test_foo`).
+6.  Sub-packages can override the directive's value.

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_generation_mode file

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.out

@@ -0,0 +1,18 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+# gazelle:python_generation_mode file
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test2_star_test_py/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_test_file_pattern *_test.py

gazelle/python/testdata/directive_python_test_file_pattern/test2_star_test_py/BUILD.out

@@ -0,0 +1,17 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_test.py
+
+py_library(
+    name = "test2_star_test_py",
+    srcs = [
+        "test_goodbye.py",
+        "test_hello.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test3_test_star_py/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py

gazelle/python/testdata/directive_python_test_file_pattern/test3_test_star_py/BUILD.out

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py
+
+py_library(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test4_glob/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py

gazelle/python/testdata/directive_python_test_file_pattern/test4_glob/BUILD.out

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py
+
+py_library(
+    name = "foo_nota_test0_Z1",
+    srcs = ["foo_nota_test0_Z1.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_helloworld_A_testA",
+    srcs = ["foo_helloworld_A_testA.py"],
+)
+
+py_test(
+    name = "foo_my_filename_B_test1",
+    srcs = ["foo_my_filename_B_test1.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test5_multiple_patterns/BUILD.in

@@ -0,0 +1,3 @@
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.

gazelle/python/testdata/directive_python_test_file_pattern/test5_multiple_patterns/BUILD.out

@@ -0,0 +1,34 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.
+
+py_library(
+    name = "test5_multiple_patterns",
+    srcs = [
+        "mylib.py",
+        "mylib2.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_hello",
+    srcs = ["foo_hello.py"],
+)
+
+py_test(
+    name = "foo_unittest",
+    srcs = ["foo_unittest.py"],
+)
+
+py_test(
+    name = "hello_foo",
+    srcs = ["hello_foo.py"],
+)
+
+py_test(
+    name = "unittest_foo",
+    srcs = ["unittest_foo.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test6_nesting/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern *_unittest.py