feat(gazelle): Add "python_test_file_pattern" directive

CHANGELOG.md

@@ -39,12 +39,18 @@ A brief description of the categories of changes:
 * (gazelle) Added a new `python_default_visibility` directive to control the
   _default_ visibility of generated targets. See the [docs][python_default_visibility]
   for details.
+* (gazelle) Added a new `python_test_file_pattern` directive. This directive tells
+  gazelle which python files should be mapped to the `py_test` rule. See the
+  [original issue][test_file_pattern_issue] and the [docs][test_file_pattern_docs]
+  for details.
 
 * (wheel) Add support for `data_files` attributes in py_wheel rule
   ([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
 
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 [python_default_visibility]: gazelle/README.md#directive-python_default_visibility
+[test_file_pattern_issue]: https://github.com/bazelbuild/rules_python/issues/1816
+[test_file_pattern_docs]: gazelle/README.md#directive-python_test_file_pattern
 
 ### Changed
 

gazelle/MODULE.bazel

@@ -17,5 +17,6 @@ use_repo(
     "com_github_bmatcuk_doublestar",
     "com_github_emirpasic_gods",
     "com_github_ghodss_yaml",
+    "com_github_gobwas_glob",
     "in_gopkg_yaml_v2",
 )

gazelle/README.md

@@ -202,6 +202,8 @@ Python-specific directives are as follows:
 | Instructs gazelle to use these visibility labels on all python targets. `labels` is a comma-separated list of labels (without spaces). | `//$python_root:__subpackages__` |
 | [`# gazelle:python_visibility label`](#directive-python_visibility) | |
 | Appends additional visibility labels to each generated target. This directive can be set multiple times. | |
+| [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` |
+| Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. |
 
 
 #### Directive: `python_root`:
@@ -359,6 +361,70 @@ py_library(
 ```
 
 
+#### Directive: `python_test_file_pattern`:
+
+This directive adjusts which python files will be mapped to the `py_test` rule.
+
++ The default is `*_test.py,test_*.py`: both `test_*.py` and `*_test.py` files
+  will generate `py_test` targets.
++ This directive must have a value. If no value is given, an error will be raised.
++ It is recommended, though not necessary, to include the `.py` extension in
+  the `glob`s: `foo*.py,?at.py`.
++ Like most directives, it applies to the current Bazel package and all subpackages
+  until the directive is set again.
++ This directive accepts multiple `glob` patterns, separated by commas without spaces:
+
+```starlark
+# gazelle:python_test_file_pattern foo*.py,?at
+
+py_library(
+    name = "mylib",
+    srcs = ["mylib.py"],
+)
+
+py_test(
+    name = "foo_bar",
+    srcs = ["foo_bar.py"],
+)
+
+py_test(
+    name = "cat",
+    srcs = ["cat.py"],
+)
+
+py_test(
+    name = "hat",
+    srcs = ["hat.py"],
+)
+```
+
+
+##### Notes
+
+Resetting to the default value (such as in a subpackage) is manual. Set:
+
+```starlark
+# gazelle:python_test_file_pattern *_test.py,test_*.py
+```
+
+There currently is no way to tell gazelle that _no_ files in a package should
+be mapped to `py_test` targets (see [Issue #1826][issue-1826]). The workaround
+is to set this directive to a pattern that will never match a `.py` file, such
+as `foo.bar`:
+
+```starlark
+# No files in this package should be mapped to py_test targets.
+# gazelle:python_test_file_pattern foo.bar
+
+py_library(
+    name = "my_test",
+    srcs = ["my_test.py"],
+)
+```
+
+[issue-1826]: https://github.com/bazelbuild/rules_python/issues/1826
+
+
 ### Libraries
 
 Python source files are those ending in `.py` but not ending in `_test.py`.
@@ -438,7 +504,7 @@ for more information on extending Gazelle.
 
 If you add new Go dependencies to the plugin source code, you need to "tidy" the go.mod file.
 After changing that file, run `go mod tidy` or `bazel run @go_sdk//:bin/go -- mod tidy`
-to update the go.mod and go.sum files. Then run `bazel run //:update_go_deps` to have gazelle
+to update the go.mod and go.sum files. Then run `bazel run //:gazelle_update_repos` to have gazelle
 add the new dependenies to the deps.bzl file. The deps.bzl file is used as defined in our /WORKSPACE
 to include the external repos Bazel loads Go dependencies from.
 

gazelle/deps.bzl

@@ -105,6 +105,12 @@ def gazelle_deps():
         sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=",
         version = "v1.0.0",
     )
+    go_repository(
+        name = "com_github_gobwas_glob",
+        importpath = "github.com/gobwas/glob",
+        sum = "h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=",
+        version = "v0.2.3",
+    )
     go_repository(
         name = "com_github_golang_glog",
         importpath = "github.com/golang/glog",

gazelle/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0
+	github.com/gobwas/glob v0.2.3
 	gopkg.in/yaml.v2 v2.4.0
 )
 

gazelle/go.sum

@@ -19,6 +19,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

gazelle/python/BUILD.bazel

@@ -42,6 +42,7 @@ go_library(
         "@com_github_emirpasic_gods//lists/singlylinkedlist",
         "@com_github_emirpasic_gods//sets/treeset",
         "@com_github_emirpasic_gods//utils",
+        "@com_github_gobwas_glob//:glob",
     ],
 )
 

gazelle/python/configure.go

@@ -65,6 +65,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.TestNamingConvention,
 		pythonconfig.DefaultVisibilty,
 		pythonconfig.Visibility,
+		pythonconfig.TestFilePattern,
 	}
 }
 
@@ -181,6 +182,13 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 			}
 		case pythonconfig.Visibility:
 			config.AppendVisibility(strings.TrimSpace(d.Value))
+		case pythonconfig.TestFilePattern:
+			value := strings.TrimSpace(d.Value)
+			if value == "" {
+				log.Fatal("ERROR: Directive 'python_test_file_pattern' requires a value.")
+			}
+			globStrings := strings.Split(value, ",")
+			config.SetTestFilePattern(globStrings)
 		}
 	}
 

gazelle/python/generate.go

@@ -32,6 +32,7 @@ import (
 	"github.com/emirpasic/gods/lists/singlylinkedlist"
 	"github.com/emirpasic/gods/sets/treeset"
 	godsutils "github.com/emirpasic/gods/utils"
+	"github.com/gobwas/glob"
 )
 
 const (
@@ -54,6 +55,27 @@ func GetActualKindName(kind string, args language.GenerateArgs) string {
 	return kind
 }
 
+func makeCompiledGlobs(globs []string) []glob.Glob {
+	compiledGlobs := []glob.Glob{}
+	for _, value := range globs {
+		compiledGlob, err := glob.Compile(value)
+		if err != nil {
+			log.Fatalf("ERROR: Failed to compile glob '%v'. Error: %v\n", value, err)
+		}
+		compiledGlobs = append(compiledGlobs, compiledGlob)
+	}
+	return compiledGlobs
+}
+
+func matchesAnyGlob(s string, globs []glob.Glob) bool {
+	for _, g := range globs {
+		if g.Match(s) {
+			return true
+		}
+	}
+	return false
+}
+
 // GenerateRules extracts build metadata from source files in a directory.
 // GenerateRules is called in each directory where an update is requested
 // in depth-first post-order.
@@ -100,6 +122,8 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 	hasPyTestEntryPointTarget := false
 	hasConftestFile := false
 
+	testFileGlobs := makeCompiledGlobs(cfg.TestFilePattern())
+
 	for _, f := range args.RegularFiles {
 		if cfg.IgnoresFile(filepath.Base(f)) {
 			continue
@@ -113,7 +137,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 				hasPyTestEntryPointFile = true
 			} else if f == conftestFilename {
 				hasConftestFile = true
-			} else if strings.HasSuffix(f, "_test.py") || strings.HasPrefix(f, "test_") {
+			} else if matchesAnyGlob(f, testFileGlobs) {
 				pyTestFilenames.Add(f)
 			} else {
 				pyLibraryFilenames.Add(f)
@@ -195,7 +219,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 							}
 						}
 						baseName := filepath.Base(path)
-						if strings.HasSuffix(baseName, "_test.py") || strings.HasPrefix(baseName, "test_") {
+						if matchesAnyGlob(baseName, testFileGlobs) {
 							pyTestFilenames.Add(srcPath)
 						} else {
 							pyLibraryFilenames.Add(srcPath)

gazelle/python/testdata/directive_python_test_file_pattern/README.md

@@ -0,0 +1,19 @@
+# Directive: `python_test_file_pattern`
+
+This test case asserts that the `# gazelle:python_test_file_pattern` directive
+works as intended.
+
+It consists of 6 cases:
+
+1.  When not set, both `*_test.py` and `test_*.py` files are mapped to the `py_test`
+    rule.
+2.  When set to a single value `*_test.py`, `test_*.py` files are mapped to the
+    `py_library` rule.
+3.  When set to a single value `test_*.py`, `*_test.py` files are mapped to the
+    `py_library` rule (ie: the inverse of case 2, but also with "file" generation
+    mode).
+4.  Arbitrary `glob` patterns are supported.
+5.  Multiple `glob` patterns are supported and that patterns don't technically
+    need to end in `.py` if they end in a wildcard (eg: we won't make a `py_test`
+    target for the extensionless file `test_foo`).
+6.  Sub-packages can override the directive's value.

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_generation_mode file

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.out

@@ -0,0 +1,18 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+# gazelle:python_generation_mode file
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test2_star_test_py/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_test_file_pattern *_test.py

gazelle/python/testdata/directive_python_test_file_pattern/test2_star_test_py/BUILD.out

@@ -0,0 +1,17 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_test.py
+
+py_library(
+    name = "test2_star_test_py",
+    srcs = [
+        "test_goodbye.py",
+        "test_hello.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test3_test_star_py/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py

gazelle/python/testdata/directive_python_test_file_pattern/test3_test_star_py/BUILD.out

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py
+
+py_library(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test4_glob/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py

gazelle/python/testdata/directive_python_test_file_pattern/test4_glob/BUILD.out

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py
+
+py_library(
+    name = "foo_nota_test0_Z1",
+    srcs = ["foo_nota_test0_Z1.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_helloworld_A_testA",
+    srcs = ["foo_helloworld_A_testA.py"],
+)
+
+py_test(
+    name = "foo_my_filename_B_test1",
+    srcs = ["foo_my_filename_B_test1.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test5_multiple_patterns/BUILD.in

@@ -0,0 +1,3 @@
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.

gazelle/python/testdata/directive_python_test_file_pattern/test5_multiple_patterns/BUILD.out

@@ -0,0 +1,34 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.
+
+py_library(
+    name = "test5_multiple_patterns",
+    srcs = [
+        "mylib.py",
+        "mylib2.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_hello",
+    srcs = ["foo_hello.py"],
+)
+
+py_test(
+    name = "foo_unittest",
+    srcs = ["foo_unittest.py"],
+)
+
+py_test(
+    name = "hello_foo",
+    srcs = ["hello_foo.py"],
+)
+
+py_test(
+    name = "unittest_foo",
+    srcs = ["unittest_foo.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test6_nesting/BUILD.in

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern *_unittest.py