Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(gazelle): Add directives for label format & normalisation #1976

Merged
merged 14 commits into from
Jun 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ A brief description of the categories of changes:
* (bzlmod) Correctly pass `isolated`, `quiet` and `timeout` values to `whl_library`
and drop the defaults from the lock file.

### Added
* (gazelle) Added new `python_label_convention` and `python_label_normalization` directives. These directive
allows altering default Gazelle label format to third-party dependencies useful for re-using Gazelle plugin
with other rules, including `rules_pycross`. See [#1939](https://github.com/bazelbuild/rules_python/issues/1939).

### Removed
* (pip): Removes the `entrypoint` macro that was replaced by `py_console_script_binary` in 0.26.0.

Expand Down
5 changes: 4 additions & 1 deletion gazelle/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,10 @@ Python-specific directives are as follows:
| Appends additional visibility labels to each generated target. This directive can be set multiple times. | |
| [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` |
| Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. |

| `# gazelle:python_label_convention` | `$distribution_name$` |
| Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. |
| `# gazelle:python_label_normalization` | `snake_case` |
| Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". |

#### Directive: `python_root`:

Expand Down
19 changes: 19 additions & 0 deletions gazelle/python/configure.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ func (py *Configurer) KnownDirectives() []string {
pythonconfig.DefaultVisibilty,
pythonconfig.Visibility,
pythonconfig.TestFilePattern,
pythonconfig.LabelConvention,
pythonconfig.LabelNormalization,
}
}

Expand Down Expand Up @@ -196,6 +198,23 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
}
}
config.SetTestFilePattern(globStrings)
case pythonconfig.LabelConvention:
value := strings.TrimSpace(d.Value)
if value == "" {
log.Fatalf("directive '%s' requires a value", pythonconfig.LabelConvention)
}
config.SetLabelConvention(value)
case pythonconfig.LabelNormalization:
switch directiveArg := strings.ToLower(strings.TrimSpace(d.Value)); directiveArg {
case "pep503":
config.SetLabelNormalization(pythonconfig.Pep503LabelNormalizationType)
case "none":
config.SetLabelNormalization(pythonconfig.NoLabelNormalizationType)
case "snake_case":
config.SetLabelNormalization(pythonconfig.SnakeCaseLabelNormalizationType)
default:
config.SetLabelNormalization(pythonconfig.DefaultLabelNormalizationType)
}
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import module1
import foo # third party package
import module1

# gazelle:include_dep //foo/bar:baz
# gazelle:include_dep //hello:world,@star_wars//rebel_alliance/luke:skywalker
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Directive: `python_label_convention`

This test case asserts that the `# gazelle:python_label_convention` directive
works as intended when set.
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
load("@rules_python//python:defs.bzl", "py_library")

py_library(
name = "test1_unset",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = [
"@gazelle_python_test//google_cloud_aiplatform",
"@gazelle_python_test//google_cloud_storage",
],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from google.cloud import aiplatform, storage


def main():
a = dir(aiplatform)
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
wingsofovnia marked this conversation as resolved.
Show resolved Hide resolved
modules_mapping:
google.cloud.aiplatform: google_cloud_aiplatform
google.cloud.storage: google_cloud_storage
pip_repository:
name: gazelle_python_test
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# gazelle:python_label_convention :$distribution_name$
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
load("@rules_python//python:defs.bzl", "py_library")

# gazelle:python_label_convention :$distribution_name$

py_library(
name = "test2_custom_prefix_colon",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = [
"@gazelle_python_test//:google_cloud_aiplatform",
"@gazelle_python_test//:google_cloud_storage",
],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from google.cloud import aiplatform, storage


def main():
a = dir(aiplatform)
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
modules_mapping:
google.cloud.aiplatform: google_cloud_aiplatform
google.cloud.storage: google_cloud_storage
pip_repository:
name: gazelle_python_test
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Directive: `python_label_normalization`

This test case asserts that the `# gazelle:python_label_normalization` directive
works as intended when set.
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# gazelle:python_label_normalization none
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
load("@rules_python//python:defs.bzl", "py_library")

# gazelle:python_label_normalization none

py_library(
name = "test1_type_none",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = ["@gazelle_python_test//google.cloud.storage"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from google.cloud import storage


def main():
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
modules_mapping:
# Weird google.cloud.storage here on purpose to make normalization apparent
google.cloud.storage: google.cloud.storage
pip_repository:
name: gazelle_python_test
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# gazelle:python_label_normalization pep503
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
load("@rules_python//python:defs.bzl", "py_library")

# gazelle:python_label_normalization pep503

py_library(
name = "test2_type_pep503",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = ["@gazelle_python_test//google-cloud-storage"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from google.cloud import storage


def main():
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
modules_mapping:
# Weird google.cloud.storage here on purpose to make normalization apparent
google.cloud.storage: google.cloud.storage
pip_repository:
name: gazelle_python_test
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# gazelle:python_label_normalization snake_case
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
load("@rules_python//python:defs.bzl", "py_library")

# gazelle:python_label_normalization snake_case

py_library(
name = "test3_type_snake_case",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = ["@gazelle_python_test//google_cloud_storage"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from google.cloud import storage


def main():
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
modules_mapping:
# Weird google.cloud.storage here on purpose to make normalization apparent
google.cloud.storage: google.cloud.storage
pip_repository:
name: gazelle_python_test
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
load("@rules_python//python:defs.bzl", "py_library")

py_library(
name = "test4_unset_defaults_to_snake_case",
srcs = ["bar.py"],
visibility = ["//:__subpackages__"],
deps = ["@gazelle_python_test//google_cloud_storage"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from google.cloud import storage


def main():
b = dir(storage)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
manifest:
modules_mapping:
# Weird google.cloud.storage here on purpose to make normalization apparent
google.cloud.storage: google.cloud.storage
pip_repository:
name: gazelle_python_test
86 changes: 74 additions & 12 deletions gazelle/pythonconfig/pythonconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package pythonconfig
import (
"fmt"
"path"
"regexp"
"strings"

"github.com/emirpasic/gods/lists/singlylinkedlist"
Expand Down Expand Up @@ -77,6 +78,13 @@ const (
// TestFilePattern represents the directive that controls which python
// files are mapped to `py_test` targets.
TestFilePattern = "python_test_file_pattern"
// LabelConvention represents the directive that defines the format of the
// labels to third-party dependencies.
LabelConvention = "python_label_convention"
// LabelNormalization represents the directive that controls how distribution
// names of labels to third-party dependencies are normalized. Supported values
// are 'none', 'pep503' and 'snake_case' (default). See LabelNormalizationType.
LabelNormalization = "python_label_normalization"
)

// GenerationModeType represents one of the generation modes for the Python
Expand All @@ -96,14 +104,19 @@ const (
)

const (
packageNameNamingConventionSubstitution = "$package_name$"
packageNameNamingConventionSubstitution = "$package_name$"
distributionNameLabelConventionSubstitution = "$distribution_name$"
)

const (
// The default visibility label, including a format placeholder for `python_root`.
DefaultVisibilityFmtString = "//%s:__subpackages__"
// The default globs used to determine pt_test targets.
DefaultTestFilePatternString = "*_test.py,test_*.py"
// The default convention of label of third-party dependencies.
DefaultLabelConvention = "$distribution_name$"
// The default normalization applied to distribution names of third-party dependency labels.
DefaultLabelNormalizationType = SnakeCaseLabelNormalizationType
)

// defaultIgnoreFiles is the list of default values used in the
Expand All @@ -112,14 +125,6 @@ var defaultIgnoreFiles = map[string]struct{}{
"setup.py": {},
}

func SanitizeDistribution(distributionName string) string {
sanitizedDistribution := strings.ToLower(distributionName)
sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_")
sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, ".", "_")

return sanitizedDistribution
}

// Configs is an extension of map[string]*Config. It provides finding methods
// on top of the mapping.
type Configs map[string]*Config
Expand Down Expand Up @@ -156,8 +161,18 @@ type Config struct {
defaultVisibility []string
visibility []string
testFilePattern []string
labelConvention string
labelNormalization LabelNormalizationType
}

type LabelNormalizationType int

const (
NoLabelNormalizationType LabelNormalizationType = iota
Pep503LabelNormalizationType
SnakeCaseLabelNormalizationType
)

// New creates a new Config.
func New(
repoRoot string,
Expand All @@ -180,6 +195,8 @@ func New(
defaultVisibility: []string{fmt.Sprintf(DefaultVisibilityFmtString, "")},
visibility: []string{},
testFilePattern: strings.Split(DefaultTestFilePatternString, ","),
labelConvention: DefaultLabelConvention,
labelNormalization: DefaultLabelNormalizationType,
}
}

Expand Down Expand Up @@ -209,6 +226,8 @@ func (c *Config) NewChild() *Config {
defaultVisibility: c.defaultVisibility,
visibility: c.visibility,
testFilePattern: c.testFilePattern,
labelConvention: c.labelConvention,
labelNormalization: c.labelNormalization,
}
}

Expand Down Expand Up @@ -263,10 +282,8 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) {
} else if gazelleManifest.PipRepository != nil {
distributionRepositoryName = gazelleManifest.PipRepository.Name
}
sanitizedDistribution := SanitizeDistribution(distributionName)

// @<repository_name>//<distribution_name>
lbl := label.New(distributionRepositoryName, sanitizedDistribution, sanitizedDistribution)
lbl := currentCfg.FormatThirdPartyDependency(distributionRepositoryName, distributionName)
return lbl.String(), true
}
}
Expand Down Expand Up @@ -443,3 +460,48 @@ func (c *Config) SetTestFilePattern(patterns []string) {
func (c *Config) TestFilePattern() []string {
return c.testFilePattern
}

// SetLabelConvention sets the label convention used for third-party dependencies.
func (c *Config) SetLabelConvention(convention string) {
c.labelConvention = convention
}

// LabelConvention returns the label convention used for third-party dependencies.
func (c *Config) LabelConvention() string {
return c.labelConvention
}

// SetLabelConvention sets the label normalization applied to distribution names of third-party dependencies.
func (c *Config) SetLabelNormalization(normalizationType LabelNormalizationType) {
c.labelNormalization = normalizationType
}

// LabelConvention returns the label normalization applied to distribution names of third-party dependencies.
func (c *Config) LabelNormalization() LabelNormalizationType {
return c.labelNormalization
}

// FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization.
func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label {
conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName)

var normConventionalDistributionName string
switch norm := c.LabelNormalization(); norm {
case SnakeCaseLabelNormalizationType:
// See /python/private/normalize_name.bzl
normConventionalDistributionName = strings.ToLower(conventionalDistributionName)
normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "_")
normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "_")
case Pep503LabelNormalizationType:
wingsofovnia marked this conversation as resolved.
Show resolved Hide resolved
// See https://packaging.python.org/en/latest/specifications/name-normalization/#name-format
normConventionalDistributionName = strings.ToLower(conventionalDistributionName) // ... "should be lowercased"
normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "-") // ... "all runs of the characters ., -, or _ replaced with a single -"
normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "-") // ... "must start and end with a letter or number"
default:
fallthrough
case NoLabelNormalizationType:
normConventionalDistributionName = conventionalDistributionName
}

return label.New(repositoryName, normConventionalDistributionName, normConventionalDistributionName)
}
Loading