From edfd648f686d5e971bb0067e3396fb2bcca470ea Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 6 Jan 2022 19:39:42 -0700 Subject: [PATCH] [wip] Merge `validate` goal with `lint` goal # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- pants.toml | 1 + .../pants/backend/project_info/regex_lint.py | 66 ++++++++++++++++++- src/python/pants/core/goals/lint.py | 52 +++++++++++---- 3 files changed, 103 insertions(+), 16 deletions(-) diff --git a/pants.toml b/pants.toml index 82bc965fde7f..3a7b0b3824aa 100644 --- a/pants.toml +++ b/pants.toml @@ -166,6 +166,7 @@ interpreter_constraints = [">=3.7,<3.10"] [regex-lint] config = "@build-support/regexes/config.yaml" +detail_level = "none" [generate-lockfiles] custom_command = "build-support/bin/generate_all_lockfiles.sh" diff --git a/src/python/pants/backend/project_info/regex_lint.py b/src/python/pants/backend/project_info/regex_lint.py index db5d44473319..94d85abb4938 100644 --- a/src/python/pants/backend/project_info/regex_lint.py +++ b/src/python/pants/backend/project_info/regex_lint.py @@ -12,13 +12,16 @@ from pants.base.deprecated import resolve_conflicting_options from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE +from pants.core.goals.lint import LintFilesRequest, LintResult, LintResults from pants.engine.collection import Collection from pants.engine.console import Console -from pants.engine.fs import Digest, DigestContents, SpecsSnapshot +from pants.engine.fs import Digest, DigestContents, PathGlobs, SpecsSnapshot from pants.engine.goal import Goal, GoalSubsystem -from pants.engine.rules import Get, collect_rules, goal_rule +from pants.engine.rules import Get, collect_rules, goal_rule, rule +from pants.engine.unions import UnionRule from pants.option.subsystem import Subsystem from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel from pants.util.memo import memoized_method logger = logging.getLogger(__name__) @@ -365,5 +368,62 @@ async def validate( return Validate(exit_code) +class RegexLintRequest(LintFilesRequest): + pass + + +@rule(desc="Lint with regex patterns", level=LogLevel.DEBUG) +async def validate_as_lint( + request: RegexLintRequest, + validate_subsystem: ValidateSubsystem, + regex_lint_subsystem: RegexLintSubsystem, +) -> LintResults: + multi_matcher = regex_lint_subsystem.get_multi_matcher() + if multi_matcher is None: + return LintResults((), linter_name="regex-lint") + + digest_contents = await Get(DigestContents, PathGlobs(request.file_paths)) + regex_match_results = RegexMatchResults( + multi_matcher.check_source_file(file_content.path, file_content.content) + for file_content in sorted(digest_contents, key=lambda fc: fc.path) + ) + + stdout = "" + detail_level = regex_lint_subsystem.detail_level(validate_subsystem) + num_matched_all = 0 + num_nonmatched_some = 0 + for rmr in regex_match_results: + if not rmr.matching and not rmr.nonmatching: + continue + if detail_level == DetailLevel.names: + if rmr.nonmatching: + stdout += f"{rmr.path}\n" + continue + + if rmr.nonmatching: + icon = "X" + num_nonmatched_some += 1 + else: + icon = "V" + num_matched_all += 1 + matched_msg = " Matched: {}".format(",".join(rmr.matching)) if rmr.matching else "" + nonmatched_msg = ( + " Didn't match: {}".format(",".join(rmr.nonmatching)) if rmr.nonmatching else "" + ) + if detail_level == DetailLevel.all or ( + detail_level == DetailLevel.nonmatching and nonmatched_msg + ): + stdout += f"{icon} {rmr.path}:{matched_msg}{nonmatched_msg}\n" + + if detail_level not in (DetailLevel.none, DetailLevel.names): + if stdout: + stdout += "\n" + stdout += f"{num_matched_all} files matched all required patterns.\n" + stdout += f"{num_nonmatched_some} files failed to match at least one required pattern." + + exit_code = PANTS_FAILED_EXIT_CODE if num_nonmatched_some else PANTS_SUCCEEDED_EXIT_CODE + return LintResults((LintResult(exit_code, stdout, ""),), linter_name="regex-lint") + + def rules(): - return collect_rules() + return (*collect_rules(), UnionRule(LintFilesRequest, RegexLintRequest)) diff --git a/src/python/pants/core/goals/lint.py b/src/python/pants/core/goals/lint.py index c06922e664d5..15352f17d3da 100644 --- a/src/python/pants/core/goals/lint.py +++ b/src/python/pants/core/goals/lint.py @@ -12,7 +12,7 @@ from pants.core.util_rules.distdir import DistDir from pants.engine.console import Console from pants.engine.engine_aware import EngineAwareReturnType -from pants.engine.fs import EMPTY_DIGEST, Digest, Workspace +from pants.engine.fs import EMPTY_DIGEST, Digest, SpecsSnapshot, Workspace from pants.engine.goal import Goal, GoalSubsystem from pants.engine.process import FallibleProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule @@ -133,6 +133,14 @@ class LintRequest(StyleRequest): """ +@union +@dataclass(frozen=True) +class LintFilesRequest: + """The entry point for linters that do not use targets.""" + + file_paths: tuple[str, ...] + + # If a user wants linter reports to show up in dist/ they must ensure that the reports # are written under this directory. E.g., # ./pants --flake8-args="--output-file=reports/report.txt" lint @@ -178,27 +186,39 @@ async def lint( console: Console, workspace: Workspace, targets: Targets, + specs_snapshot: SpecsSnapshot, lint_subsystem: LintSubsystem, union_membership: UnionMembership, dist_dir: DistDir, ) -> Lint: - request_types = cast("Iterable[type[StyleRequest]]", union_membership[LintRequest]) - requests = tuple( + target_requests = tuple( request_type( request_type.field_set_type.create(target) for target in targets if request_type.field_set_type.is_applicable(target) ) - for request_type in request_types + for request_type in union_membership[LintRequest] + ) + file_requests = tuple( + request_type(specs_snapshot.snapshot.files) + for request_type in union_membership[LintFilesRequest] ) if lint_subsystem.per_file_caching: - all_per_file_results = await MultiGet( - Get(LintResults, LintRequest, request.__class__([field_set])) - for request in requests - for field_set in request.field_sets - if request.field_sets - ) + all_requests = [ + *( + Get(LintResults, LintRequest, request.__class__([field_set])) + for request in target_requests + for field_set in request.field_sets + if request.field_sets + ), + *( + Get(LintResults, LintFilesRequest, request.__class__((fp,))) + for request in file_requests + for fp in request.file_paths + ), + ] + all_per_file_results = await MultiGet(all_requests) def key_fn(results: LintResults): return results.linter_name @@ -218,9 +238,15 @@ def key_fn(results: LintResults): ) ) else: - all_results = await MultiGet( - Get(LintResults, LintRequest, request) for request in requests if request.field_sets - ) + all_requests = [ + *( + Get(LintResults, LintRequest, request) + for request in target_requests + if request.field_sets + ), + *(Get(LintResults, LintFilesRequest, request) for request in file_requests), + ] + all_results = await MultiGet(all_requests) all_results = tuple(sorted(all_results, key=lambda results: results.linter_name))