Skip to content

Commit

Permalink
Introduce a new lock file format
Browse files Browse the repository at this point in the history 
This is designed to be more friendly when making updates,
particularly if the lock files are being reviewed as part
of a PR in GitHub.
  • Loading branch information
@shs96c
shs96c committed Jan 24, 2023
1 parent 5d266dd commit ce5de50
Show file tree
Hide file tree
Showing 42 changed files with 13,597 additions and 21,000 deletions.
2 changes: 2 additions & 0 deletions BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ bzl_library(
"//private/rules:maven_publish.bzl",
"//private/rules:maven_utils.bzl",
"//private/rules:pom_file.bzl",
"//private/rules:urls.bzl",
"//private/rules:v1_lock_file.bzl",
"//private/rules:v2_lock_file.bzl",
"//settings:stamp_manifest.bzl",
],
visibility = [
Expand Down
29 changes: 28 additions & 1 deletion WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ load("//:specs.bzl", "maven")

maven_install(
artifacts = [
"com.google.guava:guava:27.0-jre",
"com.google.guava:guava:31.1-jre",
"org.hamcrest:hamcrest-core:2.1",
],
maven_install_json = "@rules_jvm_external//:maven_install.json",
Expand Down Expand Up @@ -249,6 +249,16 @@ maven_install(
],
)

# Grab com.google.ar.sceneform:rendering because we overrode it above
http_file(
name = "com.google.ar.sceneform_rendering",
downloaded_file_path = "rendering-1.10.0.aar",
sha256 = "d2f6cd1d54eee0d5557518d1edcf77a3ba37494ae94f9bb862e570ee426a3431",
urls = [
"https://dl.google.com/android/maven2/com/google/ar/sceneform/rendering/1.10.0/rendering-1.10.0.aar",
],
)

maven_install(
name = "testonly_testing",
artifacts = [
Expand Down Expand Up @@ -542,6 +552,23 @@ maven_install(
],
)

maven_install(
name = "v1_lock_file_format",
artifacts = [
# Coordinates that are in no other `maven_install`
"io.ous:jtoml:2.0.0",
],
generate_compat_repositories = True,
maven_install_json = "//tests/custom_maven_install:v1_lock_file_format_install.json",
repositories = [
"https://repo1.maven.org/maven2",
],
)

load("@v1_lock_file_format//:defs.bzl", v1_lock_file_format_pinned_maven_install = "pinned_maven_install")

v1_lock_file_format_pinned_maven_install()

http_file(
name = "hamcrest_core_for_test",
downloaded_file_path = "hamcrest-core-1.3.jar",
Expand Down
208 changes: 51 additions & 157 deletions coursier.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
# See the License for the specific language governing permissions and

load("//private/rules:jetifier.bzl", "jetify_artifact_dependencies", "jetify_maven_coord")
load("//private/rules:v1_lock_file.bzl", "create_dependency", "v1_lock_file")
load("//private/rules:urls.bzl", "remove_auth_from_url")
load("//private/rules:v1_lock_file.bzl", "v1_lock_file")
load("//private/rules:v2_lock_file.bzl", "v2_lock_file")
load("//:specs.bzl", "parse", "utils")
load("//private:artifact_utilities.bzl", "deduplicate_and_sort_artifacts")
load("//private:coursier_utilities.bzl", "SUPPORTED_PACKAGING_TYPES", "escape", "is_maven_local_path")
Expand All @@ -24,7 +26,6 @@ load(
"COURSIER_CLI_BAZEL_MIRROR_URL",
"COURSIER_CLI_GITHUB_ASSET_URL",
"COURSIER_CLI_SHA256",
"JQ_VERSIONS",
)

_BUILD = """
Expand Down Expand Up @@ -55,27 +56,13 @@ load("%s", "aar_import")
"""

_BUILD_PIN = """
genrule(
name = "jq-binary",
cmd = "cp $< $@",
outs = ["jq"],
srcs = select({{
"@bazel_tools//src/conditions:linux_aarch64": ["jq-linux"],
"@bazel_tools//src/conditions:linux_x86_64": ["jq-linux"],
"@bazel_tools//src/conditions:darwin": ["jq-macos"],
"@bazel_tools//src/conditions:windows": ["jq-windows"],
}}),
)
sh_binary(
name = "pin",
srcs = ["pin.sh"],
args = [
"$(rootpath :jq-binary)",
"$(location :unsorted_deps.json)",
],
data = [
":jq-binary",
":unsorted_deps.json",
],
visibility = ["//visibility:public"],
Expand Down Expand Up @@ -289,27 +276,6 @@ def _windows_check(repository_ctx):
"This is typically `c:\\msys64\\usr\\bin\\bash.exe`. For more information, read " +
"https://docs.bazel.build/versions/master/install-windows.html#getting-bazel")

# Deterministically compute a signature of a list of artifacts in the dependency tree.
# This is to prevent users from manually editing maven_install.json.
def _compute_dependency_tree_signature(artifacts):
# A collection of elements from the dependency tree to be sorted and hashed
# into a signature for maven_install.json.
signature_inputs = []
for artifact in artifacts:
artifact_group = []
artifact_group.append(artifact["coord"])
if artifact["file"] != None:
artifact_group.extend([
artifact["sha256"],
_normalize_to_unix_path(artifact["file"]), # Make sure we represent files in a stable way cross-platform
])
if artifact["url"]:
artifact_group.append(artifact["url"])
if len(artifact["dependencies"]) > 0:
artifact_group.append(",".join(sorted(artifact["dependencies"])))
signature_inputs.append(":".join(artifact_group))
return hash(repr(sorted(signature_inputs)))

# Compute a signature of the list of artifacts that will be used to build
# the dependency tree. This is used as a check to see whether the dependency
# tree needs to be repinned.
Expand All @@ -327,65 +293,6 @@ def compute_dependency_inputs_signature(artifacts, repositories):
artifact_inputs.append(flattened)
return hash(repr(sorted(artifact_inputs))) ^ hash(repr(sorted(repositories)))

def extract_netrc_from_auth_url(url):
"""Return a dict showing the netrc machine, login, and password extracted from a url.
Returns:
A dict that is empty if there were no credentials in the url.
A dict that has three keys -- machine, login, password -- with their respective values. These values should be
what is needed for the netrc entry of the same name except for password whose value may be empty meaning that
there is no password for that login.
"""
if "@" not in url:
return {}
protocol, url_parts = split_url(url)
login_password_host = url_parts[0]
if "@" not in login_password_host:
return {}
login_password, host = login_password_host.rsplit("@", 1)
login_password_split = login_password.split(":", 1)
login = login_password_split[0]

# If password is not provided, then this will be a 1-length split
if len(login_password_split) < 2:
password = None
else:
password = login_password_split[1]
if not host:
fail("Got a blank host from: {}".format(url))
if not login:
fail("Got a blank login from: {}".format(url))

# Do not fail for blank password since that is sometimes a thing
return {
"machine": host,
"login": login,
"password": password,
}

def add_netrc_entries_from_mirror_urls(netrc_entries, mirror_urls):
"""Add a url's auth credentials into a netrc dict of form return[machine][login] = password."""
for url in mirror_urls:
entry = extract_netrc_from_auth_url(url)
if not entry:
continue
machine = entry["machine"]
login = entry["login"]
password = entry["password"]
if machine not in netrc_entries:
netrc_entries[machine] = {}
if login not in netrc_entries[machine]:
if netrc_entries[machine]:
print("Received multiple logins for machine '{}'! Only using '{}'".format(
machine,
netrc_entries[machine].keys()[0],
))
continue
netrc_entries[machine][login] = password
elif netrc_entries[machine][login] != password:
print("Received different passwords for {}@{}! Only using the first".format(login, machine))
return netrc_entries

def get_netrc_lines_from_entries(netrc_entries):
netrc_lines = []
for machine, login_dict in sorted(netrc_entries.items()):
Expand All @@ -406,22 +313,6 @@ def get_home_netrc_contents(repository_ctx):
return repository_ctx.read(netrcfile)
return ""

def _get_jq_http_files():
"""Returns repository targets for the `jq` dependency that `pin.sh` needs."""
lines = []
for jq in JQ_VERSIONS:
lines.extend([
" maybe(",
" http_file,",
" name = \"rules_jvm_external_jq_%s\"," % jq,
" urls = %s," % repr([JQ_VERSIONS[jq].url]),
" sha256 = %s," % repr(JQ_VERSIONS[jq].sha256),
" downloaded_file_path = \"jq\",",
" executable = True,",
" )",
])
return lines

def _add_outdated_files(repository_ctx, artifacts, repositories):
repository_ctx.file(
"outdated.artifacts",
Expand Down Expand Up @@ -476,7 +367,14 @@ def _pinned_coursier_fetch_impl(repository_ctx):
)
maven_install_json_content = json.decode(repository_ctx.read(repository_ctx.attr.maven_install_json))

importer = v1_lock_file
if v1_lock_file.is_valid_lock_file(maven_install_json_content):
importer = v1_lock_file
print("Lock file should be updated. Please run `REPIN=1 bazel run @unpinned_%s//:pin`" % repository_ctx.name)
elif v2_lock_file.is_valid_lock_file(maven_install_json_content):
importer = v2_lock_file
else:
fail("Unable to read lock file: %s" % repository_ctx.attr.maven_install_json)

# Validation steps for maven_install.json.

# Validate that there's a dependency_tree element in the parsed JSON.
Expand Down Expand Up @@ -518,7 +416,11 @@ def _pinned_coursier_fetch_impl(repository_ctx):
elif importer.compute_lock_file_hash(maven_install_json_content) != dep_tree_signature:
# Then, validate that the signature provided matches the contents of the dependency_tree.
# This is to stop users from manually modifying maven_install.json.
fail("%s_install.json contains an invalid signature and may be corrupted. " % repository_ctx.name +
fail("%s_install.json contains an invalid signature (expected %s and got %s) and may be corrupted. " % (
repository_ctx.name,
dep_tree_signature,
importer.compute_lock_file_hash(maven_install_json_content),
) +
"PLEASE DO NOT MODIFY THIS FILE DIRECTLY! To generate a new " +
"%s_install.json and re-pin the artifacts, follow these steps: \n\n" % repository_ctx.name +
" 1) In your WORKSPACE file, comment or remove the 'maven_install_json' attribute in 'maven_install'.\n" +
Expand Down Expand Up @@ -556,8 +458,6 @@ def _pinned_coursier_fetch_impl(repository_ctx):
http_files.append(" downloaded_file_path = \"%s\"," % artifact["file"])
http_files.append(" )")

http_files.extend(_get_jq_http_files())

http_files.extend(["maven_artifacts = [\n%s\n]" % (",\n".join([" \"%s\"" % artifact for artifact in maven_artifacts]))])

repository_ctx.file("defs.bzl", "\n".join(http_files), executable = False)
Expand Down Expand Up @@ -630,25 +530,6 @@ def _pinned_coursier_fetch_impl(repository_ctx):
executable = False,
)

def split_url(url):
protocol = url[:url.find("://")]
url_without_protocol = url[url.find("://") + 3:]
url_parts = url_without_protocol.split("/")
return protocol, url_parts

def remove_auth_from_url(url):
"""Returns url without `user:pass@` or `user@`."""
if "@" not in url:
return url
protocol, url_parts = split_url(url)
host = url_parts[0]
if "@" not in host:
return url
last_index = host.rfind("@", 0, None)
userless_host = host[last_index + 1:]
new_url = "{}://{}".format(protocol, "/".join([userless_host] + url_parts[1:]))
return new_url

def infer_artifact_path_from_primary_and_repos(primary_url, repository_urls):
"""Returns the artifact path inferred by comparing primary_url with urls in repository_urls.
Expand Down Expand Up @@ -859,12 +740,6 @@ def make_coursier_dep_tree(
_is_verbose(repository_ctx),
)

def _download_jq(repository_ctx):
jq_version = None

for (os, value) in JQ_VERSIONS.items():
repository_ctx.download(value.url, "jq-%s" % os, sha256 = value.sha256, executable = True)

def remove_prefix(s, prefix):
if s.startswith(prefix):
return s[len(prefix):]
Expand All @@ -886,7 +761,6 @@ def _coursier_fetch_impl(repository_ctx):
coursier_download_urls.append(coursier_url_from_env)

repository_ctx.download(coursier_download_urls, "coursier", sha256 = COURSIER_CLI_SHA256, executable = True)
_download_jq(repository_ctx)

# Try running coursier once
cmd = _generate_java_jar_command(repository_ctx, repository_ctx.path("coursier"))
Expand Down Expand Up @@ -1039,7 +913,7 @@ def _coursier_fetch_impl(repository_ctx):

# Coursier prepends the username from the provided credentials if needed to authenticate
# with the repository. We remove it from the url and file attributes if only the username is present
# and no password, as it has noe function and obfuscates changes to the pinned json
# and no password, as it has no function and obfuscates changes to the pinned json
credential_marker = primary_url.find("@")
if credential_marker > -1:
potential_credentials = remove_prefix(primary_url[:credential_marker + 1], protocol + "://")
Expand Down Expand Up @@ -1120,16 +994,42 @@ def _coursier_fetch_impl(repository_ctx):
artifact.update({"sha256": shas[path]})
artifact.update({"packages": jars_to_packages[path]})

dep_tree.update({
"__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
"__RESOLVED_ARTIFACTS_HASH": _compute_dependency_tree_signature(dep_tree["dependencies"]),
"__INPUT_ARTIFACTS_HASH": compute_dependency_inputs_signature(repository_ctx.attr.artifacts, repository_ctx.attr.repositories),
})
# Keep the original output from coursier for debugging
repository_ctx.file(
"coursier-deps.json",
content = json.encode_indent(dep_tree),
)
reformat_lock_file_cmd = _generate_java_jar_command(
repository_ctx,
repository_ctx.path(repository_ctx.attr._lock_file_converter),
)
for repo in repositories:
reformat_lock_file_cmd.extend(["--repo", repo["repo_url"]])
reformat_lock_file_cmd.extend(["--json", "coursier-deps.json"])

# But update the format to the latest lock file
result = _execute(
repository_ctx,
cmd = reformat_lock_file_cmd,
progress_message = "Updating lock file format",
)
if result.return_code:
fail("Unable to generate lock file: " + result.stderr)

lock_file_contents = json.decode(result.stdout)

repository_ctx.file(
"unsorted_deps.json",
content = v2_lock_file.render_lock_file(
lock_file_contents,
compute_dependency_inputs_signature(repository_ctx.attr.artifacts, repository_ctx.attr.repositories),
),
)

repository_ctx.report_progress("Generating BUILD targets..")
(generated_imports, jar_versionless_target_labels) = parser.generate_imports(
repository_ctx = repository_ctx,
dependencies = [create_dependency(d) for d in dep_tree["dependencies"]],
dependencies = v2_lock_file.get_artifacts(lock_file_contents),
explicit_artifacts = {
a["group"] + ":" + a["artifact"] + (":" + a["classifier"] if "classifier" in a else ""): True
for a in artifacts
Expand Down Expand Up @@ -1196,13 +1096,6 @@ def _coursier_fetch_impl(repository_ctx):
# $ bazel run @unpinned_maven//:pin
#
# Create the maven_install.json export script for unpinned repositories.
dependency_tree_json = "{ \"dependency_tree\": " + repr(dep_tree).replace("None", "null") + "}"
repository_ctx.file(
"unsorted_deps.json",
content = "{dependency_tree_json}".format(
dependency_tree_json = dependency_tree_json,
),
)
repository_ctx.template(
"pin.sh",
repository_ctx.attr._pin,
Expand All @@ -1219,7 +1112,7 @@ def _coursier_fetch_impl(repository_ctx):
"load(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_file\")",
"load(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")",
"def pinned_maven_install():",
] + _get_jq_http_files()
]
repository_ctx.file(
"defs.bzl",
"\n".join(http_files),
Expand Down Expand Up @@ -1299,6 +1192,7 @@ coursier_fetch = repository_rule(
attrs = {
"_sha256_hasher": attr.label(default = "//private/tools/prebuilt:hasher_deploy.jar"),
"_list_packages": attr.label(default = "//private/tools/prebuilt:list_packages_deploy.jar"),
"_lock_file_converter": attr.label(default = "//private/tools/prebuilt:lock_file_converter_deploy.jar"),
"_pin": attr.label(default = "//private:pin.sh"),
"_compat_repository": attr.label(default = "//private:compat_repository.bzl"),
"_outdated": attr.label(default = "//private:outdated.sh"),
Expand Down
Loading

0 comments on commit ce5de50

Please sign in to comment.