Skip to content

Commit

Permalink
Fix a non-determinism in create_embedded_tools.py.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 246804128
  • Loading branch information
philwo authored and copybara-github committed May 6, 2019
1 parent 60cd10b commit e67c961
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 22 deletions.
2 changes: 1 addition & 1 deletion scripts/bootstrap/compile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ EOF

# Set up @bazel_tools//platforms properly
mkdir -p ${BAZEL_TOOLS_REPO}/platforms
cp tools/platforms/platforms.BUILD ${BAZEL_TOOLS_REPO}/platforms/BUILD
cp tools/platforms/BUILD.tools ${BAZEL_TOOLS_REPO}/platforms/BUILD

# Overwrite tools.WORKSPACE, this is only for the bootstrap binary
chmod u+w "${OUTPUT_DIR}/classes/com/google/devtools/build/lib/bazel/rules/tools.WORKSPACE"
Expand Down
12 changes: 6 additions & 6 deletions src/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ genquery(
# Create dummy tools so we can do select to prevent building iOS target on
# Linux.
OSX_DUMMY_TARGETS = [
"src/tools/xcode/actoolwrapper/actoolwrapper",
"src/tools/xcode/ibtoolwrapper/ibtoolwrapper",
"src/tools/xcode/momcwrapper/momcwrapper",
"src/tools/xcode/actoolwrapper/actoolwrapper.sh",
"src/tools/xcode/ibtoolwrapper/ibtoolwrapper.sh",
"src/tools/xcode/momcwrapper/momcwrapper.sh",
"src/objc_tools/bundlemerge/bundlemerge_deploy.jar",
"src/objc_tools/plmerge/plmerge_deploy.jar",
"src/tools/xcode/realpath/realpath",
Expand All @@ -83,9 +83,9 @@ OSX_DUMMY_TARGETS = [
filegroup(
name = "darwin_tools",
srcs = [
"//src/tools/xcode/actoolwrapper:actoolwrapper",
"//src/tools/xcode/ibtoolwrapper:ibtoolwrapper",
"//src/tools/xcode/momcwrapper:momcwrapper",
"//src/tools/xcode/actoolwrapper:actoolwrapper.sh",
"//src/tools/xcode/ibtoolwrapper:ibtoolwrapper.sh",
"//src/tools/xcode/momcwrapper:momcwrapper.sh",
"//src/objc_tools/bundlemerge:bundlemerge_deploy.jar",
"//src/objc_tools/plmerge:plmerge_deploy.jar",
"//src/tools/xcode/realpath:realpath",
Expand Down
33 changes: 19 additions & 14 deletions src/create_embedded_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
('*tools/jdk/BUILD', lambda x: 'tools/jdk/BUILD'),
('*tools/build_defs/repo/BUILD.repo',
lambda x: 'tools/build_defs/repo/BUILD'),
('*tools/platforms/platforms.BUILD', lambda x: 'platforms/BUILD'),
('*tools/platforms/BUILD.tools', lambda x: 'platforms/BUILD'),
('*tools/platforms/*', lambda x: 'platforms/' + os.path.basename(x)),
('*tools/cpp/runfiles/generated_*',
lambda x: 'tools/cpp/runfiles/' + os.path.basename(x)[len('generated_'):]),
Expand All @@ -51,9 +51,7 @@
lambda x: 'tools/objc/make_hashed_objlist.py'),
('*xcode*realpath', lambda x: 'tools/objc/realpath'),
('*xcode*xcode-locator', lambda x: 'tools/objc/xcode-locator'),
('*src/tools/xcode/*.sh', lambda x: 'tools/objc/' + os.path.basename(x)),
('*src/tools/xcode/*',
lambda x: 'tools/objc/' + os.path.basename(x) + '.sh'),
('*src/tools/xcode/*', lambda x: 'tools/objc/' + os.path.basename(x)),
('*external/openjdk_*/file/*.tar.gz', lambda x: 'jdk.tar.gz'),
('*external/openjdk_*/file/*.zip', lambda x: 'jdk.zip'),
('*src/minimal_jdk.tar.gz', lambda x: 'jdk.tar.gz'),
Expand All @@ -70,15 +68,18 @@ def get_output_path(path):


def get_input_files(argsfile):
"""Returns a sorted list of tuples (archive_file, input_file).
"""Returns a dict of archive_file to input_file.
This describes the files that should be put into the generated archive.
Args:
argsfile: The file containing the list of input files.
Raises:
ValueError: When two input files map to the same output file.
"""
with open(argsfile, 'r') as f:
input_files = set(x.strip() for x in f.readlines())
input_files = sorted(set(x.strip() for x in f.readlines()))

result = {}
for input_file in input_files:
Expand All @@ -87,14 +88,16 @@ def get_input_files(argsfile):
input_file + '.tools' in input_files):
continue

# This gives us the same behavior as the older bash version of this
# tool: If two input files map to the same output files, the one that
# comes last in the list of input files overrides all earlier ones.
result[get_output_path(input_file)] = input_file
# It's an error to have two files map to the same output file, because the
# result is hard to predict and can easily be wrong.
output_path = get_output_path(input_file)
if output_path in result:
raise ValueError(
'Duplicate output file: Both {} and {} map to {}'.format(
result[output_path], input_file, output_path))
result[output_path] = input_file

# By sorting the file list, the resulting ZIP file will not be reproducible
# and deterministic.
return sorted(result.items())
return result


def copy_jdk_into_archive(output_zip, archive_file, input_file):
Expand Down Expand Up @@ -124,7 +127,9 @@ def main():
zipinfo.external_attr = 0o644 << 16
output_zip.writestr(zipinfo, 'workspace(name = "bazel_tools")\n')

for archive_file, input_file in input_files:
# By sorting the file list, the resulting ZIP file will be reproducible and
# deterministic.
for archive_file, input_file in sorted(input_files.items()):
if os.path.basename(archive_file) in ('jdk.tar.gz', 'jdk.zip'):
copy_jdk_into_archive(output_zip, archive_file, input_file)
else:
Expand Down
2 changes: 1 addition & 1 deletion tools/platforms/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ package(
filegroup(
name = "package-srcs",
srcs = [
"platforms.BUILD",
"BUILD.tools",
],
)

Expand Down
File renamed without changes.

0 comments on commit e67c961

Please sign in to comment.