From 41eb75cb3ca60f9f5f37179ac0c0098bd3c12541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Csomor?= <4322804+ngiloq6@users.noreply.github.com> Date: Mon, 18 Mar 2019 13:23:15 +0100 Subject: [PATCH] copy_file: add rule and tests (#123) This PR adds two new rules: copy_file and copy_xfile. Both rules solve a common problem: to copy one file to another location. The problem is routinely solved using a genrule. That however requires Bash, since genrules execute Bash commands. Requiring Bash is a problem on Windows. The new rules do not require Bash on Windows (only on other platforms). The only difference between the rules is that copy_xfile creates an executable file while copy_file doesn't. See https://github.com/bazelbuild/bazel/issues/4319 --- docs/BUILD | 7 ++ rules/BUILD | 12 +++ rules/copy_file.bzl | 29 +++++++ rules/copy_file_private.bzl | 130 +++++++++++++++++++++++++++++ tests/BUILD | 5 ++ tests/copy_file/BUILD | 119 ++++++++++++++++++++++++++ tests/copy_file/a.txt | 2 + tests/copy_file/copy_file_tests.sh | 63 ++++++++++++++ 8 files changed, 367 insertions(+) create mode 100644 rules/copy_file.bzl create mode 100644 rules/copy_file_private.bzl create mode 100644 tests/copy_file/BUILD create mode 100644 tests/copy_file/a.txt create mode 100755 tests/copy_file/copy_file_tests.sh diff --git a/docs/BUILD b/docs/BUILD index 3bf6a3c..76486aa 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -92,6 +92,13 @@ stardoc( deps = ["//lib:versions"], ) +stardoc( + name = "copy_file_docs", + out = "copy_file_doc_gen.md", + input = "//rules:copy_file.bzl", + deps = ["//rules:copy_file"], +) + stardoc( name = "maprule_docs", out = "maprule_doc_gen.md", diff --git a/rules/BUILD b/rules/BUILD index c5340f9..9509985 100644 --- a/rules/BUILD +++ b/rules/BUILD @@ -10,6 +10,18 @@ bzl_library( deps = ["//lib:new_sets"], ) +bzl_library( + name = "copy_file", + srcs = ["copy_file.bzl"], + deps = [":copy_file_private"], +) + +bzl_library( + name = "copy_file_private", + srcs = ["copy_file_private.bzl"], + visibility = ["//visibility:private"], +) + bzl_library( name = "maprule", srcs = ["maprule.bzl"], diff --git a/rules/copy_file.bzl b/rules/copy_file.bzl new file mode 100644 index 0000000..e985b3f --- /dev/null +++ b/rules/copy_file.bzl @@ -0,0 +1,29 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A rule that copies a file to another place. + +native.genrule() is sometimes used to copy files (often wishing to rename them). +The 'copy_file' rule does this with a simpler interface than genrule. + +The rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command +on Windows (no Bash is required). +""" + +load( + ":copy_file_private.bzl", + _copy_file = "copy_file", +) + +copy_file = _copy_file diff --git a/rules/copy_file_private.bzl b/rules/copy_file_private.bzl new file mode 100644 index 0000000..4c08d40 --- /dev/null +++ b/rules/copy_file_private.bzl @@ -0,0 +1,130 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of copy_file macro and underlying rules. + +These rules copy a file to another location using Bash (on Linux/macOS) or +cmd.exe (on Windows). '_copy_xfile' marks the resulting file executable, +'_copy_file' does not. +""" + +def _common_impl(ctx, is_executable): + if ctx.attr.is_windows: + # Most Windows binaries built with MSVC use a certain argument quoting + # scheme. Bazel uses that scheme too to quote arguments. However, + # cmd.exe uses different semantics, so Bazel's quoting is wrong here. + # To fix that we write the command to a .bat file so no command line + # quoting or escaping is required. + bat = ctx.actions.declare_file(ctx.label.name + "-cmd.bat") + ctx.actions.write( + output = bat, + # Do not use lib/shell.bzl's shell.quote() method, because that uses + # Bash quoting syntax, which is different from cmd.exe's syntax. + content = "@copy /Y \"%s\" \"%s\" >NUL" % ( + ctx.file.src.path.replace("/", "\\"), + ctx.outputs.out.path.replace("/", "\\"), + ), + is_executable = True, + ) + ctx.actions.run( + inputs = [ctx.file.src, bat], + outputs = [ctx.outputs.out], + executable = "cmd.exe", + arguments = ["/C", bat.path.replace("/", "\\")], + mnemonic = "CopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + else: + ctx.actions.run_shell( + inputs = [ctx.file.src], + outputs = [ctx.outputs.out], + command = "cp -f \"$1\" \"$2\"", + arguments = [ctx.file.src.path, ctx.outputs.out.path], + mnemonic = "CopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + + files = depset(direct = [ctx.outputs.out]) + runfiles = ctx.runfiles(files = [ctx.outputs.out]) + if is_executable: + return [DefaultInfo(files = files, runfiles = runfiles, executable = ctx.outputs.out)] + else: + return [DefaultInfo(files = files, runfiles = runfiles)] + +def _impl(ctx): + return _common_impl(ctx, False) + +def _ximpl(ctx): + return _common_impl(ctx, True) + +_ATTRS = { + "src": attr.label(mandatory = True, allow_single_file = True), + "out": attr.output(mandatory = True), + "is_windows": attr.bool(mandatory = True), +} + +_copy_file = rule( + implementation = _impl, + provides = [DefaultInfo], + attrs = _ATTRS, +) + +_copy_xfile = rule( + implementation = _ximpl, + executable = True, + provides = [DefaultInfo], + attrs = _ATTRS, +) + +def copy_file(name, src, out, is_executable = False, **kwargs): + """Copies a file to another location. + + `native.genrule()` is sometimes used to copy files (often wishing to rename them). The 'copy_file' rule does this with a simpler interface than genrule. + + This rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command on Windows (no Bash is required). + + Args: + name: Name of the rule. + src: A Label. The file to make a copy of. (Can also be the label of a rule + that generates a file.) + out: Path of the output file, relative to this package. + is_executable: A boolean. Whether to make the output file executable. When + True, the rule's output can be executed using `bazel run` and can be + in the srcs of binary and test rules that require executable sources. + **kwargs: further keyword arguments, e.g. `visibility` + """ + if is_executable: + _copy_xfile( + name = name, + src = src, + out = out, + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }), + **kwargs + ) + else: + _copy_file( + name = name, + src = src, + out = out, + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }), + **kwargs + ) diff --git a/tests/BUILD b/tests/BUILD index e4b67aa..13b496d 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -16,6 +16,11 @@ load(":versions_tests.bzl", "versions_test_suite") licenses(["notice"]) +exports_files( + ["unittest.bash"], + visibility = ["//tests:__subpackages__"], +) + build_test_test_suite() collections_test_suite() diff --git a/tests/copy_file/BUILD b/tests/copy_file/BUILD new file mode 100644 index 0000000..47d7887 --- /dev/null +++ b/tests/copy_file/BUILD @@ -0,0 +1,119 @@ +# This package aids testing the 'copy_file' rule. +# +# The package contains 4 copy_file rules: +# - 'copy_src' and 'copy_gen' copy a source file and a generated file +# respectively +# - 'copy_xsrc' and 'copy_xgen' copy a source file and a generated file +# respectively (both are shell scripts), and mark their output as executable +# +# The generated file is the output of the 'gen' genrule. +# +# The 'bin_src' and 'bin_gen' rules are sh_binary rules. They use the +# 'copy_xsrc' and 'copy_xgen' rules respectively. The sh_binary rule requires +# its source to be executable, so building these two rules successfully means +# that 'copy_file' managed to make its output executable. +# +# The 'run_executables' genrule runs the 'bin_src' and 'bin_gen' binaries, +# partly to ensure they can be run, and partly so we can observe their output +# and assert the contents in the 'copy_file_tests' test. +# +# The 'file_deps' filegroup depends on 'copy_src'. The filegroup rule uses the +# DefaultInfo.files field from its dependencies. When we data-depend on the +# filegroup from 'copy_file_tests', we transitively data-depend on the +# DefaultInfo.files of the 'copy_src' rule. +# +# The 'copy_file_tests' test is the actual integration test. It data-depends +# on: +# - the 'run_executables' rule, to get the outputs of 'bin_src' and 'bin_gen' +# - the 'file_deps' rule, and by nature of using a filegroup, we get the files +# from the DefaultInfo.files of the 'copy_file' rule, and thereby assert that +# that field contains the output file of the rule +# - the 'copy_nonempty_text' rule, and thereby on the DefaultInfo.runfiles field +# of it, so we assert that that field contains the output file of the rule + +load("//rules:copy_file.bzl", "copy_file") + +package(default_testonly = 1) + +sh_test( + name = "copy_file_tests", + srcs = ["copy_file_tests.sh"], + data = [ + ":run_executables", + # Use DefaultInfo.files from 'copy_src' (via 'file_deps'). + ":file_deps", + # Use DefaultInfo.runfiles from 'copy_gen'. + ":copy_gen", + "//tests:unittest.bash", + ], + deps = ["@bazel_tools//tools/bash/runfiles"], +) + +filegroup( + name = "file_deps", + # Use DefaultInfo.files from 'copy_src'. + srcs = [ + ":copy_src", + ], +) + +# If 'run_executables' is built, then 'bin_gen' and 'bin_src' are +# executable, asserting that copy_file makes the output executable. +genrule( + name = "run_executables", + outs = [ + "xsrc-out.txt", + "xgen-out.txt", + ], + cmd = ("$(location :bin_src) > $(location xsrc-out.txt) && " + + "$(location :bin_gen) > $(location xgen-out.txt)"), + output_to_bindir = 1, + tools = [ + ":bin_gen", + ":bin_src", + ], +) + +# If 'bin_src' is built, then 'copy_xsrc' made its output executable. +sh_binary( + name = "bin_src", + srcs = [":copy_xsrc"], +) + +# If 'bin_gen' is built, then 'copy_xgen' made its output executable. +sh_binary( + name = "bin_gen", + srcs = [":copy_xgen"], +) + +copy_file( + name = "copy_src", + src = "a.txt", + out = "out/a-out.txt", +) + +copy_file( + name = "copy_gen", + src = ":gen", + out = "out/gen-out.txt", +) + +copy_file( + name = "copy_xsrc", + src = "a.txt", + out = "xout/a-out.sh", + is_executable = True, +) + +copy_file( + name = "copy_xgen", + src = ":gen", + out = "xout/gen-out.sh", + is_executable = True, +) + +genrule( + name = "gen", + outs = ["b.txt"], + cmd = "echo -e '#!/bin/bash\necho potato' > $@", +) diff --git a/tests/copy_file/a.txt b/tests/copy_file/a.txt new file mode 100644 index 0000000..acd332a --- /dev/null +++ b/tests/copy_file/a.txt @@ -0,0 +1,2 @@ +#!/bin/bash +echo aaa diff --git a/tests/copy_file/copy_file_tests.sh b/tests/copy_file/copy_file_tests.sh new file mode 100755 index 0000000..23f45d5 --- /dev/null +++ b/tests/copy_file/copy_file_tests.sh @@ -0,0 +1,63 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization --- +# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). +set -euo pipefail +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else + echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" + exit 1 +fi +# --- end runfiles.bash initialization --- + +source "$(rlocation bazel_skylib/tests/unittest.bash)" \ + || { echo "Could not source bazel_skylib/tests/unittest.bash" >&2; exit 1; } + +function test_copy_src() { + cat "$(rlocation bazel_skylib/tests/copy_file/out/a-out.txt)" >"$TEST_log" + expect_log '^#!/bin/bash$' + expect_log '^echo aaa$' +} + +function test_copy_gen() { + cat "$(rlocation bazel_skylib/tests/copy_file/out/gen-out.txt)" >"$TEST_log" + expect_log '^#!/bin/bash$' + expect_log '^echo potato$' +} + +function test_copy_xsrc() { + cat "$(rlocation bazel_skylib/tests/copy_file/xsrc-out.txt)" >"$TEST_log" + expect_log '^aaa$' +} + +function test_copy_xgen() { + cat "$(rlocation bazel_skylib/tests/copy_file/xgen-out.txt)" >"$TEST_log" + expect_log '^potato$' +} + +run_suite "copy_file_tests test suite"