feat: add a tar toolchain (#468)

* feat: add a BSD tar toolchain

@thesayyn discovered that it has a feature which should make it a drop-in replacement for pkg_tar
including fine-grained file permissions and symlinks:
https://man.freebsd.org/cgi/man.cgi?mtree(8)

* show example of mtree usage

* feat: introduce tar rule

* cleanup and get test passing

* more cleanup

* chore: add support for compress flags

* chore: add docs

* chore: add docs

* feat: implement linux bsdtar toolchain (#566)

* chore: improve target naming

* WIP: args

* feat: generate mtree spec

Also allow arbitrary args

* refactor: mtree is required

* refactor: style nits

* fix: support mix of source and generated artifacts

* feat: demonstrate strip_prefix

* chore: regen docs

* fix: make host toolchain a fallback toolchain

* fix: include libarchive13.so when installing BSD tar

* chore: buildifier

* fix: aarch64 cpu constraint

* fix(ci): include libarchive13.so when running tar

* chore: add libnettle

* refactor: inputs mutated less

* refactor: remove unneeded substitution arg

* refactor: don't advertise unsupported modes

* fix: hack enough to make it run on my machine

* chore: dynamic libraries included in sh_binary under toolchain

* make sh_binary work

* refactor: drop arm64 for now

* fix toolchain

* fix test

* chore: improve test naming scheme

---------

Co-authored-by: Sahin Yort <thesayyn@gmail.com>

docs/BUILD.bazel

@@ -58,6 +58,11 @@ stardoc_with_diff_test(
     bzl_library_target = "//lib:lists",
 )
 
+stardoc_with_diff_test(
+    name = "tar",
+    bzl_library_target = "//lib:tar",
+)
+
 stardoc_with_diff_test(
     name = "utils",
     bzl_library_target = "//lib:utils",

internal_deps.bzl

@@ -4,7 +4,7 @@ Users should *not* need to install these. If users see a load()
 statement from these, that's a bug in our distribution.
 """
 
-load("//lib:repositories.bzl", "register_coreutils_toolchains", "register_jq_toolchains", "register_yq_toolchains")
+load("//lib:repositories.bzl", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains", "register_yq_toolchains")
 load("//lib:utils.bzl", http_archive = "maybe_http_archive")
 
 # buildifier: disable=unnamed-macro
@@ -66,3 +66,4 @@ def bazel_lib_internal_deps():
     register_jq_toolchains()
     register_yq_toolchains()
     register_coreutils_toolchains()
+    register_tar_toolchains()

lib/BUILD.bazel

@@ -55,6 +55,10 @@ toolchain_type(
     name = "expand_template_toolchain_type",
 )
 
+toolchain_type(
+    name = "tar_toolchain_type",
+)
+
 bzl_library(
     name = "docs",
     srcs = ["docs.bzl"],
@@ -101,6 +105,16 @@ bzl_library(
     deps = ["//lib/private/docs:utils"],
 )
 
+bzl_library(
+    name = "tar",
+    srcs = ["tar.bzl"],
+    deps = [
+        "//lib/private/docs:tar",
+        "@bazel_skylib//lib:types",
+        "@bazel_skylib//rules:write_file",
+    ],
+)
+
 bzl_library(
     name = "jq",
     srcs = ["jq.bzl"],
@@ -236,6 +250,7 @@ bzl_library(
         "//lib/private/docs:jq_toolchain",
         "//lib/private/docs:local_config_platform",
         "//lib/private/docs:source_toolchains_repo",
+        "//lib/private/docs:tar_toolchain",
         "//lib/private/docs:yq_toolchain",
     ],
 )

lib/extensions.bzl

@@ -7,6 +7,7 @@ load(
     "register_coreutils_toolchains",
     "register_expand_template_toolchains",
     "register_jq_toolchains",
+    "register_tar_toolchains",
     "register_yq_toolchains",
 )
 load("//lib/private:host_repo.bzl", "host_repo")
@@ -17,6 +18,7 @@ def _toolchain_extension(mctx):
     register_jq_toolchains(register = False)
     register_yq_toolchains(register = False)
     register_coreutils_toolchains(register = False)
+    register_tar_toolchains(register = False)
     register_expand_template_toolchains(register = False)
 
     create_host_repo = False

lib/private/docs/BUILD.bazel

@@ -121,6 +121,13 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "tar",
+    srcs = [
+        "//lib/private:tar.bzl",
+    ],
+)
+
 bzl_library(
     name = "utils",
     srcs = [
@@ -211,6 +218,12 @@ bzl_library(
     deps = [":repo_utils"],
 )
 
+bzl_library(
+    name = "tar_toolchain",
+    srcs = ["//lib/private:tar_toolchain.bzl"],
+    deps = [":repo_utils"],
+)
+
 bzl_library(
     name = "repo_utils",
     srcs = ["//lib/private:repo_utils.bzl"],

lib/private/tar.bzl

@@ -0,0 +1,131 @@
+"Implementation of tar rule"
+_tar_attrs = {
+    "args": attr.string_list(
+        doc = "Additional flags permitted by BSD tar; see the man page.",
+    ),
+    "srcs": attr.label_list(
+        doc = "Files and directories that are placed into the tar",
+        mandatory = True,
+        allow_files = True,
+    ),
+    "mode": attr.string(
+        doc = """A mode indicator from the following list, copied from the tar manpage:
+
+       - create: Create a new archive containing the specified items.
+       - append: Like `create`, but new entries are appended to the archive.
+            Note that this only works on uncompressed archives stored in regular files.
+            The -f option is required.
+       - list: List  archive contents to stdout.
+       - update: Like `append`, but new entries are added only if they have a
+            modification date newer than the corresponding entry in the archive.
+	       Note that this only works on uncompressed archives stored in
+	       regular files. The -f option	is required.
+       - extract: Extract to disk from the archive. If a file with the same name
+	       appears more than once in the archive, each copy	 will  be  extracted,
+           with  later  copies  overwriting  (replacing) earlier copies.
+        """,
+        values = ["create"],  # TODO: support other modes: ["append", "list", "update", "extract"]
+        default = "create",
+    ),
+    "mtree": attr.label(
+        doc = "An mtree specification file",
+        allow_single_file = True,
+        # Mandatory since it's the only way to set constant timestamps
+        mandatory = True,
+    ),
+    "out": attr.output(
+        doc = "Resulting tar file to write. If absent, `[name].tar` is written.",
+    ),
+    "compress": attr.string(
+        doc = "Compress the archive file with a supported algorithm.",
+        values = ["bzip2", "compress", "gzip", "lrzip", "lz4", "lzma", "lzop", "xz", "zstd"],
+    ),
+}
+
+_mtree_attrs = {
+    "srcs": attr.label_list(doc = "Files that are placed into the tar", mandatory = True, allow_files = True),
+    "out": attr.output(doc = "Resulting specification file to write"),
+}
+
+def _add_compress_options(compress, args):
+    if compress == "bzip2":
+        args.add("--bzip2")
+    if compress == "compress":
+        args.add("--compress")
+    if compress == "gzip":
+        args.add("--gzip")
+    if compress == "lrzip":
+        args.add("--lrzip")
+    if compress == "lzma":
+        args.add("--lzma")
+    if compress == "lz4":
+        args.add("--lz4")
+    if compress == "lzop":
+        args.add("--lzop")
+    if compress == "xz":
+        args.add("--xz")
+    if compress == "zstd":
+        args.add("--zstd")
+
+def _tar_impl(ctx):
+    bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"]
+    inputs = ctx.files.srcs[:]
+    args = ctx.actions.args()
+
+    # Set mode
+    args.add("--" + ctx.attr.mode)
+
+    # User-provided args first
+    args.add_all(ctx.attr.args)
+
+    # Compression args
+    _add_compress_options(ctx.attr.compress, args)
+
+    out = ctx.outputs.out or ctx.actions.declare_file(ctx.attr.name + ".tar")
+    args.add_all(["--file", out.path])
+
+    args.add("@" + ctx.file.mtree.path)
+    inputs.append(ctx.file.mtree)
+
+    ctx.actions.run(
+        executable = bsdtar.tarinfo.binary,
+        inputs = depset(direct = inputs, transitive = [bsdtar.default.files]),
+        outputs = [out],
+        arguments = [args],
+        mnemonic = "Tar",
+    )
+
+    return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))
+
+def _mtree_line(file, uid = "0", gid = "0", time = "1672560000", mode = "0755"):
+    return " ".join([
+        file.short_path,
+        "uid=" + uid,
+        "gid=" + gid,
+        "time=" + time,
+        "mode=" + mode,
+        "type=" + ("dir" if file.is_directory else "file"),
+        "content=" + file.path,
+    ])
+
+def _mtree_impl(ctx):
+    specification = []
+    out = ctx.outputs.out or ctx.actions.declare_file(ctx.attr.name + ".spec")
+    for s in ctx.files.srcs:
+        specification.append(_mtree_line(s))
+    ctx.actions.write(out, "\n".join(specification + [""]))
+    return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))
+
+tar_lib = struct(
+    attrs = _tar_attrs,
+    implementation = _tar_impl,
+    mtree_attrs = _mtree_attrs,
+    mtree_implementation = _mtree_impl,
+)
+
+tar = rule(
+    doc = "Rule that executes BSD `tar`. Most users should use the [`tar`](#tar) macro, rather than load this directly.",
+    implementation = tar_lib.implementation,
+    attrs = tar_lib.attrs,
+    toolchains = ["@aspect_bazel_lib//lib:tar_toolchain_type"],
+)