Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add a tar toolchain #468

Merged
merged 33 commits into from
Oct 3, 2023
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
858e7a8
feat: add a BSD tar toolchain
alexeagle Sep 29, 2023
1c31008
show example of mtree usage
alexeagle Sep 30, 2023
0aa8530
feat: introduce tar rule
alexeagle Sep 30, 2023
6df4e44
cleanup and get test passing
alexeagle Sep 30, 2023
f626c69
more cleanup
alexeagle Sep 30, 2023
6515fb0
chore: add support for compress flags
alexeagle Sep 30, 2023
4a8ce77
chore: add docs
alexeagle Sep 30, 2023
b3952e1
chore: add docs
alexeagle Sep 30, 2023
959c935
feat: implement linux bsdtar toolchain (#566)
thesayyn Oct 1, 2023
0a7ba0f
chore: improve target naming
alexeagle Sep 30, 2023
21c2a61
WIP: args
alexeagle Sep 30, 2023
a03cd85
feat: generate mtree spec
alexeagle Oct 1, 2023
82bdf83
refactor: mtree is required
alexeagle Oct 1, 2023
ca21629
refactor: style nits
alexeagle Oct 1, 2023
bec2335
fix: support mix of source and generated artifacts
alexeagle Oct 1, 2023
2f2a43c
feat: demonstrate strip_prefix
alexeagle Oct 1, 2023
5cd96f4
chore: regen docs
alexeagle Oct 1, 2023
61b0d38
fix: make host toolchain a fallback toolchain
thesayyn Oct 1, 2023
c8357b2
fix: include libarchive13.so when installing BSD tar
alexeagle Oct 1, 2023
419468c
chore: buildifier
alexeagle Oct 1, 2023
2a131de
fix: aarch64 cpu constraint
alexeagle Oct 2, 2023
65d9505
fix(ci): include libarchive13.so when running tar
alexeagle Oct 1, 2023
1afc20c
chore: add libnettle
alexeagle Oct 2, 2023
a4e8bc0
refactor: inputs mutated less
alexeagle Oct 2, 2023
a12fcf9
refactor: remove unneeded substitution arg
alexeagle Oct 2, 2023
01e3c51
refactor: don't advertise unsupported modes
alexeagle Oct 2, 2023
528a16c
fix: hack enough to make it run on my machine
alexeagle Oct 3, 2023
95f6add
chore: dynamic libraries included in sh_binary under toolchain
alexeagle Oct 3, 2023
d57e25a
make sh_binary work
thesayyn Oct 3, 2023
db83d35
refactor: drop arm64 for now
alexeagle Oct 3, 2023
8166519
fix toolchain
thesayyn Oct 3, 2023
7f18ac7
fix test
thesayyn Oct 3, 2023
4b246f4
chore: improve test naming scheme
alexeagle Oct 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ stardoc_with_diff_test(
bzl_library_target = "//lib:lists",
)

stardoc_with_diff_test(
name = "tar",
bzl_library_target = "//lib:tar",
)

stardoc_with_diff_test(
name = "utils",
bzl_library_target = "//lib:utils",
Expand Down
19 changes: 19 additions & 0 deletions docs/repositories.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

112 changes: 112 additions & 0 deletions docs/tar.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion internal_deps.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Users should *not* need to install these. If users see a load()
statement from these, that's a bug in our distribution.
"""

load("//lib:repositories.bzl", "register_coreutils_toolchains", "register_jq_toolchains", "register_yq_toolchains")
load("//lib:repositories.bzl", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains", "register_yq_toolchains")
load("//lib:utils.bzl", http_archive = "maybe_http_archive")

# buildifier: disable=unnamed-macro
Expand Down Expand Up @@ -66,3 +66,4 @@ def bazel_lib_internal_deps():
register_jq_toolchains()
register_yq_toolchains()
register_coreutils_toolchains()
register_tar_toolchains()
15 changes: 15 additions & 0 deletions lib/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ toolchain_type(
name = "expand_template_toolchain_type",
)

toolchain_type(
name = "tar_toolchain_type",
)

bzl_library(
name = "docs",
srcs = ["docs.bzl"],
Expand Down Expand Up @@ -101,6 +105,16 @@ bzl_library(
deps = ["//lib/private/docs:utils"],
)

bzl_library(
name = "tar",
srcs = ["tar.bzl"],
deps = [
"//lib/private/docs:tar",
"@bazel_skylib//lib:types",
"@bazel_skylib//rules:write_file",
],
)

bzl_library(
name = "jq",
srcs = ["jq.bzl"],
Expand Down Expand Up @@ -236,6 +250,7 @@ bzl_library(
"//lib/private/docs:jq_toolchain",
"//lib/private/docs:local_config_platform",
"//lib/private/docs:source_toolchains_repo",
"//lib/private/docs:tar_toolchain",
"//lib/private/docs:yq_toolchain",
],
)
Expand Down
2 changes: 2 additions & 0 deletions lib/extensions.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ load(
"register_coreutils_toolchains",
"register_expand_template_toolchains",
"register_jq_toolchains",
"register_tar_toolchains",
"register_yq_toolchains",
)
load("//lib/private:host_repo.bzl", "host_repo")
Expand All @@ -17,6 +18,7 @@ def _toolchain_extension(mctx):
register_jq_toolchains(register = False)
register_yq_toolchains(register = False)
register_coreutils_toolchains(register = False)
register_tar_toolchains(register = False)
register_expand_template_toolchains(register = False)

create_host_repo = False
Expand Down
13 changes: 13 additions & 0 deletions lib/private/docs/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ bzl_library(
],
)

bzl_library(
name = "tar",
srcs = [
"//lib/private:tar.bzl",
],
)

bzl_library(
name = "utils",
srcs = [
Expand Down Expand Up @@ -211,6 +218,12 @@ bzl_library(
deps = [":repo_utils"],
)

bzl_library(
name = "tar_toolchain",
srcs = ["//lib/private:tar_toolchain.bzl"],
deps = [":repo_utils"],
)

bzl_library(
name = "repo_utils",
srcs = ["//lib/private:repo_utils.bzl"],
Expand Down
132 changes: 132 additions & 0 deletions lib/private/tar.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
"Implementation of tar rule"
_tar_attrs = {
"args": attr.string_list(
doc = "Additional flags permitted by BSD tar; see the man page.",
),
"srcs": attr.label_list(
doc = "Files and directories that are placed into the tar",
mandatory = True,
allow_files = True,
),
"mode": attr.string(
doc = """The first option to tar is a mode indicator from the following list:
-c Create a new archive containing the specified items. The long
option form is --create.
-r Like -c, but new entries are appended to the archive. Note
that this only works on uncompressed archives stored in regular
files. The -f option is required. The long option form is
--append.
-t List archive contents to stdout. The long option form is
--list.
-u Like -r, but new entries are added only if they have a modifi-
cation date newer than the corresponding entry in the archive.
Note that this only works on uncompressed archives stored in
regular files. The -f option is required. The long form is
--update.
-x Extract to disk from the archive. If a file with the same name
appears more than once in the archive, each copy will be ex-
tracted, with later copies overwriting (replacing) earlier
copies. The long option form is --extract.
""",
values = ["create", "append", "list", "update", "extract"],
default = "create",
),
"mtree": attr.label(
doc = "An mtree specification file",
allow_single_file = True,
# Mandatory since it's the only way to set constant timestamps
mandatory = True,
),
"out": attr.output(
doc = "Resulting tar file to write. If absent, `[name].tar` is written.",
),
"compress": attr.string(
doc = "Compress the archive file with a supported algorithm.",
values = ["bzip2", "compress", "gzip", "lrzip", "lz4", "lzma", "lzop", "xz", "zstd"],
),
}

_mtree_attrs = {
"srcs": attr.label_list(doc = "Files that are placed into the tar", mandatory = True, allow_files = True),
"out": attr.output(doc = "Resulting specification file to write"),
}

def _add_compress_options(compress, args):
if compress == "bzip2":
args.add("--bzip2")
if compress == "compress":
args.add("--compress")
if compress == "gzip":
args.add("--gzip")
if compress == "lrzip":
args.add("--lrzip")
if compress == "lzma":
args.add("--lzma")
if compress == "lz4":
args.add("--lz4")
if compress == "lzop":
args.add("--lzop")
if compress == "xz":
args.add("--xz")
if compress == "zstd":
args.add("--zstd")

def _tar_impl(ctx):
inputs = ctx.files.srcs[:]
args = ctx.actions.args()
if ctx.attr.mode != "create":
fail("Only the 'create' mode is currently supported.")
args.add("--" + ctx.attr.mode)

args.add_all(ctx.attr.args)
_add_compress_options(ctx.attr.compress, args)
args.add_all(["-s", "#{}##".format(ctx.bin_dir.path)])

out = ctx.outputs.out or ctx.actions.declare_file(ctx.attr.name + ".tar")
args.add_all(["--file", out.path])

args.add("@" + ctx.file.mtree.path)
inputs.append(ctx.file.mtree)

ctx.actions.run(
executable = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"].tarinfo.binary,
inputs = inputs,
outputs = [out],
arguments = [args],
mnemonic = "Tar",
)

return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))

def _mtree_line(file, uid = "0", gid = "0", time = "1672560000", mode = "0755"):
return " ".join([
file.short_path,
"uid=" + uid,
"gid=" + gid,
"time=" + time,
"mode=" + mode,
"type=" + ("dir" if file.is_directory else "file"),
"content=" + file.path,
])

def _mtree_impl(ctx):
specification = []
out = ctx.outputs.out or ctx.actions.declare_file(ctx.attr.name + ".spec")
for s in ctx.files.srcs:
specification.append(_mtree_line(s))
ctx.actions.write(out, "\n".join(specification + [""]))
return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))

tar_lib = struct(
attrs = _tar_attrs,
implementation = _tar_impl,
mtree_attrs = _mtree_attrs,
mtree_implementation = _mtree_impl,
)

tar = rule(
doc = "Rule that executes BSD `tar`. Most users should use the [`tar`](#tar) macro, rather than load this directly.",
implementation = tar_lib.implementation,
attrs = tar_lib.attrs,
toolchains = ["@aspect_bazel_lib//lib:tar_toolchain_type"],
)
Loading
Loading