diff --git a/docs/latest.md b/docs/latest.md index eea69821..c0c98221 100755 --- a/docs/latest.md +++ b/docs/latest.md @@ -287,7 +287,7 @@ pkg_tar(name, empty_dirs, empty_files, extension, files, include_runfiles, mode, modes, mtime, out, owner, ownername, ownernames, owners, package_dir, package_dir_file, package_file_name, package_variables, portable_mtime, private_stamp_detect, remap_paths, srcs, stamp, - strip_prefix, symlinks) + strip_prefix, symlinks, symlinks) @@ -326,6 +326,7 @@ pkg_tar(name, stamp | Enable file time stamping. Possible values:
  • stamp = 1: Use the time of the build as the modification time of each file in the archive.
  • stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching.
  • stamp = -1: Control the chosen modification time using the --[no]stamp flag.
    Since 0.5.0
    | Integer | optional | 0 | | strip_prefix | (note: Use strip_prefix = "." to strip path to the package but preserve relative paths of sub directories beneath the package.) | String | optional | "" | | symlinks | - | Dictionary: String -> String | optional | {} | +| create_parents | Implicitly create parent directories with default permissions for file paths where parent directories are not specified. | Boolean | optional | True | diff --git a/pkg/private/tar/build_tar.py b/pkg/private/tar/build_tar.py index f4b9f0d3..42459825 100644 --- a/pkg/private/tar/build_tar.py +++ b/pkg/private/tar/build_tar.py @@ -81,7 +81,7 @@ def normalize_path(self, path: str) -> str: dest = self.directory + dest return dest - def add_file(self, f, destfile, mode=None, ids=None, names=None): + def add_file(self, f, destfile, mode=None, ids=None, names=None, create_parents=False): """Add a file to the tar file. Args: @@ -108,14 +108,16 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None): uid=ids[0], gid=ids[1], uname=names[0], - gname=names[1]) + gname=names[1], + create_parents=create_parents) def add_empty_file(self, destfile, mode=None, ids=None, names=None, - kind=tarfile.REGTYPE): + kind=tarfile.REGTYPE, + create_parents=False): """Add a file to the tar file. Args: @@ -143,9 +145,10 @@ def add_empty_file(self, uid=ids[0], gid=ids[1], uname=names[0], - gname=names[1]) + gname=names[1], + create_parents=create_parents) - def add_empty_dir(self, destpath, mode=None, ids=None, names=None): + def add_empty_dir(self, destpath, mode=None, ids=None, names=None, create_parents=False): """Add a directory to the tar file. Args: @@ -156,9 +159,14 @@ def add_empty_dir(self, destpath, mode=None, ids=None, names=None): file will be created as `destfile` in the layer. """ self.add_empty_file( - destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE) + destpath, + mode=mode, + ids=ids, + names=names, + kind=tarfile.DIRTYPE, + create_parents=create_parents) - def add_tar(self, tar): + def add_tar(self, tar, create_parents): """Merge a tar file into the destination tar file. All files presents in that tar will be added to the output file @@ -168,9 +176,9 @@ def add_tar(self, tar): Args: tar: the tar file to add """ - self.tarfile.add_tar(tar, numeric=True, prefix=self.directory) + self.tarfile.add_tar(tar, create_parents, numeric=True, prefix=self.directory) - def add_link(self, symlink, destination, mode=None, ids=None, names=None): + def add_link(self, symlink, destination, mode=None, ids=None, names=None, create_parents=False): """Add a symbolic link pointing to `destination`. Args: @@ -191,7 +199,8 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None): uid=ids[0], gid=ids[1], uname=names[0], - gname=names[1]) + gname=names[1], + create_parents=create_parents) def add_deb(self, deb): """Extract a debian package in the output tar. @@ -215,10 +224,10 @@ def add_deb(self, deb): tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1]) with open(tmpfile[1], 'wb') as f: f.write(current.data) - self.add_tar(tmpfile[1]) + self.add_tar(tmpfile[1], True) os.remove(tmpfile[1]) - def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None): + def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None, create_parents=False): """Add a tree artifact to the tar file. Args: @@ -283,7 +292,8 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None): mode=0o755, ids=ids, names=names, - kind=tarfile.DIRTYPE) + kind=tarfile.DIRTYPE, + create_parents=create_parents) else: # If mode is unspecified, derive the mode from the file's mode. if mode is None: @@ -297,7 +307,8 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None): uid=ids[0], gid=ids[1], uname=names[0], - gname=names[1]) + gname=names[1], + create_parents=create_parents) def add_manifest_entry(self, entry, file_attributes): # Use the pkg_tar mode/owner remapping as a fallback @@ -383,6 +394,10 @@ def main(): 'path/to/file=root.root.') parser.add_argument('--stamp_from', default='', help='File to find BUILD_STAMP in') + parser.add_argument('--create_parents', + action='store_true', + help='Automatically creates parent directories implied by a' + ' prefix if they do not exist') options = parser.parse_args() # Parse modes arguments @@ -437,11 +452,15 @@ def main(): def file_attributes(filename): if filename.startswith('/'): filename = filename[1:] - return { + + attrs = { 'mode': mode_map.get(filename, default_mode), 'ids': ids_map.get(filename, default_ids), 'names': names_map.get(filename, default_ownername), } + if options.create_parents: + attrs['create_parents'] = True + return attrs if options.manifest: with open(options.manifest, 'r') as manifest_fp: @@ -450,7 +469,7 @@ def file_attributes(filename): output.add_manifest_entry(entry, file_attributes) for tar in options.tar or []: - output.add_tar(tar) + output.add_tar(tar, options.create_parents) for deb in options.deb or []: output.add_deb(deb) diff --git a/pkg/private/tar/tar.bzl b/pkg/private/tar/tar.bzl index 963cb5a1..8bea9fbc 100644 --- a/pkg/private/tar/tar.bzl +++ b/pkg/private/tar/tar.bzl @@ -173,6 +173,9 @@ def _pkg_tar_impl(ctx): args.set_param_file_format("flag_per_line") args.use_param_file("@%s", use_always = False) + if ctx.attr.create_parents: + args.add("--create_parents") + inputs = depset( direct = ctx.files.deps + files, transitive = mapping_context.file_deps, @@ -264,6 +267,7 @@ pkg_tar_impl = rule( "compressor_args": attr.string( doc = """Arg list for `compressor`.""", ), + "create_parents": attr.bool(default = True), # Common attributes "out": attr.output(mandatory = True), diff --git a/pkg/private/tar/tar_writer.py b/pkg/private/tar/tar_writer.py index 325db761..501bdf94 100644 --- a/pkg/private/tar/tar_writer.py +++ b/pkg/private/tar/tar_writer.py @@ -191,7 +191,8 @@ def add_file(self, uname='', gname='', mtime=None, - mode=None): + mode=None, + create_parents=False): """Add a file to the current tar. Args: @@ -219,7 +220,8 @@ def add_file(self, mtime = self.default_mtime # Make directories up the file - self.add_parents(name, mtime=mtime, mode=0o755, uid=uid, gid=gid, uname=uname, gname=gname) + if create_parents: + self.add_parents(name, mtime=mtime, mode=0o755, uid=uid, gid=gid, uname=uname, gname=gname) tarinfo = tarfile.TarInfo(name) tarinfo.mtime = mtime @@ -247,6 +249,7 @@ def add_file(self, def add_tar(self, tar, + create_parents, rootuid=None, rootgid=None, numeric=False, @@ -291,14 +294,15 @@ def add_tar(self, if prefix: in_name = os.path.normpath(prefix + in_name).replace(os.path.sep, '/') tarinfo.name = in_name - self.add_parents( - path=tarinfo.name, - mtime=tarinfo.mtime, - mode=0o755, - uid=tarinfo.uid, - gid=tarinfo.gid, - uname=tarinfo.uname, - gname=tarinfo.gname) + if create_parents: + self.add_parents( + path=tarinfo.name, + mtime=tarinfo.mtime, + mode=0o755, + uid=tarinfo.uid, + gid=tarinfo.gid, + uname=tarinfo.uname, + gname=tarinfo.gname) if prefix is not None: # Relocate internal hardlinks as well to avoid breaking them. diff --git a/tests/tar/tar_writer_test.py b/tests/tar/tar_writer_test.py index ad31b0c9..6a92b97f 100644 --- a/tests/tar/tar_writer_test.py +++ b/tests/tar/tar_writer_test.py @@ -129,7 +129,7 @@ def testMergeTar(self): with tar_writer.TarFileWriter(self.tempfile) as f: datafile = self.data_files.Rlocation( "rules_pkg/tests/testdata/tar_test.tar" + ext) - f.add_tar(datafile, name_filter=lambda n: n != "./b") + f.add_tar(datafile, True, name_filter=lambda n: n != "./b") self.assertTarFileContent(self.tempfile, content) def testMergeTarRelocated(self): @@ -141,7 +141,7 @@ def testMergeTarRelocated(self): with tar_writer.TarFileWriter(self.tempfile) as f: datafile = self.data_files.Rlocation( "rules_pkg/tests/testdata/tar_test.tar") - f.add_tar(datafile, name_filter=lambda n: n != "./b", prefix="foo") + f.add_tar(datafile, True, name_filter=lambda n: n != "./b", prefix="foo") self.assertTarFileContent(self.tempfile, content) def testDefaultMtimeNotProvided(self): @@ -160,7 +160,7 @@ def testPreserveTarMtimesTrueByDefault(self): with tar_writer.TarFileWriter(self.tempfile) as f: input_tar_path = self.data_files.Rlocation( "rules_pkg/tests/testdata/tar_test.tar") - f.add_tar(input_tar_path) + f.add_tar(input_tar_path, True) input_tar = tarfile.open(input_tar_path, "r") for file_name in f.members: input_file = input_tar.getmember(file_name) @@ -171,13 +171,13 @@ def testPreserveTarMtimesFalse(self): with tar_writer.TarFileWriter(self.tempfile, preserve_tar_mtimes=False) as f: input_tar_path = self.data_files.Rlocation( "rules_pkg/tests/testdata/tar_test.tar") - f.add_tar(input_tar_path) + f.add_tar(input_tar_path, True) for output_file in f.tar: self.assertEqual(output_file.mtime, 0) def testAddingDirectoriesForFile(self): with tar_writer.TarFileWriter(self.tempfile) as f: - f.add_file("d/f") + f.add_file("d/f", create_parents=True) content = [ {"name": "d", "mode": 0o755}, {"name": "d/f"}, @@ -193,9 +193,9 @@ def testAddingDirectoriesForFileManually(self): f.add_file("a/b", tarfile.DIRTYPE) f.add_file("a/b", tarfile.DIRTYPE) f.add_file("a/b/", tarfile.DIRTYPE) - f.add_file("a/b/c/f") + f.add_file("a/b/c/f", create_parents=True) - f.add_file("x/y/f") + f.add_file("x/y/f", create_parents=True) f.add_file("x", tarfile.DIRTYPE) content = [ {"name": "d", "mode": 0o755},