Add support for no parent directory inference (bazelbuild#832)

Add feature as described in bazelbuild#832.

RELNOTES: Automatic creation of parent directory specifications for
paths with depth are prevented in pkg_tar archives.

docs/latest.md

@@ -287,7 +287,7 @@ pkg_tar(<a href="#pkg_tar-name">name</a>, <a href="#pkg_tar-allow_duplicates_wit
              <a href="#pkg_tar-empty_dirs">empty_dirs</a>, <a href="#pkg_tar-empty_files">empty_files</a>, <a href="#pkg_tar-extension">extension</a>, <a href="#pkg_tar-files">files</a>, <a href="#pkg_tar-include_runfiles">include_runfiles</a>, <a href="#pkg_tar-mode">mode</a>, <a href="#pkg_tar-modes">modes</a>, <a href="#pkg_tar-mtime">mtime</a>, <a href="#pkg_tar-out">out</a>,
              <a href="#pkg_tar-owner">owner</a>, <a href="#pkg_tar-ownername">ownername</a>, <a href="#pkg_tar-ownernames">ownernames</a>, <a href="#pkg_tar-owners">owners</a>, <a href="#pkg_tar-package_dir">package_dir</a>, <a href="#pkg_tar-package_dir_file">package_dir_file</a>, <a href="#pkg_tar-package_file_name">package_file_name</a>,
              <a href="#pkg_tar-package_variables">package_variables</a>, <a href="#pkg_tar-portable_mtime">portable_mtime</a>, <a href="#pkg_tar-private_stamp_detect">private_stamp_detect</a>, <a href="#pkg_tar-remap_paths">remap_paths</a>, <a href="#pkg_tar-srcs">srcs</a>, <a href="#pkg_tar-stamp">stamp</a>,
-             <a href="#pkg_tar-strip_prefix">strip_prefix</a>, <a href="#pkg_tar-symlinks">symlinks</a>)
+             <a href="#pkg_tar-strip_prefix">strip_prefix</a>, <a href="#pkg_tar-symlinks">symlinks</a>, <a href="#pkg_tar-create_parents">symlinks</a>)
 </pre>
 
 
@@ -326,6 +326,7 @@ pkg_tar(<a href="#pkg_tar-name">name</a>, <a href="#pkg_tar-allow_duplicates_wit
 | <a id="pkg_tar-stamp"></a>stamp |  Enable file time stamping.  Possible values: <li>stamp = 1: Use the time of the build as the modification time of each file in the archive. <li>stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching. <li>stamp = -1: Control the chosen modification time using the --[no]stamp flag. <div class="since"><i>Since 0.5.0</i></div>   | Integer | optional | 0 |
 | <a id="pkg_tar-strip_prefix"></a>strip_prefix |  (note: Use strip_prefix = "." to strip path to the package but preserve relative paths of sub directories beneath the package.)   | String | optional | "" |
 | <a id="pkg_tar-symlinks"></a>symlinks |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
+| <a id="pkg_tar-create_parents"></a>create_parents |  Implicitly create parent directories with default permissions for file paths where parent directories are not specified. | Boolean | optional | True |
 
 
 

pkg/private/tar/build_tar.py

@@ -81,7 +81,7 @@ def normalize_path(self, path: str) -> str:
       dest = self.directory + dest
     return dest
 
-  def add_file(self, f, destfile, mode=None, ids=None, names=None):
+  def add_file(self, f, destfile, mode=None, ids=None, names=None, create_parents=False):
     """Add a file to the tar file.
 
     Args:
@@ -108,14 +108,16 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        create_parents=create_parents)
 
   def add_empty_file(self,
                      destfile,
                      mode=None,
                      ids=None,
                      names=None,
-                     kind=tarfile.REGTYPE):
+                     kind=tarfile.REGTYPE,
+                     create_parents=False):
     """Add a file to the tar file.
 
     Args:
@@ -143,9 +145,10 @@ def add_empty_file(self,
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        create_parents=create_parents)
 
-  def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
+  def add_empty_dir(self, destpath, mode=None, ids=None, names=None, create_parents=False):
     """Add a directory to the tar file.
 
     Args:
@@ -156,9 +159,14 @@ def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
          file will be created as `destfile` in the layer.
     """
     self.add_empty_file(
-        destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE)
+        destpath,
+        mode=mode,
+        ids=ids,
+        names=names,
+        kind=tarfile.DIRTYPE,
+        create_parents=create_parents)
 
-  def add_tar(self, tar):
+  def add_tar(self, tar, create_parents):
     """Merge a tar file into the destination tar file.
 
     All files presents in that tar will be added to the output file
@@ -168,9 +176,9 @@ def add_tar(self, tar):
     Args:
       tar: the tar file to add
     """
-    self.tarfile.add_tar(tar, numeric=True, prefix=self.directory)
+    self.tarfile.add_tar(tar, create_parents, numeric=True, prefix=self.directory)
 
-  def add_link(self, symlink, destination, mode=None, ids=None, names=None):
+  def add_link(self, symlink, destination, mode=None, ids=None, names=None, create_parents=False):
     """Add a symbolic link pointing to `destination`.
 
     Args:
@@ -191,7 +199,8 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        create_parents=create_parents)
 
   def add_deb(self, deb):
     """Extract a debian package in the output tar.
@@ -215,10 +224,10 @@ def add_deb(self, deb):
       tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1])
       with open(tmpfile[1], 'wb') as f:
         f.write(current.data)
-      self.add_tar(tmpfile[1])
+      self.add_tar(tmpfile[1], True)
       os.remove(tmpfile[1])
 
-  def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
+  def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None, create_parents=False):
     """Add a tree artifact to the tar file.
 
     Args:
@@ -283,7 +292,8 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
             mode=0o755,
             ids=ids,
             names=names,
-            kind=tarfile.DIRTYPE)
+            kind=tarfile.DIRTYPE,
+            create_parents=create_parents)
       else:
         # If mode is unspecified, derive the mode from the file's mode.
         if mode is None:
@@ -297,7 +307,8 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
             uid=ids[0],
             gid=ids[1],
             uname=names[0],
-            gname=names[1])
+            gname=names[1],
+            create_parents=create_parents)
 
   def add_manifest_entry(self, entry, file_attributes):
     # Use the pkg_tar mode/owner remapping as a fallback
@@ -383,6 +394,10 @@ def main():
            'path/to/file=root.root.')
   parser.add_argument('--stamp_from', default='',
                       help='File to find BUILD_STAMP in')
+  parser.add_argument('--create_parents',
+                      action='store_true',
+                      help='Automatically creates parent directories implied by a'
+                           ' prefix if they do not exist')
   options = parser.parse_args()
 
   # Parse modes arguments
@@ -437,11 +452,15 @@ def main():
     def file_attributes(filename):
       if filename.startswith('/'):
         filename = filename[1:]
-      return {
+
+      attrs = {
           'mode': mode_map.get(filename, default_mode),
           'ids': ids_map.get(filename, default_ids),
           'names': names_map.get(filename, default_ownername),
       }
+      if options.create_parents:
+        attrs['create_parents'] = True
+      return attrs
 
     if options.manifest:
       with open(options.manifest, 'r') as manifest_fp:
@@ -450,7 +469,7 @@ def file_attributes(filename):
           output.add_manifest_entry(entry, file_attributes)
 
     for tar in options.tar or []:
-      output.add_tar(tar)
+      output.add_tar(tar, options.create_parents)
     for deb in options.deb or []:
       output.add_deb(deb)
 

pkg/private/tar/tar.bzl

@@ -173,6 +173,9 @@ def _pkg_tar_impl(ctx):
     args.set_param_file_format("flag_per_line")
     args.use_param_file("@%s", use_always = False)
 
+    if ctx.attr.create_parents:
+        args.add("--create_parents")
+
     inputs = depset(
         direct = ctx.files.deps + files,
         transitive = mapping_context.file_deps,
@@ -264,6 +267,7 @@ pkg_tar_impl = rule(
         "compressor_args": attr.string(
             doc = """Arg list for `compressor`.""",
         ),
+        "create_parents": attr.bool(default = True),
 
         # Common attributes
         "out": attr.output(mandatory = True),

pkg/private/tar/tar_writer.py

@@ -191,7 +191,8 @@ def add_file(self,
                uname='',
                gname='',
                mtime=None,
-               mode=None):
+               mode=None,
+               create_parents=False):
     """Add a file to the current tar.
 
     Args:
@@ -219,7 +220,8 @@ def add_file(self,
       mtime = self.default_mtime
 
     # Make directories up the file
-    self.add_parents(name, mtime=mtime, mode=0o755, uid=uid, gid=gid, uname=uname, gname=gname)
+    if create_parents:
+      self.add_parents(name, mtime=mtime, mode=0o755, uid=uid, gid=gid, uname=uname, gname=gname)
 
     tarinfo = tarfile.TarInfo(name)
     tarinfo.mtime = mtime
@@ -247,6 +249,7 @@ def add_file(self,
 
   def add_tar(self,
               tar,
+              create_parents,
               rootuid=None,
               rootgid=None,
               numeric=False,
@@ -291,14 +294,15 @@ def add_tar(self,
         if prefix:
           in_name = os.path.normpath(prefix + in_name).replace(os.path.sep, '/')
         tarinfo.name = in_name
-        self.add_parents(
-            path=tarinfo.name,
-            mtime=tarinfo.mtime,
-            mode=0o755,
-            uid=tarinfo.uid,
-            gid=tarinfo.gid,
-            uname=tarinfo.uname,
-            gname=tarinfo.gname)
+        if create_parents:
+          self.add_parents(
+              path=tarinfo.name,
+              mtime=tarinfo.mtime,
+              mode=0o755,
+              uid=tarinfo.uid,
+              gid=tarinfo.gid,
+              uname=tarinfo.uname,
+              gname=tarinfo.gname)
 
         if prefix is not None:
           # Relocate internal hardlinks as well to avoid breaking them.

tests/tar/tar_writer_test.py

@@ -129,7 +129,7 @@ def testMergeTar(self):
       with tar_writer.TarFileWriter(self.tempfile) as f:
         datafile = self.data_files.Rlocation(
             "rules_pkg/tests/testdata/tar_test.tar" + ext)
-        f.add_tar(datafile, name_filter=lambda n: n != "./b")
+        f.add_tar(datafile, True, name_filter=lambda n: n != "./b")
       self.assertTarFileContent(self.tempfile, content)
 
   def testMergeTarRelocated(self):
@@ -141,7 +141,7 @@ def testMergeTarRelocated(self):
     with tar_writer.TarFileWriter(self.tempfile) as f:
       datafile = self.data_files.Rlocation(
           "rules_pkg/tests/testdata/tar_test.tar")
-      f.add_tar(datafile, name_filter=lambda n: n != "./b", prefix="foo")
+      f.add_tar(datafile, True, name_filter=lambda n: n != "./b", prefix="foo")
     self.assertTarFileContent(self.tempfile, content)
 
   def testDefaultMtimeNotProvided(self):
@@ -160,7 +160,7 @@ def testPreserveTarMtimesTrueByDefault(self):
     with tar_writer.TarFileWriter(self.tempfile) as f:
       input_tar_path = self.data_files.Rlocation(
           "rules_pkg/tests/testdata/tar_test.tar")
-      f.add_tar(input_tar_path)
+      f.add_tar(input_tar_path, True)
       input_tar = tarfile.open(input_tar_path, "r")
       for file_name in f.members:
         input_file = input_tar.getmember(file_name)
@@ -171,13 +171,13 @@ def testPreserveTarMtimesFalse(self):
     with tar_writer.TarFileWriter(self.tempfile, preserve_tar_mtimes=False) as f:
       input_tar_path = self.data_files.Rlocation(
           "rules_pkg/tests/testdata/tar_test.tar")
-      f.add_tar(input_tar_path)
+      f.add_tar(input_tar_path, True)
       for output_file in f.tar:
         self.assertEqual(output_file.mtime, 0)
 
   def testAddingDirectoriesForFile(self):
     with tar_writer.TarFileWriter(self.tempfile) as f:
-      f.add_file("d/f")
+      f.add_file("d/f", create_parents=True)
     content = [
         {"name": "d", "mode": 0o755},
         {"name": "d/f"},
@@ -193,9 +193,9 @@ def testAddingDirectoriesForFileManually(self):
       f.add_file("a/b", tarfile.DIRTYPE)
       f.add_file("a/b", tarfile.DIRTYPE)
       f.add_file("a/b/", tarfile.DIRTYPE)
-      f.add_file("a/b/c/f")
+      f.add_file("a/b/c/f", create_parents=True)
 
-      f.add_file("x/y/f")
+      f.add_file("x/y/f", create_parents=True)
       f.add_file("x", tarfile.DIRTYPE)
     content = [
         {"name": "d", "mode": 0o755},