forked from bazelbuild/bazel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for .ar archives (and .deb files)
This implements #[15130](bazelbuild#15130). As I was updating the docs for .ar and .deb formats, I also addressed some previous formats that had been added but not propagated through to all the documentation places. Closes bazelbuild#15132. PiperOrigin-RevId: 439569440
- Loading branch information
1 parent
4a1e5e4
commit 9c98120
Showing
8 changed files
with
192 additions
and
5 deletions.
There are no files selected for viewing
81 changes: 81 additions & 0 deletions
81
src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| // Copyright 2022 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.devtools.build.lib.bazel.repository; | ||
|
|
||
| import com.google.common.io.ByteStreams; | ||
| import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor; | ||
| import com.google.devtools.build.lib.vfs.Path; | ||
| import java.io.BufferedInputStream; | ||
| import java.io.FileInputStream; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import org.apache.commons.compress.archivers.ar.ArArchiveEntry; | ||
| import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; | ||
|
|
||
| /** | ||
| * Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain | ||
| * directories. | ||
| */ | ||
| public class ArFunction implements Decompressor { | ||
|
|
||
| public static final Decompressor INSTANCE = new ArFunction(); | ||
|
|
||
| // This is the same value as picked for .tar files, which appears to have worked well. | ||
| private static final int BUFFER_SIZE = 32 * 1024; | ||
|
|
||
| private InputStream getDecompressorStream(DecompressorDescriptor descriptor) throws IOException { | ||
| return new BufferedInputStream( | ||
| new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE); | ||
| } | ||
| ; | ||
|
|
||
| @Override | ||
| public Path decompress(DecompressorDescriptor descriptor) | ||
| throws InterruptedException, IOException { | ||
| if (Thread.interrupted()) { | ||
| throw new InterruptedException(); | ||
| } | ||
|
|
||
| try (InputStream decompressorStream = getDecompressorStream(descriptor)) { | ||
| ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream); | ||
| ArArchiveEntry entry; | ||
| while ((entry = arStream.getNextArEntry()) != null) { | ||
| Path filePath = descriptor.repositoryPath().getRelative(entry.getName()); | ||
| filePath.getParentDirectory().createDirectoryAndParents(); | ||
| if (entry.isDirectory()) { | ||
| // ar archives don't contain any directory information, so this should never | ||
| // happen | ||
| continue; | ||
| } else { | ||
| // We do not have to worry about symlinks in .ar files - it's not supported | ||
| // by the .ar file format. | ||
| try (OutputStream out = filePath.getOutputStream()) { | ||
| ByteStreams.copy(arStream, out); | ||
| } | ||
| filePath.chmod(entry.getMode()); | ||
| // entry.getLastModified() appears to be in seconds, so we need to convert | ||
| // it into milliseconds for setLastModifiedTime | ||
| filePath.setLastModifiedTime(entry.getLastModified() * 1000L); | ||
| } | ||
| if (Thread.interrupted()) { | ||
| throw new InterruptedException(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return descriptor.repositoryPath(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| // Copyright 2022 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.devtools.build.lib.bazel.repository; | ||
|
|
||
| import static com.google.common.truth.Truth.assertThat; | ||
|
|
||
| import com.google.devtools.build.lib.testutil.TestConstants; | ||
| import com.google.devtools.build.lib.testutil.TestUtils; | ||
| import com.google.devtools.build.lib.unix.UnixFileSystem; | ||
| import com.google.devtools.build.lib.util.OS; | ||
| import com.google.devtools.build.lib.vfs.DigestHashFunction; | ||
| import com.google.devtools.build.lib.vfs.FileSystem; | ||
| import com.google.devtools.build.lib.vfs.JavaIoFileSystem; | ||
| import com.google.devtools.build.lib.vfs.Path; | ||
| import com.google.devtools.build.runfiles.Runfiles; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import org.junit.Test; | ||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.JUnit4; | ||
|
|
||
| /** Tests decompressing archives. */ | ||
| @RunWith(JUnit4.class) | ||
| public class ArFunctionTest { | ||
| /* | ||
| * .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md | ||
| * The files contain short UTF-8 encoded strings. | ||
| */ | ||
| private static final String ARCHIVE_NAME = "test_files.ar"; | ||
| private static final String PATH_TO_TEST_ARCHIVE = | ||
| "/com/google/devtools/build/lib/bazel/repository/"; | ||
| private static final String FIRST_FILE_NAME = "archived_first.txt"; | ||
| private static final String SECOND_FILE_NAME = "archived_second.md"; | ||
|
|
||
| @Test | ||
| public void testDecompress() throws Exception { | ||
| Path outputDir = decompress(createDescriptorBuilder()); | ||
|
|
||
| assertThat(outputDir.exists()).isTrue(); | ||
| Path firstFile = outputDir.getRelative(FIRST_FILE_NAME); | ||
| assertThat(firstFile.exists()).isTrue(); | ||
| // There are 20 bytes in the content "this is test file 1" | ||
| assertThat(firstFile.getFileSize()).isEqualTo(20); | ||
| assertThat(firstFile.isSymbolicLink()).isFalse(); | ||
|
|
||
| Path secondFile = outputDir.getRelative(SECOND_FILE_NAME); | ||
| assertThat(secondFile.exists()).isTrue(); | ||
| // There are 20 bytes in the content "this is the second test file" | ||
| assertThat(secondFile.getFileSize()).isEqualTo(29); | ||
| assertThat(secondFile.isSymbolicLink()).isFalse(); | ||
| } | ||
|
|
||
| private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception { | ||
| descriptorBuilder.setDecompressor(ArFunction.INSTANCE); | ||
| return new ArFunction().decompress(descriptorBuilder.build()); | ||
| } | ||
|
|
||
| private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException { | ||
| // This was cribbed from TestArchiveDescriptor | ||
| FileSystem testFS = | ||
| OS.getCurrent() == OS.WINDOWS | ||
| ? new JavaIoFileSystem(DigestHashFunction.SHA256) | ||
| : new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ ""); | ||
|
|
||
| // do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators | ||
| // are not duplicated | ||
| String path = | ||
| (TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/"); | ||
| Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path)); | ||
|
|
||
| Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath()); | ||
| Path outDir = workingDir.getRelative("out"); | ||
|
|
||
| return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
9 changes: 9 additions & 0 deletions
9
src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| !<arch> | ||
| // 40 ` | ||
| archived_first.txt/ | ||
| archived_second.md/ | ||
| /0 0 0 0 644 20 ` | ||
| this is test file 1 | ||
| /20 0 0 0 644 29 ` | ||
| this is the second test file | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters