diff --git a/underfs/tos/src/test/java/alluxio/underfs/tos/TOSInputStreamTest.java b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSInputStreamTest.java new file mode 100644 index 000000000000..a8c41cf3da7b --- /dev/null +++ b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSInputStreamTest.java @@ -0,0 +1,116 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.tos; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import alluxio.conf.AlluxioConfiguration; +import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.retry.CountingRetry; + +import com.volcengine.tos.TOSV2; +import com.volcengine.tos.model.object.GetObjectV2Input; +import com.volcengine.tos.model.object.GetObjectV2Output; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.util.Arrays; + +/** + * Unit tests for the {@link TOSInputStream}. + */ +public class TOSInputStreamTest { + + private static final String BUCKET_NAME = "testBucket"; + private static final String OBJECT_KEY = "testObjectKey"; + private static AlluxioConfiguration sConf = Configuration.global(); + + private TOSInputStream mTosInputStream; + private TOSV2 mTosClient; + private InputStream[] mInputStreamSpy; + private GetObjectV2Output[] mTosObject; + + /** + * The exception expected to be thrown. + */ + @Rule + public final ExpectedException mExceptionRule = ExpectedException.none(); + + @Before + public void setUp() throws IOException { + mTosClient = mock(TOSV2.class); + + byte[] input = new byte[] {1, 2, 3}; + mTosObject = new GetObjectV2Output[input.length]; + mInputStreamSpy = new InputStream[input.length]; + for (int i = 0; i < input.length; ++i) { + final long pos = (long) i; + mTosObject[i] = mock(GetObjectV2Output.class); + when(mTosClient.getObject(argThat(argument -> { + if (argument instanceof GetObjectV2Input) { + String range = ((GetObjectV2Input) argument).getOptions().getRange(); + return range.equals(MessageFormat.format("bytes={0}-", pos)); + } + return false; + }))).thenReturn(mTosObject[i]); + byte[] mockInput = Arrays.copyOfRange(input, i, input.length); + mInputStreamSpy[i] = spy(new ByteArrayInputStream(mockInput)); + when(mTosObject[i].getContent()).thenReturn(mInputStreamSpy[i]); + } + mTosInputStream = new TOSInputStream(BUCKET_NAME, OBJECT_KEY, mTosClient, new CountingRetry(1), + sConf.getBytes(PropertyKey.UNDERFS_OBJECT_STORE_MULTI_RANGE_CHUNK_SIZE)); + } + + @Test + public void close() throws IOException { + mTosInputStream.close(); + + mExceptionRule.expect(IOException.class); + mExceptionRule.expectMessage(is("Stream closed")); + mTosInputStream.read(); + } + + @Test + public void readInt() throws IOException { + assertEquals(1, mTosInputStream.read()); + assertEquals(2, mTosInputStream.read()); + assertEquals(3, mTosInputStream.read()); + } + + @Test + public void readByteArray() throws IOException { + byte[] bytes = new byte[3]; + int readCount = mTosInputStream.read(bytes, 0, 3); + assertEquals(3, readCount); + assertArrayEquals(new byte[] {1, 2, 3}, bytes); + } + + @Test + public void skip() throws IOException { + assertEquals(1, mTosInputStream.read()); + mTosInputStream.skip(1); + assertEquals(3, mTosInputStream.read()); + } +} diff --git a/underfs/tos/src/test/java/alluxio/underfs/tos/TOSLowLevelOutputStreamTest.java b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSLowLevelOutputStreamTest.java new file mode 100644 index 000000000000..9d65d9f50278 --- /dev/null +++ b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSLowLevelOutputStreamTest.java @@ -0,0 +1,204 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.tos; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import alluxio.conf.Configuration; +import alluxio.conf.InstancedConfiguration; +import alluxio.conf.PropertyKey; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.volcengine.tos.TOSV2; +import com.volcengine.tos.model.object.CompleteMultipartUploadV2Input; +import com.volcengine.tos.model.object.CompleteMultipartUploadV2Output; +import com.volcengine.tos.model.object.CreateMultipartUploadInput; +import com.volcengine.tos.model.object.CreateMultipartUploadOutput; +import com.volcengine.tos.model.object.PutObjectInput; +import com.volcengine.tos.model.object.PutObjectOutput; +import com.volcengine.tos.model.object.UploadPartV2Input; +import com.volcengine.tos.model.object.UploadPartV2Output; +import com.volcengine.tos.model.object.UploadedPartV2; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.concurrent.Callable; + +/** + * Unit tests for the {@link TOSLowLevelOutputStream}. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(TOSLowLevelOutputStream.class) +public class TOSLowLevelOutputStreamTest { + private static final String BUCKET_NAME = "testBucket"; + private static final String PARTITION_SIZE = "8MB"; + private static final String KEY = "testKey"; + private static final String UPLOAD_ID = "testUploadId"; + private InstancedConfiguration mConf = Configuration.copyGlobal(); + + private TOSV2 mMockTosClient; + private ListeningExecutorService mMockExecutor; + private ListenableFuture mMockTag; + private TOSLowLevelOutputStream mStream; + + @Before + public void before() throws Exception { + mockTOSClientAndExecutor(); + mConf.set(PropertyKey.UNDERFS_TOS_STREAMING_UPLOAD_PARTITION_SIZE, PARTITION_SIZE); + mConf.set(PropertyKey.UNDERFS_TOS_STREAMING_UPLOAD_ENABLED, "true"); + mStream = new TOSLowLevelOutputStream(BUCKET_NAME, KEY, mMockTosClient, mMockExecutor, mConf); + } + + @Test + public void writeByte() throws Exception { + mStream.write(1); + + mStream.close(); + Mockito.verify(mMockExecutor, never()).submit(any(Callable.class)); + Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class)); + Mockito.verify(mMockTosClient, never()) + .createMultipartUpload(any(CreateMultipartUploadInput.class)); + Mockito.verify(mMockTosClient, never()).completeMultipartUpload(any( + CompleteMultipartUploadV2Input.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("putTag", mStream.getContentHash().get()); + } + + @Test + public void writeByteArrayForSmallFile() throws Exception { + int partSize = (int) (8 * 1024 * 1024); // 8MB + byte[] b = new byte[partSize]; + + mStream.write(b, 0, b.length); + + mStream.close(); + Mockito.verify(mMockExecutor, never()).submit(any(Callable.class)); + Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class)); + Mockito.verify(mMockTosClient, never()) + .createMultipartUpload(any(CreateMultipartUploadInput.class)); + Mockito.verify(mMockTosClient, never()) + .completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("putTag", mStream.getContentHash().get()); + } + + @Test + public void writeByteArrayForLargeFile() throws Exception { + int partSize = (int) (8 * 1024 * 1024); // 8MB + byte[] b = new byte[partSize + 1]; + + mStream.write(b, 0, b.length); + + mStream.close(); + Mockito.verify(mMockTosClient).createMultipartUpload(any(CreateMultipartUploadInput.class)); + Mockito.verify(mMockExecutor, times(2)).submit(any(Callable.class)); + Mockito.verify(mMockTosClient) + .completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("multiTag", mStream.getContentHash().get()); + } + + @Test + public void createEmptyFile() throws Exception { + mStream.close(); + Mockito.verify(mMockExecutor, never()).submit(any(Callable.class)); + Mockito.verify(mMockTosClient, never()) + .createMultipartUpload(any(CreateMultipartUploadInput.class)); + Mockito.verify(mMockTosClient, never()) + .completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)); + Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("emptyTag", mStream.getContentHash().get()); + } + + @Test + public void flush() throws Exception { + int partSize = (int) (8 * 1024 * 1024); // 8MB + byte[] b = new byte[2 * partSize - 1]; + + mStream.write(b, 0, b.length); + + mStream.flush(); + Mockito.verify(mMockExecutor, times(2)).submit(any(Callable.class)); + Mockito.verify(mMockTag, times(2)).get(); + + mStream.close(); + Mockito.verify(mMockTosClient) + .completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("multiTag", mStream.getContentHash().get()); + } + + @Test + public void close() throws Exception { + mStream.close(); + Mockito.verify(mMockTosClient, never()) + .createMultipartUpload(any(CreateMultipartUploadInput.class)); + Mockito.verify(mMockTosClient, never()) + .completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)); + assertTrue(mStream.getContentHash().isPresent()); + assertEquals("emptyTag", mStream.getContentHash().get()); + } + + private void mockTOSClientAndExecutor() throws Exception { + mMockTosClient = PowerMockito.mock(TOSV2.class); + + CreateMultipartUploadOutput createOutput = new CreateMultipartUploadOutput(); + createOutput.setUploadID(UPLOAD_ID); + when(mMockTosClient.createMultipartUpload(any(CreateMultipartUploadInput.class))) + .thenReturn(createOutput); + + UploadPartV2Output uploadPartOutput = new UploadPartV2Output(); + uploadPartOutput.setEtag("partTag"); + when(mMockTosClient.uploadPart(any(UploadPartV2Input.class))).thenReturn(uploadPartOutput); + + // Use Answer to dynamically return PutObjectOutput based on the input + when(mMockTosClient.putObject(any(PutObjectInput.class))) + .thenAnswer(new Answer() { + @Override + public PutObjectOutput answer(InvocationOnMock invocation) throws Throwable { + PutObjectInput input = invocation.getArgument(0); + PutObjectOutput output = new PutObjectOutput(); + // Determine the Etag value based on the input condition + if (input.getContentLength() == 0) { + output.setEtag("emptyTag"); + } else { + output.setEtag("putTag"); + } + return output; + } + }); + + CompleteMultipartUploadV2Output completeOutput = new CompleteMultipartUploadV2Output(); + completeOutput.setEtag("multiTag"); + when(mMockTosClient.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class))) + .thenReturn(completeOutput); + + mMockTag = (ListenableFuture) PowerMockito.mock(ListenableFuture.class); + when(mMockTag.get()).thenReturn(new UploadedPartV2().setPartNumber(1).setEtag("partTag")); + mMockExecutor = Mockito.mock(ListeningExecutorService.class); + when(mMockExecutor.submit(any(Callable.class))).thenReturn(mMockTag); + } +} diff --git a/underfs/tos/src/test/java/alluxio/underfs/tos/TOSOutputStreamTest.java b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSOutputStreamTest.java new file mode 100644 index 000000000000..125103ab2854 --- /dev/null +++ b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSOutputStreamTest.java @@ -0,0 +1,210 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.tos; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; + +import alluxio.conf.AlluxioConfiguration; +import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; + +import com.volcengine.tos.TOSV2; +import com.volcengine.tos.TosException; +import com.volcengine.tos.model.object.PutObjectInput; +import com.volcengine.tos.model.object.PutObjectOutput; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.security.DigestOutputStream; + +/** + * Unit tests for the {@link TOSOutputStream}. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({ TOSOutputStream.class, Files.class }) +public class TOSOutputStreamTest { + private TOSV2 mTosClient; + private File mFile; + private BufferedOutputStream mLocalOutputStream; + private static AlluxioConfiguration sConf = Configuration.global(); + private final String mEtag = "someTag"; + + /** + * The exception expected to be thrown. + */ + @Rule + public final ExpectedException mThrown = ExpectedException.none(); + + /** + * Sets the properties and configuration before each test runs. + */ + @Before + public void before() throws Exception { + mTosClient = Mockito.mock(TOSV2.class); + PutObjectOutput result = Mockito.mock(PutObjectOutput.class); + Mockito.when(result.getEtag()).thenReturn(mEtag); + Mockito.when(mTosClient.putObject(any(PutObjectInput.class))) + .thenReturn(result); + mFile = Mockito.mock(File.class); + PowerMockito.mockStatic(Files.class); + mLocalOutputStream = Mockito.mock(BufferedOutputStream.class); + } + + /** + * Tests to ensure IOException is thrown if {@link FileOutputStream}() throws an IOException. + */ + @Test + public void testConstructor() throws Exception { + PowerMockito.whenNew(File.class).withArguments(Mockito.anyString()).thenReturn(mFile); + String errorMessage = "protocol doesn't support output"; + PowerMockito.when(Files.newOutputStream(mFile.toPath())) + .thenThrow(new IOException(errorMessage)); + mThrown.expect(IOException.class); + mThrown.expectMessage(errorMessage); + new TOSOutputStream("testBucketName", "testKey", mTosClient, + sConf.getList(PropertyKey.TMP_DIRS)).close(); + } + + /** + * Tests to ensure {@link TOSOutputStream#write(int)} calls {@link OutputStream#write(int)}. + */ + @Test + public void testWrite1() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, + sConf.getList(PropertyKey.TMP_DIRS)); + stream.write(1); + stream.close(); + assertEquals(mEtag, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(1); + } + + /** + * Tests to ensure {@link TOSOutputStream#write(byte[], int, int)} calls + * {@link OutputStream#write(byte[], int, int)} . + */ + @Test + public void testWrite2() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, + sConf.getList(PropertyKey.TMP_DIRS)); + byte[] b = new byte[1]; + stream.write(b, 0, 1); + stream.close(); + assertEquals(mEtag, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(b, 0, 1); + } + + /** + * Tests to ensure {@link TOSOutputStream#write(byte[])} calls {@link OutputStream#write(byte[])}. + */ + @Test + public void testWrite3() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, sConf + .getList(PropertyKey.TMP_DIRS)); + byte[] b = new byte[1]; + stream.write(b); + stream.close(); + assertEquals(mEtag, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(b, 0, 1); + } + + /** + * Tests to ensure IOException is thrown if + * {@link TOSV2#putObject(PutObjectInput)} throws an + * TosException. + */ + @Test + public void testCloseError() throws Exception { + // Mock the necessary objects and their behaviors + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + + // Simulate an exception thrown by the TOS client when attempting to put an object + TosException simulatedException = new TosException(); + PowerMockito.when(mTosClient.putObject(any(PutObjectInput.class))) + .thenThrow(simulatedException); + + // Instantiate the TOSOutputStream object with the mocked TOS client + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, + sConf.getList(PropertyKey.TMP_DIRS)); + + // Set the expected exception type and message + mThrown.expect(AlluxioTosException.class); + + // Attempt to close the stream, which should trigger the exception + stream.close(); + + // Verify that the content hash is still retrievable and matches the expected value + assertEquals(mEtag, stream.getContentHash().get()); + } + + /** + * Tests to ensure {@link File#delete()} is called when close the stream. + */ + @Test + public void testCloseSuccess() throws Exception { + PowerMockito.whenNew(File.class).withArguments(Mockito.anyString()).thenReturn(mFile); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, sConf + .getList(PropertyKey.TMP_DIRS)); + stream.close(); + assertEquals(mEtag, stream.getContentHash().get()); + Mockito.verify(mFile).delete(); + } + + /** + * Tests to ensure {@link TOSOutputStream#flush()} calls {@link OutputStream#flush()}. + */ + @Test + public void testFlush() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + TOSOutputStream stream = new TOSOutputStream("testBucketName", "testKey", mTosClient, sConf + .getList(PropertyKey.TMP_DIRS)); + stream.flush(); + stream.close(); + assertEquals(mEtag, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).flush(); + } +} + diff --git a/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemFactoryTest.java b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemFactoryTest.java new file mode 100644 index 000000000000..eff47aa7f9e6 --- /dev/null +++ b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemFactoryTest.java @@ -0,0 +1,81 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.tos; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import alluxio.conf.InstancedConfiguration; +import alluxio.conf.PropertyKey; +import alluxio.underfs.UnderFileSystemConfiguration; +import alluxio.underfs.UnderFileSystemFactory; +import alluxio.underfs.UnderFileSystemFactoryRegistry; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class TOSUnderFileSystemFactoryTest { + private final String mTOSPath = "tos://tos-alluxio-test/"; + private InstancedConfiguration mAlluxioConf; + private UnderFileSystemConfiguration mConf; + private UnderFileSystemFactory mFactory1; + + @Before + public void setUp() { + mAlluxioConf = Mockito.mock(InstancedConfiguration.class); + Mockito.when(mAlluxioConf.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS)) + .thenReturn("tos://tos-alluxio-test/"); + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_ACCESS_KEY)) + .thenReturn("mockAccessKey"); + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_SECRET_KEY)) + .thenReturn("mockSecretKey"); + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_ENDPOINT_KEY)) + .thenReturn("tos-cn-beijing.volces.com"); + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_REGION)) + .thenReturn("cn-beijing"); + + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + mFactory1 = UnderFileSystemFactoryRegistry.find(mTOSPath, mAlluxioConf); + } + + @Test + public void factory() { + assertNotNull(mFactory1); + } + + @Test + public void createInstanceWithNullPath() { + Exception e = assertThrows(NullPointerException.class, () -> mFactory1.create(null, mConf)); + assertTrue(e.getMessage().contains("path")); + } + + @Test + public void supportsPath() { + assertTrue(mFactory1.supportsPath(mTOSPath)); + assertFalse(mFactory1.supportsPath("s3a://test-bucket/path")); + assertFalse(mFactory1.supportsPath("InvalidPath")); + assertFalse(mFactory1.supportsPath(null)); + } + + @Test + public void createInstanceWithMissingCredentials() { + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_ACCESS_KEY)).thenReturn(null); + Mockito.when(mAlluxioConf.get(PropertyKey.TOS_SECRET_KEY)).thenReturn(null); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + + Exception e = assertThrows(RuntimeException.class, () -> mFactory1.create(mTOSPath, mConf)); + assertTrue(e.getMessage().contains("TOS Credentials not available")); + } +} diff --git a/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemTest.java b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemTest.java new file mode 100644 index 000000000000..157bee4029a9 --- /dev/null +++ b/underfs/tos/src/test/java/alluxio/underfs/tos/TOSUnderFileSystemTest.java @@ -0,0 +1,150 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.tos; + +import alluxio.AlluxioURI; +import alluxio.conf.Configuration; +import alluxio.conf.InstancedConfiguration; +import alluxio.conf.PropertyKey; +import alluxio.underfs.UnderFileSystemConfiguration; +import alluxio.underfs.options.DeleteOptions; + +import com.volcengine.tos.TOSV2; +import com.volcengine.tos.TosClientException; +import com.volcengine.tos.TosServerException; +import com.volcengine.tos.model.object.HeadObjectV2Input; +import com.volcengine.tos.model.object.ListObjectsType2Input; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for the {@link TOSUnderFileSystem}. + */ +public class TOSUnderFileSystemTest { + private static final String PATH = "path"; + private static final String SRC = "src"; + private static final String DST = "dst"; + private static final InstancedConfiguration CONF = Configuration.copyGlobal(); + + private static final String BUCKET_NAME = "bucket"; + + private TOSUnderFileSystem mTOSUnderFileSystem; + private TOSV2 mClient; + + @Rule + public final ExpectedException mThrown = ExpectedException.none(); + + @Before + public void setUp() { + mClient = Mockito.mock(TOSV2.class); + mTOSUnderFileSystem = new TOSUnderFileSystem(new AlluxioURI("tos://" + BUCKET_NAME), + mClient, BUCKET_NAME, UnderFileSystemConfiguration.defaults(CONF)); + } + + /** + * Test case for {@link TOSUnderFileSystem#deleteDirectory(String, DeleteOptions)}. + */ + @Test + public void deleteNonRecursiveOnTosClientException() throws IOException { + Mockito.when(mClient.listObjectsType2(ArgumentMatchers.any(ListObjectsType2Input.class))) + .thenThrow(TosClientException.class); + + boolean ret = + mTOSUnderFileSystem.deleteDirectory(PATH, DeleteOptions.defaults().setRecursive(false)); + Assert.assertFalse(ret); + } + + /** + * Test case for {@link TOSUnderFileSystem#deleteDirectory(String, DeleteOptions)}. + */ + @Test + public void deleteRecursiveOnTosClientException() throws IOException { + Mockito.when(mClient.listObjectsType2(ArgumentMatchers.any(ListObjectsType2Input.class))) + .thenThrow(TosClientException.class); + + boolean ret = + mTOSUnderFileSystem.deleteDirectory(PATH, DeleteOptions.defaults().setRecursive(true)); + Assert.assertFalse(ret); + } + + @Test + public void isFile404() throws IOException { + TosServerException e = new TosServerException(404); + Mockito.when( + mClient.headObject(ArgumentMatchers.any(HeadObjectV2Input.class))) + .thenThrow(e); + + Assert.assertFalse(mTOSUnderFileSystem.isFile(SRC)); + } + + @Test + public void isFileException() throws IOException { + TosServerException e = new TosServerException(403); + Mockito.when( + mClient.headObject(ArgumentMatchers.any(HeadObjectV2Input.class))) + .thenThrow(e); + + mThrown.expect(AlluxioTosException.class); + mTOSUnderFileSystem.isFile(SRC); + } + + /** + * Test case for {@link TOSUnderFileSystem#renameFile(String, String)}. + */ + @Test + public void renameOnTosClientException() throws IOException { + Mockito.when( + mClient.headObject(ArgumentMatchers.any(HeadObjectV2Input.class))) + .thenThrow(TosClientException.class); + + mThrown.expect(AlluxioTosException.class); + mTOSUnderFileSystem.renameFile(SRC, DST); + } + + @Test + public void createCredentialsFromConf() { + // Assume appropriate methods to set configuration and create credentials provider + Map conf = new HashMap<>(); + conf.put(PropertyKey.TOS_ACCESS_KEY, "key1"); + conf.put(PropertyKey.TOS_SECRET_KEY, "key2"); + + UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(CONF); + } + + @Test + public void stripPrefixIfPresent() { + Assert.assertEquals("", mTOSUnderFileSystem.stripPrefixIfPresent("tos://" + BUCKET_NAME)); + Assert.assertEquals("", mTOSUnderFileSystem.stripPrefixIfPresent("tos://" + BUCKET_NAME + "/")); + Assert.assertEquals("test/", + mTOSUnderFileSystem.stripPrefixIfPresent("tos://" + BUCKET_NAME + "/test/")); + Assert.assertEquals("test", mTOSUnderFileSystem.stripPrefixIfPresent("test")); + Assert.assertEquals("test/", mTOSUnderFileSystem.stripPrefixIfPresent("test/")); + Assert.assertEquals("test/", mTOSUnderFileSystem.stripPrefixIfPresent("/test/")); + Assert.assertEquals("test", mTOSUnderFileSystem.stripPrefixIfPresent("/test")); + Assert.assertEquals("", mTOSUnderFileSystem.stripPrefixIfPresent("")); + Assert.assertEquals("", mTOSUnderFileSystem.stripPrefixIfPresent("/")); + } + + @Test + public void getFolderSuffix() { + Assert.assertEquals("/", mTOSUnderFileSystem.getFolderSuffix()); + } +}