Skip to content

Commit

Permalink
Add native object copy support
Browse files Browse the repository at this point in the history 
Fixes #46.

This lacks support for replacing system metadata, e.g., Content-Type,
due to jclouds limitations.
  • Loading branch information
@gaul
gaul committed Apr 4, 2015
1 parent 7271a5a commit 55083d4
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 53 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jclouds.version>1.9.0</jclouds.version>
<jclouds.version>2.0.0-SNAPSHOT</jclouds.version>
</properties>

<prerequisites>
Expand Down
88 changes: 41 additions & 47 deletions src/main/java/org/gaul/s3proxy/S3ProxyHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
import org.jclouds.blobstore.domain.ContainerAccess;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.CopyOptions;
import org.jclouds.blobstore.options.CreateContainerOptions;
import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
Expand Down Expand Up @@ -1093,58 +1094,51 @@ private void handleCopyBlob(HttpServletRequest request,
throw new S3Exception(S3ErrorCode.INVALID_REQUEST);
}

Blob blob = blobStore.getBlob(sourceContainerName, sourceBlobName);
if (blob == null) {
throw new S3Exception(S3ErrorCode.NO_SUCH_KEY);
CopyOptions.Builder options = CopyOptions.builder();
if (replaceMetadata) {
// TODO: content disposition
// TODO: content encoding
// TODO: content language
// TODO: content type
// TODO: expires
ImmutableMap.Builder<String, String> metadata =
ImmutableMap.builder();
for (String headerName : Collections.list(
request.getHeaderNames())) {
if (headerName.toLowerCase().startsWith(USER_METADATA_PREFIX)) {
metadata.put(
headerName.substring(USER_METADATA_PREFIX.length()),
Strings.nullToEmpty(request.getHeader(headerName)));
}
}
options.userMetadata(metadata.build());
}

try (InputStream is = blob.getPayload().openStream()) {
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
long contentLength = metadata.getContentLength();
BlobBuilder.PayloadBlobBuilder builder = blobStore
.blobBuilder(destBlobName)
.payload(is)
.contentLength(contentLength);
if (replaceMetadata) {
addContentMetdataFromHttpRequest(builder, request);
} else {
builder.contentDisposition(metadata.getContentDisposition())
.contentEncoding(metadata.getContentEncoding())
.contentLanguage(metadata.getContentLanguage())
.contentType(metadata.getContentType())
.userMetadata(blob.getMetadata().getUserMetadata());
}
String eTag = blobStore.copyBlob(
sourceContainerName, sourceBlobName,
destContainerName, destBlobName, options.build());
BlobMetadata blobMetadata = blobStore.blobMetadata(destContainerName,
destBlobName);
try (Writer writer = response.getWriter()) {
XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
writer);
xml.writeStartDocument();
xml.writeStartElement("CopyObjectResult");
xml.writeDefaultNamespace(AWS_XMLNS);

PutOptions options = new PutOptions();
String blobStoreType = getBlobStoreType(blobStore);
if (blobStoreType.equals("azureblob") &&
contentLength > 64 * 1024 * 1024) {
options.multipart(true);
}
String eTag = blobStore.putBlob(destContainerName,
builder.build(), options);
Date lastModified = blob.getMetadata().getLastModified();
try (Writer writer = response.getWriter()) {
XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(
writer);
xml.writeStartDocument();
xml.writeStartElement("CopyObjectResult");
xml.writeDefaultNamespace(AWS_XMLNS);

xml.writeStartElement("LastModified");
xml.writeCharacters(blobStore.getContext().utils().date()
.iso8601DateFormat(lastModified));
xml.writeEndElement();
xml.writeStartElement("LastModified");
xml.writeCharacters(blobStore.getContext().utils().date()
.iso8601DateFormat(blobMetadata.getLastModified()));
xml.writeEndElement();

xml.writeStartElement("ETag");
xml.writeCharacters("\"" + eTag + "\"");
xml.writeEndElement();
xml.writeStartElement("ETag");
xml.writeCharacters("\"" + eTag + "\"");
xml.writeEndElement();

xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
}
xml.writeEndElement();
xml.flush();
} catch (XMLStreamException xse) {
throw new IOException(xse);
}
}

Expand Down
101 changes: 96 additions & 5 deletions src/test/java/org/gaul/s3proxy/S3AwsSdkTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package org.gaul.s3proxy;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.InputStream;
import java.net.URI;
import java.util.Properties;
Expand All @@ -26,8 +28,12 @@
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource;
import com.google.common.io.Resources;
import com.google.inject.Module;

Expand Down Expand Up @@ -58,7 +64,7 @@ public final class S3AwsSdkTest {
private S3Proxy s3Proxy;
private BlobStoreContext context;
private String containerName;
private BasicAWSCredentials awsCreds;
private AmazonS3 client;

@Before
public void setUp() throws Exception {
Expand All @@ -80,7 +86,6 @@ public void setUp() throws Exception {
S3ProxyConstants.PROPERTY_IDENTITY);
String s3Credential = s3ProxyProperties.getProperty(
S3ProxyConstants.PROPERTY_CREDENTIAL);
awsCreds = new BasicAWSCredentials(s3Identity, s3Credential);
s3Endpoint = new URI(s3ProxyProperties.getProperty(
S3ProxyConstants.PROPERTY_ENDPOINT));
String keyStorePath = s3ProxyProperties.getProperty(
Expand Down Expand Up @@ -127,6 +132,11 @@ public void setUp() throws Exception {
s3Endpoint = new URI(s3Endpoint.getScheme(), s3Endpoint.getUserInfo(),
s3Endpoint.getHost(), s3Proxy.getPort(), s3Endpoint.getPath(),
s3Endpoint.getQuery(), s3Endpoint.getFragment());

BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3Identity,
s3Credential);
client = new AmazonS3Client(awsCreds);
client.setEndpoint(s3Endpoint.toString());
}

@After
Expand All @@ -142,9 +152,6 @@ public void tearDown() throws Exception {

@Test
public void testAwsV4Failure() throws Exception {
AmazonS3 client = new AmazonS3Client(awsCreds);
client.setEndpoint(s3Endpoint.toString());

final String code = "InvalidArgument";
thrown.expect(new CustomTypeSafeMatcher<AmazonS3Exception>(code) {
@Override
Expand All @@ -155,6 +162,90 @@ public boolean matchesSafely(AmazonS3Exception ase) {
client.getObject(containerName, "foo");
}

@Test
public void testCopyObjectPreserveMetadata() throws Exception {
String fromName = "from-name";
String toName = "to-name";
ByteSource byteSource = ByteSource.wrap(new byte[42]);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentDisposition("attachment; filename=old.jpg");
metadata.setContentEncoding("gzip");
metadata.setContentType("application/ogg");
metadata.setUserMetadata(ImmutableMap.of(
"key1", "value1",
"key2", "value2"));

try (InputStream is = byteSource.openStream()) {
client.putObject(containerName, fromName, is, metadata);
}

client.copyObject(containerName, fromName, containerName, toName);

ObjectMetadata copiedMetadata = client.getObjectMetadata(
containerName, toName);
assertThat(copiedMetadata.getContentDisposition()).isEqualTo(
metadata.getContentDisposition());
assertThat(copiedMetadata.getContentEncoding()).isEqualTo(
metadata.getContentEncoding());
// TODO: AWS SDK does not support Content-Language?
/*
assertThat(copiedMetadata.getContentLanguage()).isEqualTo(
metadata.getContentLanguage());
*/
assertThat(copiedMetadata.getContentType()).isEqualTo(
metadata.getContentType());
assertThat(copiedMetadata.getUserMetadata()).isEqualTo(
metadata.getUserMetadata());
// TODO: Expires
}

@Test
public void testCopyObjectReplaceMetadata() throws Exception {
String fromName = "from-name";
String toName = "to-name";
ByteSource byteSource = ByteSource.wrap(new byte[42]);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentDisposition("attachment; filename=old.jpg");
metadata.setContentEncoding("gzip");
metadata.setContentType("audio/ogg");
metadata.setUserMetadata(ImmutableMap.of(
"key1", "value1",
"key2", "value2"));

try (InputStream is = byteSource.openStream()) {
client.putObject(containerName, fromName, is, metadata);
}

CopyObjectRequest request = new CopyObjectRequest(
containerName, fromName, containerName, toName);
ObjectMetadata newMetadata = new ObjectMetadata();
newMetadata.setContentType("audio/mp4");
newMetadata.setContentDisposition("attachment; filename=new.jpg");
newMetadata.setContentEncoding("");
newMetadata.setUserMetadata(ImmutableMap.of(
"key3", "value3",
"key4", "value4"));
request.setNewObjectMetadata(newMetadata);
client.copyObject(request);

ObjectMetadata copiedMetadata = client.getObjectMetadata(
containerName, toName);
// TODO: system metadata
/*
assertThat(copiedMetadata.getContentDisposition()).isEqualTo(
newMetadata.getContentDisposition());
assertThat(copiedMetadata.getContentEncoding()).isEqualTo(
newMetadata.getContentEncoding());
assertThat(copiedMetadata.getContentLanguage()).isEqualTo(
newMetadata.getContentLanguage());
assertThat(copiedMetadata.getContentType()).isEqualTo(
newMetadata.getContentType());
// TODO: Expires
*/
assertThat(copiedMetadata.getUserMetadata()).isEqualTo(
newMetadata.getUserMetadata());
}

private static String createRandomContainerName() {
return "s3proxy-" + new Random().nextInt(Integer.MAX_VALUE);
}
Expand Down

0 comments on commit 55083d4

Please sign in to comment.