Skip to content

Commit

Permalink
Move Streams.copy into elasticsearch-core and make a multi-release jar (
Browse files Browse the repository at this point in the history
#29322)

* Move Streams.copy into elasticsearch-core and make a multi-release jar

This moves the method `Streams.copy(InputStream in, OutputStream out)` into the
`elasticsearch-core` project (inside the `o.e.core.internal.io` package). It
also makes this class into a multi-release class where the Java 9 equivalent
uses `InputStream#transferTo`.

This is a followup from
#29300 (comment)
  • Loading branch information
dakrone authored Apr 6, 2018
1 parent a93c942 commit a07ba9e
Show file tree
Hide file tree
Showing 17 changed files with 278 additions and 55 deletions.
63 changes: 53 additions & 10 deletions libs/elasticsearch-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,45 @@ apply plugin: 'nebula.maven-scm'

archivesBaseName = 'elasticsearch-core'

// we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 9 so we do not include this source set in our IDEs
if (!isEclipse && !isIdea) {
sourceSets {
java9 {
java {
srcDirs = ['src/main/java9']
}
}
}

configurations {
java9Compile.extendsFrom(compile)
}

dependencies {
java9Compile sourceSets.main.output
}

compileJava9Java {
sourceCompatibility = 9
targetCompatibility = 9
}

/* Enable this when forbiddenapis was updated to 2.6.
* See: https://github.com/elastic/elasticsearch/issues/29292
forbiddenApisJava9 {
targetCompatibility = 9
}
*/

jar {
metaInf {
into 'versions/9'
from sourceSets.java9.output
}
manifest.attributes('Multi-Release': 'true')
}
}

publishing {
publications {
nebula {
Expand All @@ -39,6 +78,10 @@ dependencies {
testCompile "junit:junit:${versions.junit}"
testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"

if (!isEclipse && !isIdea) {
java9Compile sourceSets.main.output
}

if (isEclipse == false || project.path == ":libs:elasticsearch-core-tests") {
testCompile("org.elasticsearch.test:framework:${version}") {
exclude group: 'org.elasticsearch', module: 'elasticsearch-core'
Expand Down Expand Up @@ -66,14 +109,14 @@ if (isEclipse) {
}

thirdPartyAudit.excludes = [
// from log4j
'org/osgi/framework/AdaptPermission',
'org/osgi/framework/AdminPermission',
'org/osgi/framework/Bundle',
'org/osgi/framework/BundleActivator',
'org/osgi/framework/BundleContext',
'org/osgi/framework/BundleEvent',
'org/osgi/framework/SynchronousBundleListener',
'org/osgi/framework/wiring/BundleWire',
'org/osgi/framework/wiring/BundleWiring'
// from log4j
'org/osgi/framework/AdaptPermission',
'org/osgi/framework/AdminPermission',
'org/osgi/framework/Bundle',
'org/osgi/framework/BundleActivator',
'org/osgi/framework/BundleContext',
'org/osgi/framework/BundleEvent',
'org/osgi/framework/SynchronousBundleListener',
'org/osgi/framework/wiring/BundleWire',
'org/osgi/framework/wiring/BundleWiring'
]
Original file line number Diff line number Diff line change
Expand Up @@ -41,45 +41,73 @@ private IOUtils() {
}

/**
* Closes all given <tt>Closeable</tt>s. Some of the <tt>Closeable</tt>s may be null; they are ignored. After everything is closed, the
* method either throws the first exception it hit while closing, or completes normally if there were no exceptions.
* Closes all given <tt>Closeable</tt>s. Some of the <tt>Closeable</tt>s may be null; they are
* ignored. After everything is closed, the method either throws the first exception it hit
* while closing with other exceptions added as suppressed, or completes normally if there were
* no exceptions.
*
* @param objects objects to close
*/
public static void close(final Closeable... objects) throws IOException {
close(Arrays.asList(objects));
close(null, Arrays.asList(objects));
}

/**
* Closes all given {@link Closeable}s.
* Closes all given <tt>Closeable</tt>s. Some of the <tt>Closeable</tt>s may be null; they are
* ignored. After everything is closed, the method adds any exceptions as suppressed to the
* original exception, or throws the first exception it hit if {@code Exception} is null. If
* no exceptions are encountered and the passed in exception is null, it completes normally.
*
* @param objects objects to close
*/
public static void close(final Exception e, final Closeable... objects) throws IOException {
close(e, Arrays.asList(objects));
}

/**
* Closes all given <tt>Closeable</tt>s. Some of the <tt>Closeable</tt>s may be null; they are
* ignored. After everything is closed, the method either throws the first exception it hit
* while closing with other exceptions added as suppressed, or completes normally if there were
* no exceptions.
*
* @see #close(Closeable...)
* @param objects objects to close
*/
public static void close(final Iterable<? extends Closeable> objects) throws IOException {
Exception ex = null;
close(null, objects);
}

/**
* Closes all given {@link Closeable}s. If a non-null exception is passed in, or closing a
* stream causes an exception, throws the exception with other {@link RuntimeException} or
* {@link IOException} exceptions added as suppressed.
*
* @param ex existing Exception to add exceptions occurring during close to
* @param objects objects to close
*
* @see #close(Closeable...)
*/
public static void close(final Exception ex, final Iterable<? extends Closeable> objects) throws IOException {
Exception firstException = ex;
for (final Closeable object : objects) {
try {
if (object != null) {
object.close();
}
} catch (final IOException | RuntimeException e) {
if (ex == null) {
ex = e;
if (firstException == null) {
firstException = e;
} else {
ex.addSuppressed(e);
firstException.addSuppressed(e);
}
}
}

if (ex != null) {
if (ex instanceof IOException) {
throw (IOException) ex;
if (firstException != null) {
if (firstException instanceof IOException) {
throw (IOException) firstException;
} else {
// since we only assigned an IOException or a RuntimeException to ex above, in this case ex must be a RuntimeException
throw (RuntimeException) ex;
throw (RuntimeException) firstException;
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.core.internal.io;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Objects;

/**
* Simple utility methods for file and stream copying.
* All copy methods use a block size of 4096 bytes,
* and close all affected streams when done.
* <p>
* Mainly for use within the framework,
* but also useful for application code.
*/
public class Streams {

/**
* Copy the contents of the given InputStream to the given OutputStream.
* Closes both streams when done.
*
* @param in the stream to copy from
* @param out the stream to copy to
* @return the number of bytes copied
* @throws IOException in case of I/O errors
*/
public static long copy(final InputStream in, final OutputStream out) throws IOException {
Objects.requireNonNull(in, "No InputStream specified");
Objects.requireNonNull(out, "No OutputStream specified");
final byte[] buffer = new byte[8192];
Exception err = null;
try {
long byteCount = 0;
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
byteCount += bytesRead;
}
out.flush();
return byteCount;
} catch (IOException | RuntimeException e) {
err = e;
throw e;
} finally {
IOUtils.close(err, in, out);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.core.internal.io;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* Simple utility methods for file and stream copying.
* All copy methods close all affected streams when done.
* <p>
* Mainly for use within the framework,
* but also useful for application code.
*/
public abstract class Streams {

/**
* Copy the contents of the given InputStream to the given OutputStream.
* Closes both streams when done.
*
* @param in the stream to copy from
* @param out the stream to copy to
* @return the number of bytes copied
* @throws IOException in case of I/O errors
*/
public static long copy(final InputStream in, final OutputStream out) throws IOException {
Exception err = null;
try {
final long byteCount = in.transferTo(out);
out.flush();
return byteCount;
} catch (IOException | RuntimeException e) {
err = e;
throw e;
} finally {
IOUtils.close(err, in, out);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.core.internal.io;

import org.elasticsearch.test.ESTestCase;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;

import static org.hamcrest.Matchers.equalTo;

public class StreamsTests extends ESTestCase {
public void testCopyFromInputStream() throws IOException {
byte[] content = "content".getBytes(StandardCharsets.UTF_8);
ByteArrayInputStream in = new ByteArrayInputStream(content);
ByteArrayOutputStream out = new ByteArrayOutputStream(content.length);
long count = Streams.copy(in, out);

assertThat(count, equalTo((long) content.length));
assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.internal.io.Streams;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.core.internal.io.Streams;
import org.elasticsearch.mocksocket.MockHttpServer;
import org.elasticsearch.repositories.gcs.GoogleCloudStorageTestServer.Response;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.core.internal.io.Streams;

import java.io.BufferedInputStream;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -128,7 +128,7 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize) t
}
final Path file = path.resolve(blobName);
try (OutputStream outputStream = Files.newOutputStream(file, StandardOpenOption.CREATE_NEW)) {
Streams.copy(inputStream, outputStream, new byte[blobStore.bufferSizeInBytes()]);
Streams.copy(inputStream, outputStream);
}
IOUtils.fsync(file, false);
IOUtils.fsync(path, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@

import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.core.internal.io.Streams;

import java.io.IOException;
import java.util.Objects;
Expand Down
Loading

0 comments on commit a07ba9e

Please sign in to comment.