From 044f53630a08b0021097e422c9f31f15fb98219d Mon Sep 17 00:00:00 2001 From: "Daniel Doubrovkine (dB.)" Date: Tue, 8 Mar 2022 14:48:51 -0500 Subject: [PATCH] Set target and source compatibility to 11, required by Lucene 9. (#2407) * Set target and source compatibility to 11, required by Lucene 9. Signed-off-by: dblock * Uncomment commented code in #2321 for killing child processes that uses JDK9+ ProcessInfo. Signed-off-by: dblock * Set distribution checker target JDK compatibility to 11. Signed-off-by: dblock * Supress processing warnings. Signed-off-by: dblock --- DEVELOPER_GUIDE.md | 27 ++++++++- build.gradle | 2 +- buildSrc/build.gradle | 4 +- .../org/opensearch/gradle/ReaperPlugin.java | 29 ++-------- .../org/opensearch/gradle/ReaperService.java | 6 +- .../gradle/testclusters/OpenSearchNode.java | 56 ++++++++++--------- .../SymbolicLinkPreservingUntarTransform.java | 6 +- .../gradle/vagrant/VagrantBasePlugin.java | 3 +- .../src/main/resources/minimumRuntimeVersion | 2 +- client/rest/build.gradle | 4 +- client/sniffer/build.gradle | 4 +- client/test/build.gradle | 4 +- .../tools/java-version-checker/build.gradle | 5 +- libs/core/build.gradle | 6 +- server/build.gradle | 6 +- 15 files changed, 82 insertions(+), 82 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 6e3886a04e6da..58444441e3258 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -61,7 +61,19 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS #### JDK 11 -OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. +OpenSearch builds using Java 11 at a minimum, using the Adoptium distribution. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. This is configured in [buildSrc/build.gradle](buildSrc/build.gradle) and [distribution/tools/java-version-checker/build.gradle](distribution/tools/java-version-checker/build.gradle). + +``` +allprojects { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} +``` + +``` +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 +``` Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). @@ -69,9 +81,18 @@ Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk1 To run the full suite of tests, download and install [JDK 14](https://jdk.java.net/archive/) and set `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility). -#### Runtime JDK +#### JDK 17 + +By default, the test tasks use bundled JDK runtime, configured in [buildSrc/version.properties](buildSrc/version.properties), and set to JDK 17 (LTS). + +``` +bundled_jdk_vendor = adoptium +bundled_jdk = 17.0.2+8 +``` + +#### Custom Runtime JDK -By default, the test tasks use bundled JDK runtime, configured in `buildSrc/version.properties` and set to JDK 17 (LTS). Other kind of test tasks (integration, cluster, ... ) use the same runtime as `JAVA_HOME`. However, the build supports compiling with JDK 11 and testing on a different version of JDK runtime. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-14`. Alternatively, the runtime JDK version could be provided as the command line argument, using combination of `runtime.java=` property and `JAVA_HOME` environment variable, for example `./gradlew -Druntime.java=17 ...` (in this case, the tooling expects `JAVA17_HOME` environment variable to be set). +Other kind of test tasks (integration, cluster, etc.) use the same runtime as `JAVA_HOME`. However, the build also supports compiling with one version of JDK, and testing on a different version. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-14`. Alternatively, the runtime JDK version could be provided as the command line argument, using combination of `runtime.java=` property and `JAVA_HOME` environment variable, for example `./gradlew -Druntime.java=17 ...` (in this case, the tooling expects `JAVA17_HOME` environment variable to be set). #### Windows diff --git a/build.gradle b/build.gradle index c12f7ece4d39c..374bfb3ccfae3 100644 --- a/build.gradle +++ b/build.gradle @@ -244,7 +244,7 @@ allprojects { compile.options.compilerArgs << '-Xlint:opens' compile.options.compilerArgs << '-Xlint:overloads' compile.options.compilerArgs << '-Xlint:overrides' - compile.options.compilerArgs << '-Xlint:processing' + compile.options.compilerArgs << '-Xlint:-processing' compile.options.compilerArgs << '-Xlint:rawtypes' compile.options.compilerArgs << '-Xlint:removal' compile.options.compilerArgs << '-Xlint:requires-automatic' diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 37bfc5e764dda..ff79cc5df0df0 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -158,8 +158,8 @@ if (project != rootProject) { apply plugin: 'opensearch.publish' allprojects { - targetCompatibility = 8 - sourceCompatibility = 8 + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 } // groovydoc succeeds, but has some weird internal exception... diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java index af9dc6e053fb2..16e5cba4b5b23 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java @@ -36,7 +36,6 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; -import java.lang.management.ManagementFactory; import java.nio.file.Path; /** @@ -52,31 +51,15 @@ public void apply(Project project) { project.getPlugins().apply(GlobalBuildInfoPlugin.class); - Path inputDir = project.getRootDir().toPath().resolve(".gradle").resolve("reaper").resolve("build-" + getProcessId("xx")); + Path inputDir = project.getRootDir() + .toPath() + .resolve(".gradle") + .resolve("reaper") + .resolve("build-" + ProcessHandle.current().pid()); + ReaperService service = project.getExtensions() .create("reaper", ReaperService.class, project, project.getBuildDir().toPath(), inputDir); project.getGradle().buildFinished(result -> service.shutdown()); } - - private static String getProcessId(final String fallback) { - // Note: may fail in some JVM implementations - // therefore fallback has to be provided - - // something like '@', at least in SUN / Oracle JVMs - final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); - final int index = jvmName.indexOf('@'); - - if (index < 1) { - // part before '@' empty (index = 0) / '@' not found (index = -1) - return fallback; - } - - try { - return Long.toString(Long.parseLong(jvmName.substring(0, index))); - } catch (NumberFormatException e) { - // ignore - } - return fallback; - } } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java index 19660c672af3a..498bd68ca2a91 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java @@ -179,11 +179,7 @@ private Path locateReaperJar() { InputStream jarInput = this.getClass().getResourceAsStream("/META-INF/reaper.jar"); ) { logger.info("Copying reaper.jar..."); - byte[] buffer = new byte[4096]; - int len; - while ((len = jarInput.read(buffer)) > 0) { - out.write(buffer, 0, len); - } + jarInput.transferTo(out); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index a726fc53a1f37..b051c15e81d6d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -93,7 +93,9 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.function.Function; @@ -908,7 +910,7 @@ private void startOpenSearchProcess() { } catch (IOException e) { throw new TestClustersException("Failed to start " + currentConfig.command + " process for " + this, e); } - // reaper.registerPid(toString(), opensearchProcess.pid()); + reaper.registerPid(toString(), opensearchProcess.pid()); } @Internal @@ -975,7 +977,7 @@ public synchronized void stop(boolean tailLogs) { LOGGER.info("Stopping `{}`, tailLogs: {}", this, tailLogs); requireNonNull(opensearchProcess, "Can't stop `" + this + "` as it was not started or already stopped."); // Test clusters are not reused, don't spend time on a graceful shutdown - stopHandle(opensearchProcess, true); + stopProcess(opensearchProcess.toHandle(), true); reaper.unregister(toString()); if (tailLogs) { logFileContents("Standard output of node", currentConfig.stdoutFile); @@ -1000,9 +1002,9 @@ public void setNameCustomization(Function nameCustomizer) { this.nameCustomization = nameCustomizer; } - private void stopHandle(Process process, boolean forcibly) { + private void stopProcess(ProcessHandle processHandle, boolean forcibly) { // No-op if the process has already exited by itself. - if (process.isAlive() == false) { + if (processHandle.isAlive() == false) { LOGGER.info("Process was not running when we tried to terminate it."); return; } @@ -1011,19 +1013,19 @@ private void stopHandle(Process process, boolean forcibly) { // they'll be recorded as having failed and won't restart when the cluster restarts. // ES could actually be a child when there's some wrapper process like on Windows, // and in that case the ML processes will be grandchildren of the wrapper. - // List children = process.children().collect(Collectors.toList()); + List children = processHandle.children().collect(Collectors.toList()); try { - // logProcessInfo( - // "Terminating " + currentConfig.command + " process" + (forcibly ? " forcibly " : "gracefully") + ":", - // process.info() - // ); + logProcessInfo( + "Terminating " + currentConfig.command + " process" + (forcibly ? " forcibly " : "gracefully") + ":", + processHandle.info() + ); if (forcibly) { - process.destroyForcibly(); + processHandle.destroyForcibly(); } else { - process.destroy(); - waitForProcessToExit(process); - if (process.isAlive() == false) { + processHandle.destroy(); + waitForProcessToExit(processHandle); + if (processHandle.isAlive() == false) { return; } LOGGER.info( @@ -1031,25 +1033,24 @@ private void stopHandle(Process process, boolean forcibly) { OPENSEARCH_DESTROY_TIMEOUT, OPENSEARCH_DESTROY_TIMEOUT_UNIT ); - process.destroyForcibly(); + processHandle.destroyForcibly(); } - waitForProcessToExit(process); - if (process.isAlive()) { + waitForProcessToExit(processHandle); + if (processHandle.isAlive()) { throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this); } } finally { - // children.forEach(each -> stopHandle(each, forcibly)); + children.forEach(each -> stopProcess(each, forcibly)); } - // waitForProcessToExit(process); - // if (process.isAlive()) { - // throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this); - // } + waitForProcessToExit(processHandle); + if (processHandle.isAlive()) { + throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this); + } } - /* - private void logProcessInfo(String prefix, Process info) { + private void logProcessInfo(String prefix, ProcessHandle.Info info) { LOGGER.info( prefix + " commandLine:`{}` command:`{}` args:`{}`", info.commandLine().orElse("-"), @@ -1057,7 +1058,6 @@ private void logProcessInfo(String prefix, Process info) { Arrays.stream(info.arguments().orElse(new String[] {})).map(each -> "'" + each + "'").collect(Collectors.joining(" ")) ); } - */ private void logFileContents(String description, Path from) { final Map errorsAndWarnings = new LinkedHashMap<>(); @@ -1126,14 +1126,16 @@ private String normalizeLogLine(String line) { return line; } - private void waitForProcessToExit(Process process) { + private void waitForProcessToExit(ProcessHandle processHandle) { try { - process.waitFor(OPENSEARCH_DESTROY_TIMEOUT, OPENSEARCH_DESTROY_TIMEOUT_UNIT); + processHandle.onExit().get(OPENSEARCH_DESTROY_TIMEOUT, OPENSEARCH_DESTROY_TIMEOUT_UNIT); } catch (InterruptedException e) { LOGGER.info("Interrupted while waiting for {} process", currentConfig.command, e); Thread.currentThread().interrupt(); - } catch (NullPointerException e) { + } catch (ExecutionException e) { LOGGER.info("Failure while waiting for process to exist", e); + } catch (TimeoutException e) { + LOGGER.info("Timed out waiting for process to exit", e); } } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java b/buildSrc/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java index 4b3e92e23925e..5ff8168a9bed2 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java @@ -84,11 +84,7 @@ public void unpack(File tarFile, File targetDir) throws IOException { // copy the file from the archive using a small buffer to avoid heaping Files.createFile(destination); try (FileOutputStream fos = new FileOutputStream(destination.toFile())) { - byte[] buffer = new byte[4096]; - int len; - while ((len = tar.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } + tar.transferTo(fos); } } if (entry.isSymbolicLink() == false) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java index 4b918bb38e3c2..9d957a301dde4 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java @@ -42,6 +42,7 @@ import org.gradle.api.tasks.TaskState; import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.function.Consumer; import java.util.regex.Matcher; @@ -101,7 +102,7 @@ void checkVersion(Project project, String tool, Pattern versionRegex, int... min spec.setCommandLine(tool, "--version"); spec.setStandardOutput(pipe); }); - String output = pipe.toString().trim(); + String output = pipe.toString(StandardCharsets.UTF_8).trim(); Matcher matcher = versionRegex.matcher(output); if (matcher.find() == false) { throw new IllegalStateException( diff --git a/buildSrc/src/main/resources/minimumRuntimeVersion b/buildSrc/src/main/resources/minimumRuntimeVersion index 468437494697b..9d607966b721a 100644 --- a/buildSrc/src/main/resources/minimumRuntimeVersion +++ b/buildSrc/src/main/resources/minimumRuntimeVersion @@ -1 +1 @@ -1.8 \ No newline at end of file +11 \ No newline at end of file diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 2271fed252793..5c1252061443a 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -33,8 +33,8 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' -targetCompatibility = JavaVersion.VERSION_1_8 -sourceCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.VERSION_11 +sourceCompatibility = JavaVersion.VERSION_11 group = 'org.opensearch.client' archivesBaseName = 'opensearch-rest-client' diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index f81f4ccc3b1e8..bc4be1dd153e8 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -30,8 +30,8 @@ apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' -targetCompatibility = JavaVersion.VERSION_1_8 -sourceCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.VERSION_11 +sourceCompatibility = JavaVersion.VERSION_11 group = 'org.opensearch.client' archivesBaseName = 'opensearch-rest-client-sniffer' diff --git a/client/test/build.gradle b/client/test/build.gradle index 7d1333a84eae7..07d874cf01ea7 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -29,8 +29,8 @@ */ apply plugin: 'opensearch.build' -targetCompatibility = JavaVersion.VERSION_1_8 -sourceCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.VERSION_11 +sourceCompatibility = JavaVersion.VERSION_11 group = "${group}.client.test" diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle index d3b1422de475a..9480a86ce6fb7 100644 --- a/distribution/tools/java-version-checker/build.gradle +++ b/distribution/tools/java-version-checker/build.gradle @@ -11,8 +11,9 @@ apply plugin: 'opensearch.build' -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 + // targetting very old java versions enables a warning by default on newer JDK: disable it. compileJava.options.compilerArgs += '-Xlint:-options' diff --git a/libs/core/build.gradle b/libs/core/build.gradle index edb05cd1c22b0..374f2fe572a12 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -54,13 +54,13 @@ if (!isEclipse) { } compileJava11Java { - sourceCompatibility = 11 - targetCompatibility = 11 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } forbiddenApisJava11 { if (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_11) { - targetCompatibility = JavaVersion.VERSION_11.getMajorVersion() + targetCompatibility = JavaVersion.VERSION_11 } replaceSignatureFiles 'jdk-signatures' } diff --git a/server/build.gradle b/server/build.gradle index aa467cd0528bf..3a11428ca7919 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -64,14 +64,14 @@ if (!isEclipse) { } compileJava11Java { - sourceCompatibility = 11 - targetCompatibility = 11 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } tasks.named('forbiddenApisJava11').configure { doFirst { if (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_11) { - targetCompatibility = JavaVersion.VERSION_11.getMajorVersion() + targetCompatibility = JavaVersion.VERSION_11 } } }