Merge remote-tracking branch 'origin/main' into replace-master-internal

Signed-off-by: Tianli Feng <ftianli@amazon.com>
opensearch-project · Mar 16, 2022 · 4f1211c · 4f1211c
2 parents 91d768c + 10b9986
commit 4f1211c
Show file tree

Hide file tree

Showing 517 changed files with 5,561 additions and 4,486 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -37,3 +37,4 @@ BWC_VERSION:
   - "1.2.4"
   - "1.2.5"
   - "1.3.0"
+  - "1.4.0"
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -33,6 +33,8 @@
       - [runtimeOnly](#runtimeonly)
       - [compileOnly](#compileonly)
       - [testImplementation](#testimplementation)
+    - [Gradle Plugins](#gradle-plugins)
+      - [Distribution Download Plugin](#distribution-download-plugin)
   - [Misc](#misc)
     - [git-secrets](#git-secrets)
       - [Installation](#installation)
@@ -61,17 +63,38 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS
 
 #### JDK 11
 
-OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`.
+OpenSearch builds using Java 11 at a minimum, using the Adoptium distribution. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. This is configured in [buildSrc/build.gradle](buildSrc/build.gradle) and [distribution/tools/java-version-checker/build.gradle](distribution/tools/java-version-checker/build.gradle).
+
+```
+allprojects {
+  targetCompatibility = JavaVersion.VERSION_11
+  sourceCompatibility = JavaVersion.VERSION_11
+}
+```
+
+```
+sourceCompatibility = JavaVersion.VERSION_11
+targetCompatibility = JavaVersion.VERSION_11
+```
 
 Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11).
 
 #### JDK 14
 
 To run the full suite of tests, download and install [JDK 14](https://jdk.java.net/archive/) and set `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility).
 
-#### Runtime JDK
+#### JDK 17
+
+By default, the test tasks use bundled JDK runtime, configured in [buildSrc/version.properties](buildSrc/version.properties), and set to JDK 17 (LTS).
+
+```
+bundled_jdk_vendor = adoptium
+bundled_jdk = 17.0.2+8
+```
+
+#### Custom Runtime JDK
 
-By default, the test tasks use bundled JDK runtime, configured in `buildSrc/version.properties` and set to JDK 17 (LTS). Other kind of test tasks (integration, cluster, ... ) use the same runtime as `JAVA_HOME`. However, the build supports compiling with JDK 11 and testing on a different version of JDK runtime. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-14`. Alternatively, the runtime JDK version could be provided as the command line argument, using combination of `runtime.java=<major JDK version>` property and `JAVA<major JDK version>_HOME` environment variable, for example `./gradlew -Druntime.java=17 ...` (in this case, the tooling expects `JAVA17_HOME` environment variable to be set).
+Other kind of test tasks (integration, cluster, etc.) use the same runtime as `JAVA_HOME`. However, the build also supports compiling with one version of JDK, and testing on a different version. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-14`. Alternatively, the runtime JDK version could be provided as the command line argument, using combination of `runtime.java=<major JDK version>` property and `JAVA<major JDK version>_HOME` environment variable, for example `./gradlew -Druntime.java=17 ...` (in this case, the tooling expects `JAVA17_HOME` environment variable to be set).
 
 #### Windows
 
@@ -340,6 +363,16 @@ somehow. OpenSearch plugins use this configuration to include dependencies that
 Code that is on the classpath for compiling tests that are part of this project but not production code. The canonical example
 of this is `junit`.
 
+### Gradle Plugins
+
+#### Distribution Download Plugin
+
+The Distribution Download plugin downloads the latest version of OpenSearch by default, and supports overriding this behavior by setting `customDistributionUrl`.
+```
+./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz"
+```
+
+
 ## Misc
 
 ### git-secrets

diff --git a/README.md b/README.md
@@ -39,3 +39,9 @@ This project is licensed under the [Apache v2.0 License](LICENSE.txt).
 ## Copyright
 
 Copyright OpenSearch Contributors. See [NOTICE](NOTICE.txt) for details.
+
+## Trademark
+
+OpenSearch is a registered trademark of Amazon Web Services.
+
+OpenSearch includes certain Apache-licensed Elasticsearch code from Elasticsearch B.V. and other source code. Elasticsearch B.V. is not the source of that other source code. ELASTICSEARCH is a registered trademark of Elasticsearch B.V.
diff --git a/build.gradle b/build.gradle
@@ -230,7 +230,10 @@ tasks.register("branchConsistency") {
 allprojects {
   // configure compiler options
   tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
-    compile.options.compilerArgs << '-Werror'
+    // See please https://bugs.openjdk.java.net/browse/JDK-8209058
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
+        compile.options.compilerArgs << '-Werror'
+    }
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'
     compile.options.compilerArgs << '-Xlint:classfile'
@@ -244,7 +247,7 @@ allprojects {
     compile.options.compilerArgs << '-Xlint:opens'
     compile.options.compilerArgs << '-Xlint:overloads'
     compile.options.compilerArgs << '-Xlint:overrides'
-    compile.options.compilerArgs << '-Xlint:processing'
+    compile.options.compilerArgs << '-Xlint:-processing'
     compile.options.compilerArgs << '-Xlint:rawtypes'
     compile.options.compilerArgs << '-Xlint:removal'
     compile.options.compilerArgs << '-Xlint:requires-automatic'

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -158,8 +158,8 @@ if (project != rootProject) {
   apply plugin: 'opensearch.publish'
 
   allprojects {
-    targetCompatibility = 8
-    sourceCompatibility = 8
+    targetCompatibility = JavaVersion.VERSION_11
+    sourceCompatibility = JavaVersion.VERSION_11
   }
 
   // groovydoc succeeds, but has some weird internal exception...

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java
@@ -195,15 +195,22 @@ private static void setupDownloadServiceRepo(Project project) {
         if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
             return;
         }
-        addIvyRepo(
-            project,
-            DOWNLOAD_REPO_NAME,
-            "https://artifacts.opensearch.org",
-            FAKE_IVY_GROUP,
-            "/releases" + RELEASE_PATTERN_LAYOUT,
-            "/release-candidates" + RELEASE_PATTERN_LAYOUT
-        );
-        addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+        Object customDistributionUrl = project.findProperty("customDistributionUrl");
+        // checks if custom Distribution Url has been passed by user from plugins
+        if (customDistributionUrl != null) {
+            addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
+            addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
+        } else {
+            addIvyRepo(
+                project,
+                DOWNLOAD_REPO_NAME,
+                "https://artifacts.opensearch.org",
+                FAKE_IVY_GROUP,
+                "/releases" + RELEASE_PATTERN_LAYOUT,
+                "/release-candidates" + RELEASE_PATTERN_LAYOUT
+            );
+            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+        }
 
         addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
         addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java
@@ -36,7 +36,6 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
-import java.lang.management.ManagementFactory;
 import java.nio.file.Path;
 
 /**
@@ -52,31 +51,15 @@ public void apply(Project project) {
 
         project.getPlugins().apply(GlobalBuildInfoPlugin.class);
 
-        Path inputDir = project.getRootDir().toPath().resolve(".gradle").resolve("reaper").resolve("build-" + getProcessId("xx"));
+        Path inputDir = project.getRootDir()
+            .toPath()
+            .resolve(".gradle")
+            .resolve("reaper")
+            .resolve("build-" + ProcessHandle.current().pid());
+
         ReaperService service = project.getExtensions()
             .create("reaper", ReaperService.class, project, project.getBuildDir().toPath(), inputDir);
 
         project.getGradle().buildFinished(result -> service.shutdown());
     }
-
-    private static String getProcessId(final String fallback) {
-        // Note: may fail in some JVM implementations
-        // therefore fallback has to be provided
-
-        // something like '<pid>@<hostname>', at least in SUN / Oracle JVMs
-        final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
-        final int index = jvmName.indexOf('@');
-
-        if (index < 1) {
-            // part before '@' empty (index = 0) / '@' not found (index = -1)
-            return fallback;
-        }
-
-        try {
-            return Long.toString(Long.parseLong(jvmName.substring(0, index)));
-        } catch (NumberFormatException e) {
-            // ignore
-        }
-        return fallback;
-    }
 }
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java
@@ -179,11 +179,7 @@ private Path locateReaperJar() {
                 InputStream jarInput = this.getClass().getResourceAsStream("/META-INF/reaper.jar");
             ) {
                 logger.info("Copying reaper.jar...");
-                byte[] buffer = new byte[4096];
-                int len;
-                while ((len = jarInput.read(buffer)) > 0) {
-                    out.write(buffer, 0, len);
-                }
+                jarInput.transferTo(out);
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java
@@ -404,11 +404,6 @@ public void upgradeAllNodesAndPluginsToNextVersion(List<Provider<RegularFile>> p
         writeUnicastHostsFiles();
     }
 
-    public void fullRestart() {
-        stop(false);
-        start();
-    }
-
     public void nextNodeToNextVersion() {
         OpenSearchNode node = upgradeNodeToNextVersion();
         node.start();

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java
@@ -93,7 +93,9 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
@@ -908,7 +910,7 @@ private void startOpenSearchProcess() {
         } catch (IOException e) {
             throw new TestClustersException("Failed to start " + currentConfig.command + " process for " + this, e);
         }
-        // reaper.registerPid(toString(), opensearchProcess.pid());
+        reaper.registerPid(toString(), opensearchProcess.pid());
     }
 
     @Internal
@@ -975,7 +977,7 @@ public synchronized void stop(boolean tailLogs) {
         LOGGER.info("Stopping `{}`, tailLogs: {}", this, tailLogs);
         requireNonNull(opensearchProcess, "Can't stop `" + this + "` as it was not started or already stopped.");
         // Test clusters are not reused, don't spend time on a graceful shutdown
-        stopHandle(opensearchProcess, true);
+        stopProcess(opensearchProcess.toHandle(), true);
         reaper.unregister(toString());
         if (tailLogs) {
             logFileContents("Standard output of node", currentConfig.stdoutFile);
@@ -1000,9 +1002,9 @@ public void setNameCustomization(Function<String, String> nameCustomizer) {
         this.nameCustomization = nameCustomizer;
     }
 
-    private void stopHandle(Process process, boolean forcibly) {
+    private void stopProcess(ProcessHandle processHandle, boolean forcibly) {
         // No-op if the process has already exited by itself.
-        if (process.isAlive() == false) {
+        if (processHandle.isAlive() == false) {
             LOGGER.info("Process was not running when we tried to terminate it.");
             return;
         }
@@ -1011,53 +1013,51 @@ private void stopHandle(Process process, boolean forcibly) {
         // they'll be recorded as having failed and won't restart when the cluster restarts.
         // ES could actually be a child when there's some wrapper process like on Windows,
         // and in that case the ML processes will be grandchildren of the wrapper.
-        // List<Process> children = process.children().collect(Collectors.toList());
+        List<ProcessHandle> children = processHandle.children().collect(Collectors.toList());
         try {
-            // logProcessInfo(
-            // "Terminating " + currentConfig.command + " process" + (forcibly ? " forcibly " : "gracefully") + ":",
-            // process.info()
-            // );
+            logProcessInfo(
+                "Terminating " + currentConfig.command + " process" + (forcibly ? " forcibly " : "gracefully") + ":",
+                processHandle.info()
+            );
 
             if (forcibly) {
-                process.destroyForcibly();
+                processHandle.destroyForcibly();
             } else {
-                process.destroy();
-                waitForProcessToExit(process);
-                if (process.isAlive() == false) {
+                processHandle.destroy();
+                waitForProcessToExit(processHandle);
+                if (processHandle.isAlive() == false) {
                     return;
                 }
                 LOGGER.info(
                     "process did not terminate after {} {}, stopping it forcefully",
                     OPENSEARCH_DESTROY_TIMEOUT,
                     OPENSEARCH_DESTROY_TIMEOUT_UNIT
                 );
-                process.destroyForcibly();
+                processHandle.destroyForcibly();
             }
 
-            waitForProcessToExit(process);
-            if (process.isAlive()) {
+            waitForProcessToExit(processHandle);
+            if (processHandle.isAlive()) {
                 throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this);
             }
         } finally {
-            // children.forEach(each -> stopHandle(each, forcibly));
+            children.forEach(each -> stopProcess(each, forcibly));
         }
 
-        // waitForProcessToExit(process);
-        // if (process.isAlive()) {
-        // throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this);
-        // }
+        waitForProcessToExit(processHandle);
+        if (processHandle.isAlive()) {
+            throw new TestClustersException("Was not able to terminate " + currentConfig.command + " process for " + this);
+        }
     }
 
-    /*
-    private void logProcessInfo(String prefix, Process info) {
+    private void logProcessInfo(String prefix, ProcessHandle.Info info) {
         LOGGER.info(
             prefix + " commandLine:`{}` command:`{}` args:`{}`",
             info.commandLine().orElse("-"),
             info.command().orElse("-"),
             Arrays.stream(info.arguments().orElse(new String[] {})).map(each -> "'" + each + "'").collect(Collectors.joining(" "))
         );
     }
-    */
 
     private void logFileContents(String description, Path from) {
         final Map<String, Integer> errorsAndWarnings = new LinkedHashMap<>();
@@ -1126,14 +1126,16 @@ private String normalizeLogLine(String line) {
         return line;
     }
 
-    private void waitForProcessToExit(Process process) {
+    private void waitForProcessToExit(ProcessHandle processHandle) {
         try {
-            process.waitFor(OPENSEARCH_DESTROY_TIMEOUT, OPENSEARCH_DESTROY_TIMEOUT_UNIT);
+            processHandle.onExit().get(OPENSEARCH_DESTROY_TIMEOUT, OPENSEARCH_DESTROY_TIMEOUT_UNIT);
         } catch (InterruptedException e) {
             LOGGER.info("Interrupted while waiting for {} process", currentConfig.command, e);
             Thread.currentThread().interrupt();
-        } catch (NullPointerException e) {
+        } catch (ExecutionException e) {
             LOGGER.info("Failure while waiting for process to exist", e);
+        } catch (TimeoutException e) {
+            LOGGER.info("Timed out waiting for process to exit", e);
         }
     }
 

diff --git a/...c/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java b/...c/src/main/java/org/opensearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java
@@ -84,11 +84,7 @@ public void unpack(File tarFile, File targetDir) throws IOException {
                     // copy the file from the archive using a small buffer to avoid heaping
                     Files.createFile(destination);
                     try (FileOutputStream fos = new FileOutputStream(destination.toFile())) {
-                        byte[] buffer = new byte[4096];
-                        int len;
-                        while ((len = tar.read(buffer)) > 0) {
-                            fos.write(buffer, 0, len);
-                        }
+                        tar.transferTo(fos);
                     }
                 }
                 if (entry.isSymbolicLink() == false) {

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantBasePlugin.java
@@ -42,6 +42,7 @@
 import org.gradle.api.tasks.TaskState;
 
 import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
@@ -101,7 +102,7 @@ void checkVersion(Project project, String tool, Pattern versionRegex, int... min
                 spec.setCommandLine(tool, "--version");
                 spec.setStandardOutput(pipe);
             });
-            String output = pipe.toString().trim();
+            String output = pipe.toString(StandardCharsets.UTF_8).trim();
             Matcher matcher = versionRegex.matcher(output);
             if (matcher.find() == false) {
                 throw new IllegalStateException(

diff --git a/buildSrc/src/main/resources/minimumRuntimeVersion b/buildSrc/src/main/resources/minimumRuntimeVersion
@@ -1 +1 @@
-1.8
+11
-Original file line number
+Diff line change
@@ Expand Up / @@ -37,3 +37,4 @@ BWC_VERSION: @@
       - "1.2.4"
       - "1.2.5"
       - "1.3.0"
+      - "1.4.0"