Merge remote-tracking branch 'origin/main' into feature/identity

Signed-off-by: Peter Nied <petern@amazon.com>
opensearch-project · Nov 18, 2022 · 71281f1 · 71281f1
2 parents 01f7a50 + 6c84cbd
commit 71281f1
Show file tree

Hide file tree

Showing 229 changed files with 1,279 additions and 2,477 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -53,4 +53,5 @@ BWC_VERSION:
   - "2.3.0"
   - "2.3.1"
   - "2.4.0"
+  - "2.4.1"
   - "2.5.0"
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -27,4 +27,3 @@ jobs:
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           head_template: backport/backport-<%= number %>-to-<%= base %>
-          files_to_skip: 'CHANGELOG.md'
diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
@@ -71,7 +71,7 @@ jobs:
           files: ./codeCoverage.xml
 
       - name: Create Comment Success
-        if: ${{ github.event_name == 'pull_request_target' && success() }}
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
@@ -81,15 +81,42 @@ jobs:
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
 
+      - name: Extract Test Failure
+        if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
+        run: |
+            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+            if [[ "$TEST_FAILURES" != "" ]]
+            then
+              echo "test_failures<<EOF" >> $GITHUB_ENV
+              echo "" >> $GITHUB_ENV
+              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "$TEST_FAILURES" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "EOF" >> $GITHUB_ENV
+            fi
+
+      - name: Create Comment Flaky
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
+
       - name: Create Comment Failure
         if: ${{ github.event_name == 'pull_request_target' && failure() }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
           body: |
               ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x:
+              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
+              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
               Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https://github.com/opensearch-project/OpenSearch/pull/4746))
 - Add project health badges to the README.md ([#4843](https://github.com/opensearch-project/OpenSearch/pull/4843))
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
+- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
 - [Identity] Document identity roadmap and feature branch processes ([#4583](https://github.com/opensearch-project/OpenSearch/pull/4583))
 - [Identity] Add stubs for AccessTokenManager  ([#4612](https://github.com/opensearch-project/OpenSearch/pull/4612))
 - [Identity] Permissions check API ([#4516](https://github.com/opensearch-project/OpenSearch/pull/4516))
@@ -21,7 +23,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Identity] Strategy for Delegated Authority using Tokens ([#4826](https://github.com/opensearch-project/OpenSearch/pull/4826))
 - [Identity] User operations: create update delete ([#4741](https://github.com/opensearch-project/OpenSearch/pull/4741))
 
-
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
 - Bumps `reactor-netty-http` from 1.0.18 to 1.0.23
@@ -44,20 +45,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
-- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
 
 ### Changed
-<<<<<<< HEAD
 
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
 - Weighted round-robin scheduling policy for shard coordination traffic ([#4241](https://github.com/opensearch-project/OpenSearch/pull/4241))
 - Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253))
 - [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402))
-=======
->>>>>>> origin/main
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
 - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
@@ -81,7 +78,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
 
 ### Fixed
-<<<<<<< HEAD
 
 - `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289))
 - PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296))
@@ -129,11 +125,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix decommission status update to non leader nodes ([4800](https://github.com/opensearch-project/OpenSearch/pull/4800))
 - Fix recovery path for searchable snapshots ([4813](https://github.com/opensearch-project/OpenSearch/pull/4813))
 - Fix bug in AwarenessAttributeDecommissionIT([4822](https://github.com/opensearch-project/OpenSearch/pull/4822))
-=======
->>>>>>> origin/main
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
-
+- Add jvm option to allow security manager ([#5194](https://github.com/opensearch-project/OpenSearch/pull/5194))
 ### Security
 <<<<<<< HEAD
 
@@ -157,6 +151,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 ### Dependencies
+<<<<<<< HEAD
+>>>>>>> origin/main
+=======
+- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
+- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995))
+- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
+- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633))
+- Bump `geoip2` from 3.0.1 to 3.0.2 ([#5103](https://github.com/opensearch-project/OpenSearch/pull/5103))
 >>>>>>> origin/main
 ### Changed
 ### Deprecated

diff --git a/build.gradle b/build.gradle
@@ -316,10 +316,7 @@ allprojects {
     javadoc.options.encoding = 'UTF8'
     javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
     javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"]
-    if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) {
-      javadoc.options.addBooleanOption("-enable-preview", true)
-      javadoc.options.addStringOption("-release", BuildParams.runtimeJavaVersion.majorVersion)
-    }
+    javadoc.options.addStringOption("-release", targetCompatibility.majorVersion)
   }
 
   // support for reproducible builds
@@ -416,6 +413,9 @@ gradle.projectsEvaluated {
         if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
           task.jvmArgs += ["-Djava.security.manager=allow"]
         }
+        if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) {
+          task.jvmArgs += ["--enable-preview"]
+        }
       }
     }
 

diff --git a/buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/package-info.java b/buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/package-info.java
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java
@@ -51,8 +51,7 @@
 
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
+import java.util.Objects;
 
 /**
  * A plugin to manage getting and extracting distributions of OpenSearch.
@@ -71,12 +70,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String SNAPSHOT_REPO_NAME = "opensearch-snapshots";
     public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "opensearch_distro_extracted_";
 
-    // for downloading Elasticsearch OSS distributions to run BWC
-    private static final String FAKE_IVY_GROUP_ES = "elasticsearch-distribution";
-    private static final String DOWNLOAD_REPO_NAME_ES = "elasticsearch-downloads";
-    private static final String SNAPSHOT_REPO_NAME_ES = "elasticsearch-snapshots";
-    private static final String FAKE_SNAPSHOT_IVY_GROUP_ES = "elasticsearch-distribution-snapshot";
-
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
@@ -159,35 +152,20 @@ private DistributionDependency resolveDependencyNotation(Project p, OpenSearchDi
         return distributionsResolutionStrategiesContainer.stream()
             .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
             .map(r -> r.getResolver().resolve(p, distribution))
-            .filter(d -> d != null)
+            .filter(Objects::nonNull)
             .findFirst()
             .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group, String... patternLayout) {
-        final List<IvyArtifactRepository> repos = Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
-            repo.setName(name);
-            repo.setUrl(url);
-            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-            repo.patternLayout(layout -> layout.artifact(pattern));
-        })).collect(Collectors.toList());
-
         project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
             exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()]));
-        });
-    }
-
-    private static void addIvyRepo2(Project project, String name, String url, String group) {
-        IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
-            repo.setName(name);
-            repo.setUrl(url);
-            repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
-            repo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/elasticsearch-oss-[revision](-[classifier]).[ext]"));
-        });
-        project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
-            exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(ivyRepo);
+            exclusiveContentRepository.forRepositories(Arrays.stream(patternLayout).map(pattern -> project.getRepositories().ivy(repo -> {
+                repo.setName(name);
+                repo.setUrl(url);
+                repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+                repo.patternLayout(layout -> layout.artifact(pattern));
+            })).toArray(IvyArtifactRepository[]::new));
         });
     }
 
@@ -211,9 +189,6 @@ private static void setupDownloadServiceRepo(Project project) {
             );
             addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
         }
-
-        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
-        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }
 
     /**
@@ -222,59 +197,37 @@ private static void setupDownloadServiceRepo(Project project) {
      * The returned object is suitable to be passed to {@link DependencyHandler}.
      * The concrete type of the object will be a set of maven coordinates as a {@link String}.
      * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
-     * coordinates that resolve to the Elastic download service through an ivy repository.
+     * coordinates that resolve to the OpenSearch download service through an ivy repository.
      */
     private String dependencyNotation(OpenSearchDistribution distribution) {
         Version distroVersion = Version.fromString(distribution.getVersion());
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
-            if (distroVersion.onOrAfter("1.0.0")) {
-                return "org.opensearch.distribution.integ-test-zip:opensearch:" + distribution.getVersion() + "@zip";
-            } else {
-                return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
-            }
+            return "org.opensearch.distribution.integ-test-zip:opensearch:" + distribution.getVersion() + "@zip";
         }
 
         String extension = distribution.getType().toString();
         String classifier = distroVersion.onOrAfter("1.0.0") ? ":x64" : ":x86_64";
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
 
-            if (distroVersion.onOrAfter("1.0.0")) {
-                switch (distribution.getArchitecture()) {
-                    case ARM64:
-                        classifier = ":" + distribution.getPlatform() + "-arm64";
-                        break;
-                    case X64:
-                        classifier = ":" + distribution.getPlatform() + "-x64";
-                        break;
-                    case S390X:
-                        classifier = ":" + distribution.getPlatform() + "-s390x";
-                        break;
-                    default:
-                        throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
-                }
-            } else if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":" + distribution.getPlatform() + "-x86_64";
-            } else {
-                classifier = "";
+            switch (distribution.getArchitecture()) {
+                case ARM64:
+                    classifier = ":" + distribution.getPlatform() + "-arm64";
+                    break;
+                case X64:
+                    classifier = ":" + distribution.getPlatform() + "-x64";
+                    break;
+                case S390X:
+                    classifier = ":" + distribution.getPlatform() + "-s390x";
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unsupported architecture: " + distribution.getArchitecture());
             }
         } else if (distribution.getType() == Type.DEB) {
-            if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":amd64";
-            } else {
-                classifier = "";
-            }
-        } else if (distribution.getType() == Type.RPM && distroVersion.before("7.0.0")) {
-            classifier = "";
+            classifier = ":amd64";
         }
 
-        String group;
-        if (distroVersion.onOrAfter("1.0.0")) {
-            group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
-            return group + ":opensearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
-        } else {
-            group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP_ES : FAKE_IVY_GROUP_ES;
-            return group + ":elasticsearch-oss" + ":" + distribution.getVersion() + classifier + "@" + extension;
-        }
+        String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
+        return group + ":opensearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 }
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
@@ -37,6 +37,7 @@
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
+import org.gradle.internal.os.OperatingSystem;
 
 import java.io.File;
 import java.util.Arrays;
@@ -169,7 +170,7 @@ public Object getBinJavaPath() {
         return new Object() {
             @Override
             public String toString() {
-                return getHomeRoot() + "/bin/java";
+                return OperatingSystem.current().getExecutableName(getHomeRoot() + "/bin/java");
             }
         };
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -53,4 +53,5 @@ BWC_VERSION: @@
       - "2.3.0"
       - "2.3.1"
       - "2.4.0"
+      - "2.4.1"
       - "2.5.0"