Skip to content

Commit

Permalink
Merge branch 'opensearch-project:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
gashutos authored Jun 12, 2023
2 parents 106f43a + 9b2b3c9 commit 52cd6bd
Show file tree
Hide file tree
Showing 75 changed files with 2,921 additions and 63 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
- OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.83 ([7933](https://github.com/opensearch-project/OpenSearch/pull/7933/))
- Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928)))
- Bump `com.azure:azure-storage-blob` from 12.21.1 to 12.22.2 ([7930](https://github.com/opensearch-project/OpenSearch/pull/7930))

Expand All @@ -60,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
- Align range and default value for deletes_pct_allowed in merge policy ([#7730](https://github.com/opensearch-project/OpenSearch/pull/7730))

### Deprecated

Expand Down Expand Up @@ -90,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

## [Unreleased 2.x]
### Added
- Add task cancellation monitoring service ([#7642](https://github.com/opensearch-project/OpenSearch/pull/7642))
- Add TokenManager Interface ([#7452](https://github.com/opensearch-project/OpenSearch/pull/7452))
- Add Remote store as a segment replication source ([#7653](https://github.com/opensearch-project/OpenSearch/pull/7653))
- Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967))
Expand All @@ -98,6 +101,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Dependencies
- Bump `jackson` from 2.15.1 to 2.15.2 ([#7897](https://github.com/opensearch-project/OpenSearch/pull/7897))
- Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901))
- Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068))

### Changed
- Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
Expand Down
6 changes: 4 additions & 2 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ dependencies {
api 'commons-codec:commons-codec:1.15'
api 'org.apache.commons:commons-compress:1.23.0'
api 'org.apache.ant:ant:1.10.13'
api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0'
api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
api 'com.netflix.nebula:gradle-info-plugin:12.1.3'
api 'org.apache.rat:apache-rat:0.15'
Expand All @@ -118,7 +118,9 @@ dependencies {
api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
api 'org.apache.maven:maven-model:3.9.2'
api 'com.networknt:json-schema-validator:1.0.81'
api 'com.networknt:json-schema-validator:1.0.83'
api 'org.jruby.jcodings:jcodings:1.0.58'
api 'org.jruby.joni:joni:2.1.48'
api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"

testFixturesApi "junit:junit:${props.getProperty('junit')}"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

/*
* Copyright 2014-2016 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opensearch.gradle.plugin

import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.publish.PublishingExtension
import org.gradle.api.publish.ivy.IvyPublication
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.publish.plugins.PublishingPlugin

/**
* Including the support of `optional` dependencies: https://github.com/nebula-plugins/gradle-extra-configurations-plugin/blob/v9.0.0/src/main/groovy/nebula/plugin/extraconfigurations/OptionalBasePlugin.groovy
*/
class OptionalDependenciesPlugin implements Plugin<Project> {
static final String OPTIONAL_IDENTIFIER = 'optional'

@Override
void apply(Project project) {
enhanceProjectModel(project)
configureMavenPublishPlugin(project)
configureIvyPublishPlugin(project)
}

/**
* Enhances the Project domain object by adding
*
* a) a extra property List that holds optional dependencies
* b) a extra method that can be executed as parameter when declaring dependencies
*
* @param project Project
*/
private void enhanceProjectModel(Project project) {
project.ext.optionalDeps = []

project.ext.optional = { dep ->
project.ext.optionalDeps << dep
}
}

/**
* Configures Maven Publishing plugin to ensure that published dependencies receive the optional element.
*
* @param project Project
*/
private void configureMavenPublishPlugin(Project project) {
project.plugins.withType(PublishingPlugin) {
project.publishing {
publications {
project.extensions.findByType(PublishingExtension)?.publications?.withType(MavenPublication) { MavenPublication pub ->
pub.pom.withXml {
project.ext.optionalDeps.each { dep ->
def foundDep = asNode().dependencies.dependency.find {
it.groupId.text() == dep.group && it.artifactId.text() == dep.name
}

if (foundDep) {
if (foundDep.optional) {
foundDep.optional.value = 'true'
} else {
foundDep.appendNode(OPTIONAL_IDENTIFIER, 'true')
}
}
}
}
}
}
}
}
}

/**
* Configures Ivy Publishing plugin to ensure that published dependencies receive the correct conf attribute value.
*
* @param project Project
*/
private void configureIvyPublishPlugin(Project project) {
project.plugins.withType(PublishingPlugin) {
project.publishing {
publications {
project.extensions.findByType(PublishingExtension)?.publications?.withType(IvyPublication) { IvyPublication pub ->
pub.descriptor.withXml {
def rootNode = asNode()

// Add optional configuration if it doesn't exist yet
if (!rootNode.configurations.find { it.@name == OPTIONAL_IDENTIFIER }) {
rootNode.configurations[0].appendNode('conf', [name: OPTIONAL_IDENTIFIER, visibility: 'public'])
}

// Replace dependency "runtime->default" conf attribute value with "optional"
project.ext.optionalDeps.each { dep ->
def foundDep = rootNode.dependencies.dependency.find { it.@name == dep.name }
foundDep?.@conf = OPTIONAL_IDENTIFIER
}
}
}
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ public void validate(InputChanges inputChanges) throws IOException {
File jsonSchemaOnDisk = getJsonSchema();
getLogger().debug("JSON schema : [{}]", jsonSchemaOnDisk.getAbsolutePath());
SchemaValidatorsConfig config = new SchemaValidatorsConfig();
config.setEcma262Validator(true);
JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7);
JsonSchema jsonSchema = factory.getSchema(mapper.readTree(jsonSchemaOnDisk), config);
Map<File, Set<String>> errors = new LinkedHashMap<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#

implementation-class=org.opensearch.gradle.plugin.OptionalDependenciesPlugin
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.gradle.plugin;

import org.apache.maven.model.Dependency;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import org.gradle.testkit.runner.BuildResult;
import org.gradle.testkit.runner.GradleRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.rules.TemporaryFolder;
import org.opensearch.gradle.test.GradleUnitTestCase;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Optional;

import static org.gradle.testkit.runner.TaskOutcome.SUCCESS;
import static org.hamcrest.CoreMatchers.is;

public class OptionalDependenciesPluginTests extends GradleUnitTestCase {
private TemporaryFolder projectDir;

@Before
public void setUp() throws Exception {
projectDir = new TemporaryFolder();
projectDir.create();
}

@After
public void tearDown() {
projectDir.delete();
}

public void testApply() throws FileNotFoundException, IOException, XmlPullParserException {
final File mavenRepoDir = new File(projectDir.getRoot(), "mavenrepo");

try (InputStream in = getClass().getClassLoader().getResourceAsStream("plugin/optional-dependencies.gradle")) {
try (OutputStream out = new FileOutputStream(projectDir.newFile("build.gradle"))) {
in.transferTo(out);
}
}

GradleRunner runner = GradleRunner.create()
.forwardOutput()
.withPluginClasspath()
.withArguments("publish", "-PrepoUrl=" + mavenRepoDir.toURI().toURL())
.withProjectDir(projectDir.getRoot());

BuildResult result = runner.build();
assertEquals(SUCCESS, result.task(":" + "publish").getOutcome());

final String name = projectDir.getRoot().getName();
assertDependency(mavenRepoDir, name);
}

private void assertDependency(File repoUrl, String name) throws FileNotFoundException, IOException, XmlPullParserException {
final File pom = new File(repoUrl, "org/custom/group/" + name + "/1.0.0/" + name + "-1.0.0.pom");
assertThat(pom.exists(), is(true));

final MavenXpp3Reader reader = new MavenXpp3Reader();
final Model model = reader.read(new FileReader(pom));

final Optional<Dependency> dependecyOpt = model.getDependencies()
.stream()
.filter(d -> d.getArtifactId().equals("commons-lang3"))
.findAny();

assertThat(dependecyOpt.isPresent(), is(true));
assertThat(dependecyOpt.get().isOptional(), is(true));
}
}
39 changes: 39 additions & 0 deletions buildSrc/src/test/resources/plugin/optional-dependencies.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

plugins {
id 'java'
id 'maven-publish'
id 'opensearch.optional-dependencies'
}
group = "org.custom.group"
version = '1.0.0'

repositories {
mavenCentral()
}

dependencies {
implementation 'org.apache.commons:commons-lang3:3.3.2', optional
}

publishing {
publications {
mavenJava(MavenPublication) {
from components.java
}
}
repositories {
maven {
url "${repoUrl}"
}
}
}
2 changes: 2 additions & 0 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,5 @@ jmh = 1.35

# compression
zstd = 1.5.5-3

jzlib = 1.1.3
1 change: 0 additions & 1 deletion libs/cli/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
* under the License.
*/
apply plugin: 'opensearch.build'
apply plugin: 'com.netflix.nebula.optional-base'
apply plugin: 'opensearch.publish'

dependencies {
Expand Down
1 change: 0 additions & 1 deletion libs/core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@

import org.opensearch.gradle.info.BuildParams

apply plugin: 'com.netflix.nebula.optional-base'
apply plugin: 'opensearch.publish'

archivesBaseName = 'opensearch-core'
Expand Down
4 changes: 0 additions & 4 deletions modules/transport-netty4/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -187,10 +187,6 @@ thirdPartyAudit {

'com.google.protobuf.nano.CodedOutputByteBufferNano',
'com.google.protobuf.nano.MessageNano',
'com.jcraft.jzlib.Deflater',
'com.jcraft.jzlib.Inflater',
'com.jcraft.jzlib.JZlib$WrapperType',
'com.jcraft.jzlib.JZlib',
'com.ning.compress.BufferRecycler',
'com.ning.compress.lzf.ChunkDecoder',
'com.ning.compress.lzf.ChunkEncoder',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ protected Ec2Client buildClient(
ProxyConfiguration proxyConfiguration,
ClientOverrideConfiguration overrideConfiguration,
String endpoint,
Region region,
String region,
long readTimeoutMillis
) {
ApacheHttpClient.Builder clientBuilder = ApacheHttpClient.builder()
Expand All @@ -94,14 +94,18 @@ protected Ec2Client buildClient(
Ec2ClientBuilder builder = Ec2Client.builder()
.overrideConfiguration(overrideConfiguration)
.httpClientBuilder(clientBuilder)
.credentialsProvider(awsCredentialsProvider)
.region(region);
.credentialsProvider(awsCredentialsProvider);

if (Strings.hasText(endpoint)) {
logger.debug("using explicit ec2 endpoint [{}]", endpoint);
builder.endpointOverride(URI.create(endpoint));
}

if (Strings.hasText(region)) {
logger.debug("using explicit ec2 region [{}]", region);
builder.region(Region.of(region));
}

return SocketAccess.doPrivileged(builder::build);
}

Expand Down
Loading

0 comments on commit 52cd6bd

Please sign in to comment.