Skip to content

Commit

Permalink
Split repository into separate plugins (#191)
Browse files Browse the repository at this point in the history
(cherry picked from commit eb77879)
  • Loading branch information
msfroh authored and github-actions[bot] committed Aug 18, 2023
1 parent 3050a3c commit 8131021
Show file tree
Hide file tree
Showing 87 changed files with 313 additions and 59 deletions.
123 changes: 123 additions & 0 deletions amazon-kendra-intelligent-ranking/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import org.opensearch.gradle.test.RestIntegTestTask

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.yaml-rest-test'
apply plugin: 'jacoco'

group = 'org.opensearch'

def pluginName = 'amazon-kendra-intelligent-ranking'
def pluginDescription = 'Rerank search results using Amazon Kendra Intelligent Ranking'
def projectPath = 'org.opensearch'
def pathToPlugin = 'search.relevance'
def pluginClassName = 'AmazonKendraIntelligentRankingPlugin'

opensearchplugin {
name "opensearch-${pluginName}-${plugin_version}.0"
version "${plugin_version}"
description pluginDescription
classname "${projectPath}.${pathToPlugin}.${pluginClassName}"
licenseFile rootProject.file('LICENSE')
noticeFile rootProject.file('NOTICE')
}

// This requires an additional Jar not published as part of build-tools
loggerUsageCheck.enabled = false

// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

buildscript {
ext {
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
opensearch_version = System.getProperty("opensearch.version", "3.0.0")
plugin_version = opensearch_version
if (isSnapshot) {
opensearch_version += "-SNAPSHOT"
}
}

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
}

dependencies {
classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
}
}

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
}

dependencies {
implementation 'com.ibm.icu:icu4j:57.2'
implementation 'org.apache.commons:commons-lang3:3.12.0'
implementation 'org.apache.httpcomponents:httpclient:4.5.14'
implementation 'org.apache.httpcomponents:httpcore:4.4.16'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.0'
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.0'
implementation 'commons-logging:commons-logging:1.2'
implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300'
implementation 'com.amazonaws:aws-java-sdk-core:1.12.300'
}


allprojects {
plugins.withId('jacoco') {
jacoco.toolVersion = '0.8.9'
}
}


test {
include '**/*Tests.class'
finalizedBy jacocoTestReport
}

task integTest(type: RestIntegTestTask) {
description = "Run tests against a cluster"
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
// The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
if (System.getProperty("test.debug") != null) {
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
}
}

testClusters.integTest {
testDistribution = "ARCHIVE"

// This installs our plugin into the testClusters
plugin(project.tasks.bundlePlugin.archiveFile)
}

run {
useCluster testClusters.integTest
}

jacocoTestReport {
dependsOn test
reports {
xml.required = true
html.required = true
}
}

// TODO: Enable these checks
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.Setting;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
Expand All @@ -27,19 +27,14 @@
import org.opensearch.search.relevance.actionfilter.SearchActionFilter;
import org.opensearch.search.relevance.client.OpenSearchClient;
import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient;
import org.opensearch.search.relevance.configuration.SearchConfigurationExtBuilder;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker;
import org.opensearch.search.relevance.transformer.ResultTransformer;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfigurationFactory;
import org.opensearch.search.relevance.transformer.kendraintelligentranking.pipeline.KendraRankingResponseProcessor;
import org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor;
import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient;
import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings;
import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParametersExtBuilder;
import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRanker;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

Expand All @@ -51,13 +46,12 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;

public class SearchRelevancePlugin extends Plugin implements ActionPlugin, SearchPlugin, SearchPipelinePlugin {
public class AmazonKendraIntelligentRankingPlugin extends Plugin implements ActionPlugin, SearchPlugin, SearchPipelinePlugin {

private OpenSearchClient openSearchClient;
private KendraHttpClient kendraClient;
private KendraIntelligentRanker kendraIntelligentRanker;
private KendraClientSettings kendraClientSettings;
private PersonalizeClientSettings personalizeClientSettings;

private Collection<ResultTransformer> getAllResultTransformers() {
// Initialize and add other transformers here
Expand All @@ -78,7 +72,6 @@ public List<Setting<?>> getSettings() {
// NOTE: cannot use kendraIntelligentRanker.getTransformerSettings because the object is not yet created
List<Setting<?>> allTransformerSettings = new ArrayList<>();
allTransformerSettings.addAll(KendraIntelligentRankerSettings.getAllSettings());
allTransformerSettings.addAll(PersonalizeClientSettings.getAllSettings());
// Add settings for other transformers here
return allTransformerSettings;
}
Expand All @@ -101,7 +94,6 @@ public Collection<Object> createComponents(
this.kendraClientSettings = KendraClientSettings.getClientSettings(environment.settings());
this.kendraClient = new KendraHttpClient(this.kendraClientSettings);
this.kendraIntelligentRanker = new KendraIntelligentRanker(this.kendraClient);
this.personalizeClientSettings = PersonalizeClientSettings.getClientSettings(environment.settings());

return Arrays.asList(
this.openSearchClient,
Expand All @@ -117,15 +109,11 @@ public List<SearchExtSpec<?>> getSearchExts() {
.collect(Collectors.toMap(ResultTransformerConfigurationFactory::getName, i -> i));
return List.of(new SearchExtSpec<>(SearchConfigurationExtBuilder.NAME,
input -> new SearchConfigurationExtBuilder(input, resultTransformerMap),
parser -> SearchConfigurationExtBuilder.parse(parser, resultTransformerMap)),
new SearchExtSpec<>(PersonalizeRequestParametersExtBuilder.NAME,
input -> new PersonalizeRequestParametersExtBuilder(input),
parser -> PersonalizeRequestParametersExtBuilder.parse(parser)));
parser -> SearchConfigurationExtBuilder.parse(parser, resultTransformerMap)));
}

@Override
public Map<String, Processor.Factory<SearchResponseProcessor>> getResponseProcessors(Parameters parameters) {
return Map.of(PersonalizeRankingResponseProcessor.TYPE, new PersonalizeRankingResponseProcessor.Factory(this.personalizeClientSettings ),
KendraRankingResponseProcessor.TYPE, new KendraRankingResponseProcessor.Factory(this.kendraClientSettings));
return Map.of(KendraRankingResponseProcessor.TYPE, new KendraRankingResponseProcessor.Factory(this.kendraClientSettings));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchException;
import org.opensearch.action.ActionListener;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionResponse;
import org.opensearch.action.search.SearchAction;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.search.SearchResponseSections;
import org.opensearch.action.support.ActionFilter;
import org.opensearch.action.support.ActionFilterChain;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.StreamInput;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
*/
package org.opensearch.search.relevance.client;

import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.settings.get.GetSettingsAction;
import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.opensearch.client.Client;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.action.ActionListener;

public class OpenSearchClient {
private final Client client;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
*/
package org.opensearch.search.relevance.configuration;

import org.opensearch.action.ActionListener;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.common.settings.Settings;
import org.opensearch.search.SearchExtBuilder;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
package org.opensearch.search.relevance;

import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.opensearch.client.Request;
import org.opensearch.client.Response;
import org.opensearch.test.rest.OpenSearchRestTestCase;

import java.io.IOException;

public class AmazonKendraIntelligentRankingPluginIT extends OpenSearchRestTestCase {

public void testPluginInstalled() throws IOException, ParseException {
Response response = client().performRequest(new Request("GET", "/_cat/plugins"));
String body = EntityUtils.toString(response.getEntity());

logger.info("response body: {}", body);
assertNotNull(body);
assertTrue(body.contains("amazon-kendra-intelligent-ranking"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import org.apache.lucene.search.TotalHits;
import org.mockito.Mockito;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.settings.get.GetSettingsAction;
import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
Expand All @@ -24,6 +23,7 @@
import org.opensearch.action.search.ShardSearchFailure;
import org.opensearch.action.support.ActionFilterChain;
import org.opensearch.client.Client;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.document.DocumentField;
import org.opensearch.core.common.io.stream.StreamInput;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
package org.opensearch.search.relevance;

import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;


public class AmazonKendraIntelligentRankingClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {

public AmazonKendraIntelligentRankingClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}

@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return OpenSearchClientYamlSuiteTestCase.createParameters();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"Test that the plugin is loaded in OpenSearch":
- do:
cat.plugins:
local: true
h: component

- match:
$body: /^opensearch-amazon-kendra-intelligent-ranking-\d+.\d+.\d+.\d+\n$/

- do:
indices.create:
index: test

- do:
search:
index: test
body: { }
Loading

0 comments on commit 8131021

Please sign in to comment.