Skip to content

Commit

Permalink
Merge first batch of feature/extensions into main (#5347)
Browse files Browse the repository at this point in the history
* Merge first batch of feature/extensions into main

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fixed CHANGELOG

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fixed newline errors

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Renaming and CHANGELOG fixes

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Refactor extension loading into private method

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Removed skipValidation and added connectToExtensionNode method

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Remove unnecessary feature flag calls

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Renaming and exception handling

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Change latches to CompletableFuture

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Removed unnecessary validateSettingKey call

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fix azure-core dependency

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Update SHAs

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Remove unintended dependency changes

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Removed dynamic settings regitration, removed info() method, and added NoopExtensionsManager

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Add javadoc

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fixed spotless failure

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Removed NoopExtensionsManager

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Added functioning NoopExtensionsManager

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Added missing javadoc

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Remove forbiddenAPI

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fix spotless

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Change logger.info to logger.error in handleException

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Fix ExtensionsManagerTests

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Removing unrelated change

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

* Update SHAs

Signed-off-by: Ryan Bogan <rbogan@amazon.com>

Signed-off-by: Ryan Bogan <rbogan@amazon.com>
  • Loading branch information
ryanbogan authored Dec 10, 2022
1 parent d4e5a28 commit e41cbe5
Show file tree
Hide file tree
Showing 29 changed files with 2,070 additions and 54 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
- Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
- Added jackson dependency to server ([#5366] (https://github.com/opensearch-project/OpenSearch/pull/5366))
- Added experimental extensions to main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))

### Dependencies
- Bumps `log4j-core` from 2.18.0 to 2.19.0
Expand Down
1 change: 0 additions & 1 deletion plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ dependencies {
api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
api 'org.apache.avro:avro:1.11.1'
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
api 'com.google.code.gson:gson:2.10'
runtimeOnly 'com.google.guava:guava:31.1-jre'
api 'com.google.protobuf:protobuf-java:3.21.9'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(waitForTimedOut);
}

@Override
public String toString() {
return "ClusterStateResponse{" + "clusterState=" + clusterState + '}';
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down
4 changes: 4 additions & 0 deletions server/src/main/java/org/opensearch/bootstrap/Security.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.opensearch.common.SuppressForbidden;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.env.Environment;
import org.opensearch.http.HttpTransportSettings;
import org.opensearch.plugins.PluginInfo;
Expand Down Expand Up @@ -316,6 +317,9 @@ static void addFilePermissions(Permissions policy, Environment environment) thro
addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libDir(), "read,readlink", false);
addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesDir(), "read,readlink", false);
addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsDir(), "read,readlink", false);
if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) {
addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.extensionDir(), "read,readlink", false);
}
addDirectoryPath(policy, "path.conf'", environment.configDir(), "read,readlink", false);
// read-write dirs
addDirectoryPath(policy, "java.io.tmpdir", environment.tmpDir(), "read,readlink,write,delete", false);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.cluster;

import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Settings;
import org.opensearch.transport.TransportResponse;

import java.io.IOException;
import java.util.Objects;

/**
* PluginSettings Response for Extensibility
*
* @opensearch.internal
*/
public class ClusterSettingsResponse extends TransportResponse {
private final Settings clusterSettings;

public ClusterSettingsResponse(ClusterService clusterService) {
this.clusterSettings = clusterService.getSettings();
}

public ClusterSettingsResponse(StreamInput in) throws IOException {
super(in);
this.clusterSettings = Settings.readSettingsFromStream(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
Settings.writeSettingsToStream(clusterSettings, out);
}

@Override
public String toString() {
return "ClusterSettingsResponse{" + "clusterSettings=" + clusterSettings + '}';
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ClusterSettingsResponse that = (ClusterSettingsResponse) o;
return Objects.equals(clusterSettings, that.clusterSettings);
}

@Override
public int hashCode() {
return Objects.hash(clusterSettings);
}

}
60 changes: 60 additions & 0 deletions server/src/main/java/org/opensearch/cluster/LocalNodeResponse.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.cluster;

import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.transport.TransportResponse;

import java.io.IOException;
import java.util.Objects;

/**
* LocalNode Response for Extensibility
*
* @opensearch.internal
*/
public class LocalNodeResponse extends TransportResponse {
private final DiscoveryNode localNode;

public LocalNodeResponse(ClusterService clusterService) {
this.localNode = clusterService.localNode();
}

public LocalNodeResponse(StreamInput in) throws IOException {
super(in);
this.localNode = new DiscoveryNode(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
this.localNode.writeTo(out);
}

@Override
public String toString() {
return "LocalNodeResponse{" + "localNode=" + localNode + '}';
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
LocalNodeResponse that = (LocalNodeResponse) o;
return Objects.equals(localNode, that.localNode);
}

@Override
public int hashCode() {
return Objects.hash(localNode);
}

}
76 changes: 76 additions & 0 deletions server/src/main/java/org/opensearch/discovery/PluginRequest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.discovery;

import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.extensions.DiscoveryExtensionNode;
import org.opensearch.transport.TransportRequest;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

/**
* PluginRequest to intialize plugin
*
* @opensearch.internal
*/
public class PluginRequest extends TransportRequest {
private final DiscoveryNode sourceNode;
/*
* TODO change DiscoveryNode to Extension information
*/
private final List<DiscoveryExtensionNode> extensions;

public PluginRequest(DiscoveryNode sourceNode, List<DiscoveryExtensionNode> extensions) {
this.sourceNode = sourceNode;
this.extensions = extensions;
}

public PluginRequest(StreamInput in) throws IOException {
super(in);
sourceNode = new DiscoveryNode(in);
extensions = in.readList(DiscoveryExtensionNode::new);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
sourceNode.writeTo(out);
out.writeList(extensions);
}

public List<DiscoveryExtensionNode> getExtensions() {
return extensions;
}

public DiscoveryNode getSourceNode() {
return sourceNode;
}

@Override
public String toString() {
return "PluginRequest{" + "sourceNode=" + sourceNode + ", extensions=" + extensions + '}';
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PluginRequest that = (PluginRequest) o;
return Objects.equals(sourceNode, that.sourceNode) && Objects.equals(extensions, that.extensions);
}

@Override
public int hashCode() {
return Objects.hash(sourceNode, extensions);
}
}
88 changes: 88 additions & 0 deletions server/src/main/java/org/opensearch/discovery/PluginResponse.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

package org.opensearch.discovery;

import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.transport.TransportResponse;

import java.io.IOException;
import java.util.Objects;

/**
* PluginResponse to intialize plugin
*
* @opensearch.internal
*/
public class PluginResponse extends TransportResponse {
private String name;

public PluginResponse(String name) {
this.name = name;
}

public PluginResponse(StreamInput in) throws IOException {
name = in.readString();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
}

/**
* @return the node that is currently leading, according to the responding node.
*/

public String getName() {
return this.name;
}

@Override
public String toString() {
return "PluginResponse{" + "name" + name + "}";
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PluginResponse that = (PluginResponse) o;
return Objects.equals(name, that.name);
}

@Override
public int hashCode() {
return Objects.hash(name);
}
}
7 changes: 7 additions & 0 deletions server/src/main/java/org/opensearch/env/Environment.java
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ public class Environment {

private final Path pluginsDir;

private final Path extensionsDir;

private final Path modulesDir;

private final Path sharedDataDir;
Expand Down Expand Up @@ -137,6 +139,7 @@ public Environment(final Settings settings, final Path configPath, final boolean
tmpDir = Objects.requireNonNull(tmpPath);

pluginsDir = homeFile.resolve("plugins");
extensionsDir = homeFile.resolve("extensions");

List<String> dataPaths = PATH_DATA_SETTING.get(settings);
if (nodeLocalStorage) {
Expand Down Expand Up @@ -308,6 +311,10 @@ public Path pluginsDir() {
return pluginsDir;
}

public Path extensionDir() {
return extensionsDir;
}

public Path binDir() {
return binDir;
}
Expand Down
Loading

0 comments on commit e41cbe5

Please sign in to comment.