REST API for GET,PUT and DELETE

Signed-off-by: vamsi-amazon <reddyvam@amazon.com>
opensearch-project · Mar 29, 2023 · 4b1cd49 · 4b1cd49
1 parent 12bc2b4
commit 4b1cd49
Show file tree

Hide file tree

Showing 27 changed files with 1,168 additions and 95 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java
@@ -27,9 +27,21 @@ public interface DataSourceService {
    * Returns all dataSource Metadata objects. The returned objects won't contain
    * any of the credential info.
    *
+   * @param isDefaultDataSourceRequired is used to specify
+   *      if default opensearch connector is required in the output list.
    * @return set of {@link DataSourceMetadata}.
    */
-  Set<DataSourceMetadata> getDataSourceMetadataSet();
+  Set<DataSourceMetadata> getDataSourceMetadataSet(boolean isDefaultDataSourceRequired);
+
+
+  /**
+   * Returns dataSourceMetadata object with specific name.
+   * The returned objects won't contain any crendetial info.
+   *
+   * @param name name of the {@link DataSource}.
+   * @return set of {@link DataSourceMetadata}.
+   */
+  DataSourceMetadata getDataSourceMetadataSet(String name);
 
   /**
    * Register {@link DataSource} defined by {@link DataSourceMetadata}.

diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java
@@ -9,6 +9,8 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -18,6 +20,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import org.opensearch.sql.common.utils.StringUtils;
+import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.datasource.model.DataSourceType;
@@ -59,12 +62,25 @@ public DataSourceServiceImpl(Set<DataSourceFactory> dataSourceFactories,
   }
 
   @Override
-  public Set<DataSourceMetadata> getDataSourceMetadataSet() {
+  public Set<DataSourceMetadata> getDataSourceMetadataSet(boolean isDefaultDataSourceRequired) {
     List<DataSourceMetadata> dataSourceMetadataList
         = this.dataSourceMetadataStorage.getDataSourceMetadata();
     Set<DataSourceMetadata> dataSourceMetadataSet = new HashSet<>(dataSourceMetadataList);
-    dataSourceMetadataSet.add(DataSourceMetadata.defaultOpenSearchDataSourceMetadata());
-    return dataSourceMetadataSet;
+    if (isDefaultDataSourceRequired) {
+      dataSourceMetadataSet.add(DataSourceMetadata.defaultOpenSearchDataSourceMetadata());
+    }
+    return removeAuthInfo(dataSourceMetadataSet);
+  }
+
+  @Override
+  public DataSourceMetadata getDataSourceMetadataSet(String datasourceName) {
+    Optional<DataSourceMetadata> dataSourceMetadataOptional
+        = this.dataSourceMetadataStorage.getDataSourceMetadata(datasourceName);
+    if (dataSourceMetadataOptional.isEmpty()) {
+      throw new IllegalArgumentException("DataSource with name: " + datasourceName
+          + " doesn't exist.");
+    }
+    return removeAuthInfo(dataSourceMetadataOptional.get());
   }
 
 
@@ -73,7 +89,7 @@ public DataSource getDataSource(String dataSourceName) {
     Optional<DataSourceMetadata>
         dataSourceMetadataOptional = getDataSourceMetadata(dataSourceName);
     if (dataSourceMetadataOptional.isEmpty()) {
-      throw new IllegalArgumentException(
+      throw new DataSourceNotFoundException(
           String.format("DataSource with name %s doesn't exist.", dataSourceName));
     } else {
       DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get();
@@ -95,12 +111,28 @@ public void createDataSource(DataSourceMetadata metadata) {
 
   @Override
   public void updateDataSource(DataSourceMetadata dataSourceMetadata) {
-    throw new UnsupportedOperationException("will be supported in future");
+    validateDataSourceMetaData(dataSourceMetadata);
+    if (!dataSourceMetadata.getName().equals(DEFAULT_DATASOURCE_NAME)) {
+      this.dataSourceMetadataStorage.updateDataSourceMetadata(dataSourceMetadata);
+      clearDataSourceWithSameName(dataSourceMetadata.getName());
+      dataSourceMap.put(dataSourceMetadata,
+          dataSourceFactoryMap.get(dataSourceMetadata.getConnector())
+              .createDataSource(dataSourceMetadata));
+    } else {
+      throw new UnsupportedOperationException(
+          "Not allowed to update default datasource :" + DEFAULT_DATASOURCE_NAME);
+    }
   }
 
   @Override
   public void deleteDataSource(String dataSourceName) {
-    throw new UnsupportedOperationException("will be supported in future");
+    if (dataSourceName.equals(DEFAULT_DATASOURCE_NAME)) {
+      throw new UnsupportedOperationException(
+          "Not allowed to delete default datasource :" + DEFAULT_DATASOURCE_NAME);
+    } else {
+      this.dataSourceMetadataStorage.deleteDataSourceMetadata(dataSourceName);
+      clearDataSourceWithSameName(dataSourceName);
+    }
   }
 
   @Override
@@ -128,6 +160,8 @@ private void validateDataSourceMetaData(DataSourceMetadata metadata) {
         !Objects.isNull(metadata.getProperties()),
         "Missing properties field in datasource configuration."
             + " Properties are required parameters.");
+    this.dataSourceFactoryMap.get(metadata.getConnector())
+        .validateDataSourceConfigProperties(metadata.getProperties());
   }
 
   private Optional<DataSourceMetadata> getDataSourceMetadata(String dataSourceName) {
@@ -140,17 +174,37 @@ private Optional<DataSourceMetadata> getDataSourceMetadata(String dataSourceName
 
   private DataSource getDataSourceFromMetadata(DataSourceMetadata dataSourceMetadata) {
     if (!dataSourceMap.containsKey(dataSourceMetadata)) {
-      clearDataSource(dataSourceMetadata);
+      clearDataSourceWithSameName(dataSourceMetadata.getName());
       dataSourceMap.put(dataSourceMetadata,
           dataSourceFactoryMap.get(dataSourceMetadata.getConnector())
               .createDataSource(dataSourceMetadata));
     }
     return dataSourceMap.get(dataSourceMetadata);
   }
 
-  private void clearDataSource(DataSourceMetadata dataSourceMetadata) {
+  private void clearDataSourceWithSameName(String dataSourceName) {
     dataSourceMap.entrySet()
-        .removeIf(entry -> entry.getKey().getName().equals(dataSourceMetadata.getName()));
+        .removeIf(entry -> entry.getKey().getName().equals(dataSourceName));
   }
 
+
+  private Set<DataSourceMetadata> removeAuthInfo(Set<DataSourceMetadata> dataSourceMetadataSet) {
+    return dataSourceMetadataSet.stream()
+        .map(this::removeAuthInfo)
+        .collect(Collectors.toSet());
+  }
+
+  private DataSourceMetadata removeAuthInfo(DataSourceMetadata dataSourceMetadata) {
+    HashMap<String, String> filteredPropertiesMap = new HashMap<>();
+    dataSourceMetadata.getProperties().entrySet()
+      .stream()
+      .filter(entry -> !entry.getKey().contains("auth"))
+      .forEach(entry -> filteredPropertiesMap.put(entry.getKey(), entry.getValue()));
+    DataSourceMetadata result = new DataSourceMetadata();
+    result.setName(dataSourceMetadata.getName());
+    result.setConnector(dataSourceMetadata.getConnector());
+    result.setAllowedRoles(new ArrayList<>(dataSourceMetadata.getAllowedRoles()));
+    result.setProperties(filteredPropertiesMap);
+    return result;
+  }
 }
diff --git a/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java b/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java
@@ -0,0 +1,18 @@
+/*
+ *
+ *  * Copyright OpenSearch Contributors
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.opensearch.sql.datasource.exceptions;
+
+/**
+ * DataSourceNotFoundException.
+ */
+public class DataSourceNotFoundException extends RuntimeException {
+  public DataSourceNotFoundException(String message) {
+    super(message);
+  }
+
+}
diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java
@@ -48,7 +48,7 @@ public String explain() {
   public void open() {
     List<ExprValue> exprValues = new ArrayList<>();
     Set<DataSourceMetadata> dataSourceMetadataSet
-        = dataSourceService.getDataSourceMetadataSet();
+        = dataSourceService.getDataSourceMetadataSet(true);
     for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataSet) {
       exprValues.add(
           new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(

diff --git a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java
@@ -7,6 +7,7 @@
 
 package org.opensearch.sql.storage;
 
+import java.util.Map;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
@@ -28,4 +29,12 @@ public interface DataSourceFactory {
    * Create {@link DataSource}.
    */
   DataSource createDataSource(DataSourceMetadata metadata);
+
+  /**
+   * Validate dataSourceMetadataConfig.
+   *
+   * @param dataSourceMetadataConfig dataSourceMetadataConfig.
+   */
+  void validateDataSourceConfigProperties(Map<String, String> dataSourceMetadataConfig);
+
 }
diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java
@@ -185,13 +185,18 @@ private class DefaultDataSourceService implements DataSourceService {
 
 
     @Override
-    public Set<DataSourceMetadata> getDataSourceMetadataSet() {
+    public Set<DataSourceMetadata> getDataSourceMetadataSet(boolean isDefaultDataSourceRequired) {
       return Stream.of(opensearchDataSource, prometheusDataSource)
           .map(ds -> new DataSourceMetadata(ds.getName(),
               ds.getConnectorType(),Collections.emptyList(),
               ImmutableMap.of())).collect(Collectors.toSet());
     }
 
+    @Override
+    public DataSourceMetadata getDataSourceMetadataSet(String name) {
+      return null;
+    }
+
     @Override
     public void createDataSource(DataSourceMetadata metadata) {
       throw new UnsupportedOperationException("unsupported operation");