KNOX-2963 - CM service discovery should work when legacy mode is turn…

…ed off (#801)
apache · Oct 11, 2023 · 838dcd8 · 838dcd8
1 parent ff6bcbc
commit 838dcd8
Show file tree

Hide file tree

Showing 39 changed files with 162 additions and 132 deletions.
diff --git a/...n/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java b/...n/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
@@ -270,6 +270,9 @@ private ClouderaManagerCluster discoverCluster(DiscoveryApiClient client, String
       cmService.setType(CM_SERVICE_TYPE);
       serviceList.add(cmService);
 
+      // if Legacy Cloudera Manager API Clients Compatibility is turned off, some HDFS settings are in CORE_SETTINGS
+      ApiServiceConfig coreSettingsConfig = coreSettingsConfig(client, servicesResourceApi, serviceList);
+
       for (ApiService service : serviceList) {
         final List<ServiceModelGenerator> modelGenerators = serviceModelGeneratorsHolder.getServiceModelGenerators(service.getType());
         if (shouldSkipServiceDiscovery(modelGenerators, includedServices)) {
@@ -299,7 +302,7 @@ private ClouderaManagerCluster discoverCluster(DiscoveryApiClient client, String
                 ServiceModelGeneratorHandleResponse response = serviceModelGenerator.handles(service, serviceConfig, role, roleConfig);
                 if (response.handled()) {
                   serviceModelGenerator.setApiClient(client);
-                  ServiceModel serviceModel = serviceModelGenerator.generateService(service, serviceConfig, role, roleConfig);
+                  ServiceModel serviceModel = serviceModelGenerator.generateService(service, serviceConfig, role, roleConfig, coreSettingsConfig);
                   serviceModels.add(serviceModel);
                 } else if (!response.getConfigurationIssues().isEmpty()) {
                   log.serviceRoleHasConfigurationIssues(roleName, String.join(";", response.getConfigurationIssues()));
@@ -320,6 +323,15 @@ private ClouderaManagerCluster discoverCluster(DiscoveryApiClient client, String
     return null;
   }
 
+  private ApiServiceConfig coreSettingsConfig(DiscoveryApiClient client, ServicesResourceApi servicesResourceApi, List<ApiService> serviceList) throws ApiException {
+    for (ApiService service : serviceList) {
+      if (CORE_SETTINGS_TYPE.equals(service.getType())) {
+        return getServiceConfig(client.getConfig(), servicesResourceApi, service);
+      }
+    }
+    return null;
+  }
+
   private boolean shouldSkipServiceDiscovery(List<ServiceModelGenerator> modelGenerators, Collection<String> includedServices) {
     if (includedServices == null || includedServices.isEmpty()) {
       // per the contract of org.apache.knox.gateway.topology.discovery.ServiceDiscovery.discover(GatewayConfig, ServiceDiscoveryConfig, String, Collection<String>):

diff --git a/...cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ServiceModelGenerator.java b/...cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ServiceModelGenerator.java
@@ -45,6 +45,5 @@ public interface ServiceModelGenerator {
 
   ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig);
 
-  ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig) throws ApiException;
-
+  ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException;
 }
diff --git a/...va/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGenerator.java b/...va/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGenerator.java
@@ -86,4 +86,15 @@ public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServic
     return new ServiceModelGeneratorHandleResponse(getServiceType().equals(service.getType()) && getRoleType().equals(role.getType()));
   }
 
+  protected String getCoreOrServiceConfig(ApiServiceConfig serviceConfig, ApiServiceConfig coreSettingsConfig, String configName) {
+    if (coreSettingsConfig != null) {
+      // in Legacy Cloudera Manager API Clients Compatibility certain configs,
+      // such as hdfs_hadoop_ssl_enabled are in the CORE_CONFIG
+      String configValue = getServiceConfigValue(coreSettingsConfig, configName);
+      if (configValue != null) {
+        return configValue;
+      }
+    }
+    return getServiceConfigValue(serviceConfig, configName); // Fall back to service config
+  }
 }
diff --git a/...org/apache/knox/gateway/topology/discovery/cm/model/atlas/AtlasServiceModelGenerator.java b/...org/apache/knox/gateway/topology/discovery/cm/model/atlas/AtlasServiceModelGenerator.java
@@ -56,10 +56,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port;

diff --git a/.../knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java b/.../knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
@@ -71,7 +71,7 @@ public ServiceModel.Type getModelType() {
    */
   @Override
   public ServiceModel generateService(ApiService service,
-      ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig)
+                                 ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig)
       throws ApiException {
 
     final String basePath = getClient().getBasePath();

diff --git a/...org/apache/knox/gateway/topology/discovery/cm/model/flink/FlinkServiceModelGenerator.java b/...org/apache/knox/gateway/topology/discovery/cm/model/flink/FlinkServiceModelGenerator.java
@@ -59,7 +59,7 @@ public Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     final String hostname = role.getHostRef().getHostname();
     final String port = getRoleConfigValue(roleConfig, WEB_PORT);
     final boolean sslEnabled = Boolean.parseBoolean(getRoleConfigValue(roleConfig, SSL_ENABLED));

diff --git a/...g/apache/knox/gateway/topology/discovery/cm/model/hbase/HBaseUIServiceModelGenerator.java b/...g/apache/knox/gateway/topology/discovery/cm/model/hbase/HBaseUIServiceModelGenerator.java
@@ -55,10 +55,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port = getRoleConfigValue(roleConfig, MASTER_INFO_PORT); // TODO: Is there an SSL port, or is this property re-used?

diff --git a/.../apache/knox/gateway/topology/discovery/cm/model/hbase/WebHBaseServiceModelGenerator.java b/.../apache/knox/gateway/topology/discovery/cm/model/hbase/WebHBaseServiceModelGenerator.java
@@ -55,10 +55,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port = getRoleConfigValue(roleConfig, REST_SERVER_PORT);

diff --git a/...org/apache/knox/gateway/topology/discovery/cm/model/hdfs/HdfsUIServiceModelGenerator.java b/...org/apache/knox/gateway/topology/discovery/cm/model/hdfs/HdfsUIServiceModelGenerator.java
@@ -47,11 +47,13 @@ public ServiceModel.Type getModelType() {
   public ServiceModel generateService(ApiService       service,
                                       ApiServiceConfig serviceConfig,
                                       ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+                                      ApiConfigList    roleConfig,
+                                      ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port;
-    boolean sslEnabled = Boolean.parseBoolean(getServiceConfigValue(serviceConfig, SSL_ENABLED));
+    String sslEnabledConfig = getCoreOrServiceConfig(serviceConfig, coreSettingsConfig, SSL_ENABLED);
+    boolean sslEnabled = Boolean.parseBoolean(sslEnabledConfig);
     if(sslEnabled) {
       scheme = "https";
       port = getRoleConfigValue(roleConfig, HTTPS_PORT);
@@ -62,11 +64,11 @@ public ServiceModel generateService(ApiService       service,
     String namenodeUrl = String.format(Locale.getDefault(), "%s://%s:%s", scheme, hostname, port);
 
     ServiceModel model = createServiceModel(namenodeUrl);
-    model.addServiceProperty(SSL_ENABLED, getServiceConfigValue(serviceConfig, SSL_ENABLED));
+    model.addServiceProperty(SSL_ENABLED, sslEnabledConfig);
     model.addRoleProperty(role.getType(), HTTPS_PORT, getRoleConfigValue(roleConfig, HTTPS_PORT));
     model.addRoleProperty(role.getType(), HTTP_PORT, getRoleConfigValue(roleConfig, HTTP_PORT));
 
-    ServiceModel parent = super.generateService(service, serviceConfig, role, roleConfig);
+    ServiceModel parent = super.generateService(service, serviceConfig, role, roleConfig, coreSettingsConfig);
     addParentModelMetadata(model, parent);
 
     return model;

diff --git a/...g/apache/knox/gateway/topology/discovery/cm/model/hdfs/NameNodeServiceModelGenerator.java b/...g/apache/knox/gateway/topology/discovery/cm/model/hdfs/NameNodeServiceModelGenerator.java
@@ -61,7 +61,8 @@ public ServiceModel.Type getModelType() {
   public ServiceModel generateService(ApiService       service,
                                       ApiServiceConfig serviceConfig,
                                       ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+                                      ApiConfigList    roleConfig,
+                                      ApiServiceConfig coreSettingsConfig) throws ApiException {
     boolean haEnabled = Boolean.parseBoolean(getRoleConfigValue(roleConfig, AUTOFAILOVER_ENABLED));
     String serviceUrl;
     if (haEnabled) {

diff --git a/...rg/apache/knox/gateway/topology/discovery/cm/model/hdfs/WebHdfsServiceModelGenerator.java b/...rg/apache/knox/gateway/topology/discovery/cm/model/hdfs/WebHdfsServiceModelGenerator.java
@@ -55,8 +55,8 @@ public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServic
   }
 
   @Override
-  public ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig) throws ApiException {
-    ServiceModel parent = super.generateService(service, serviceConfig, role, roleConfig);
+  public ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
+    ServiceModel parent = super.generateService(service, serviceConfig, role, roleConfig, coreSettingsConfig);
     String serviceUrl = parent.getServiceUrl() + WEBHDFS_SUFFIX;
 
     ServiceModel model = createServiceModel(serviceUrl);

diff --git a/.../apache/knox/gateway/topology/discovery/cm/model/hive/HiveOnTezServiceModelGenerator.java b/.../apache/knox/gateway/topology/discovery/cm/model/hive/HiveOnTezServiceModelGenerator.java
@@ -49,11 +49,11 @@ protected String getHttpPort(ApiConfigList roleConfig) {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
-    ServiceModel model = super.generateService(service, serviceConfig, role, roleConfig);
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
+    ServiceModel model = super.generateService(service, serviceConfig, role, roleConfig, coreSettingsConfig);
     model.addRoleProperty(getRoleType(),
                           HIVEONTEZ_HTTP_PORT,
                           getRoleConfigValue(roleConfig, HIVEONTEZ_HTTP_PORT));

diff --git a/...a/org/apache/knox/gateway/topology/discovery/cm/model/hive/HiveServiceModelGenerator.java b/...a/org/apache/knox/gateway/topology/discovery/cm/model/hive/HiveServiceModelGenerator.java
@@ -74,10 +74,10 @@ public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServic
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
     boolean sslEnabled = Boolean.parseBoolean(getServiceConfigValue(serviceConfig, SSL_ENABLED));
     String scheme = sslEnabled ? "https" : "http";

diff --git a/...rg/apache/knox/gateway/topology/discovery/cm/model/hive/WebHCatServiceModelGenerator.java b/...rg/apache/knox/gateway/topology/discovery/cm/model/hive/WebHCatServiceModelGenerator.java
@@ -55,10 +55,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
     String port = getRoleConfigValue(roleConfig, WEBHCAT_PORT);
 

diff --git a/...a/org/apache/knox/gateway/topology/discovery/cm/model/hue/HueLBServiceModelGenerator.java b/...a/org/apache/knox/gateway/topology/discovery/cm/model/hue/HueLBServiceModelGenerator.java
@@ -54,10 +54,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme = "http";
     String port = getRoleConfigValue(roleConfig, LISTEN_PORT);

diff --git a/...ava/org/apache/knox/gateway/topology/discovery/cm/model/hue/HueServiceModelGenerator.java b/...ava/org/apache/knox/gateway/topology/discovery/cm/model/hue/HueServiceModelGenerator.java
@@ -55,10 +55,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port = getRoleConfigValue(roleConfig, HUE_HTTP_PORT);

diff --git a/...g/apache/knox/gateway/topology/discovery/cm/model/impala/ImpalaServiceModelGenerator.java b/...g/apache/knox/gateway/topology/discovery/cm/model/impala/ImpalaServiceModelGenerator.java
@@ -85,10 +85,10 @@ public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServic
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
 
     boolean sslEnabled = Boolean.parseBoolean(getServiceConfigValue(serviceConfig, SSL_ENABLED));

diff --git a/...apache/knox/gateway/topology/discovery/cm/model/impala/ImpalaUIServiceModelGenerator.java b/...apache/knox/gateway/topology/discovery/cm/model/impala/ImpalaUIServiceModelGenerator.java
@@ -72,10 +72,10 @@ public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServic
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
 
     String sslEnabled = getServiceConfigValue(serviceConfig, SSL_ENABLED);

diff --git a/...org/apache/knox/gateway/topology/discovery/cm/model/kudu/KuduUIServiceModelGenerator.java b/...org/apache/knox/gateway/topology/discovery/cm/model/kudu/KuduUIServiceModelGenerator.java
@@ -56,10 +56,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) throws ApiException {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
     String hostname = role.getHostRef().getHostname();
 
     String sslEnabled = getRoleConfigValue(roleConfig, SSL_ENABLED);

diff --git a/...a/org/apache/knox/gateway/topology/discovery/cm/model/livy/LivyServiceModelGenerator.java b/...a/org/apache/knox/gateway/topology/discovery/cm/model/livy/LivyServiceModelGenerator.java
@@ -55,10 +55,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port = getRoleConfigValue(roleConfig, LIVY_SERVER_PORT);

diff --git a/...ache/knox/gateway/topology/discovery/cm/model/nifi/NifiRegistryServiceModelGenerator.java b/...ache/knox/gateway/topology/discovery/cm/model/nifi/NifiRegistryServiceModelGenerator.java
@@ -69,7 +69,7 @@ public ServiceModel.Type getModelType() {
 
   @Override
   public ServiceModel generateService(ApiService service,
-      ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig)
+                                 ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig)
       throws ApiException {
     String hostname = role.getHostRef().getHostname();
     String scheme;

diff --git a/...a/org/apache/knox/gateway/topology/discovery/cm/model/nifi/NifiServiceModelGenerator.java b/...a/org/apache/knox/gateway/topology/discovery/cm/model/nifi/NifiServiceModelGenerator.java
@@ -69,7 +69,7 @@ public ServiceModel.Type getModelType() {
 
   @Override
   public ServiceModel generateService(ApiService service,
-      ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig)
+                                 ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig)
       throws ApiException {
     String hostname = role.getHostRef().getHostname();
     String scheme;

diff --git a/...org/apache/knox/gateway/topology/discovery/cm/model/oozie/OozieServiceModelGenerator.java b/...org/apache/knox/gateway/topology/discovery/cm/model/oozie/OozieServiceModelGenerator.java
@@ -56,10 +56,10 @@ public ServiceModel.Type getModelType() {
   }
 
   @Override
-  public ServiceModel generateService(ApiService       service,
-                                      ApiServiceConfig serviceConfig,
-                                      ApiRole          role,
-                                      ApiConfigList    roleConfig) {
+  public ServiceModel generateService(ApiService service,
+                                 ApiServiceConfig serviceConfig,
+                                 ApiRole role,
+                                 ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) {
     String hostname = role.getHostRef().getHostname();
     String scheme;
     String port;