From 14954a0f1614ab6c4d4120bf701b8f6f5f414a40 Mon Sep 17 00:00:00 2001 From: Sandor Molnar Date: Tue, 12 Dec 2023 14:49:08 +0100 Subject: [PATCH] KNOX-2956 - Removing CM-specific 'advanced service discovery' handler and have everything process by the HXR parser (#821) Change-Id: Ib1837610e4b222282af7bef98fc6f27af5169e88 --- .../apache/knox/gateway/GatewayServer.java | 5 - ...ServiceDiscoveryConfigurationMessages.java | 39 ------ .../AdvancedServiceDiscoveryConfig.java | 88 ------------- ...dServiceDiscoveryConfigChangeListener.java | 28 ----- ...dServiceDiscoveryConfigurationMonitor.java | 116 ------------------ .../hadoop/xml/HadoopXmlResourceMessages.java | 11 +- .../hadoop/xml/HadoopXmlResourceMonitor.java | 32 ++--- .../hadoop/xml/HadoopXmlResourceParser.java | 111 +++++------------ .../xml/HadoopXmlResourceParserTest.java | 80 +++--------- .../testDescriptorWithServiceList.xml | 34 +++++ 10 files changed, 96 insertions(+), 448 deletions(-) delete mode 100644 gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java delete mode 100644 gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java delete mode 100644 gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java delete mode 100644 gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java create mode 100644 gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java index ce62b1c796..01c04cf7fa 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java @@ -48,7 +48,6 @@ import org.apache.knox.gateway.topology.Topology; import org.apache.knox.gateway.topology.TopologyEvent; import org.apache.knox.gateway.topology.TopologyListener; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigurationMonitor; import org.apache.knox.gateway.topology.hadoop.xml.HadoopXmlResourceMonitor; import org.apache.knox.gateway.topology.hadoop.xml.HadoopXmlResourceParser; import org.apache.knox.gateway.trace.AccessHandler; @@ -719,10 +718,6 @@ public void run() { private void handleHadoopXmlResources() { final HadoopXmlResourceParser hadoopXmlResourceParser = new HadoopXmlResourceParser(config); final HadoopXmlResourceMonitor hadoopXmlResourceMonitor = new HadoopXmlResourceMonitor(config, hadoopXmlResourceParser); - final AdvancedServiceDiscoveryConfigurationMonitor advancedServiceDiscoveryConfigurationMonitor = new AdvancedServiceDiscoveryConfigurationMonitor(config); - advancedServiceDiscoveryConfigurationMonitor.registerListener(hadoopXmlResourceParser); - advancedServiceDiscoveryConfigurationMonitor.registerListener(hadoopXmlResourceMonitor); - advancedServiceDiscoveryConfigurationMonitor.init(); hadoopXmlResourceMonitor.setupMonitor(); } diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java deleted file mode 100644 index b2a4f6316e..0000000000 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvanceServiceDiscoveryConfigurationMessages.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.knox.gateway.topology.discovery.advanced; - -import org.apache.knox.gateway.i18n.messages.Message; -import org.apache.knox.gateway.i18n.messages.MessageLevel; -import org.apache.knox.gateway.i18n.messages.Messages; -import org.apache.knox.gateway.i18n.messages.StackTrace; - -@Messages(logger = "org.apache.knox.gateway.topology.discovery.advanced") -public interface AdvanceServiceDiscoveryConfigurationMessages { - - @Message(level = MessageLevel.INFO, text = "Monitoring {0}/{1}* for advanced service discovery configuration changes.") - void monitorStarted(String directory, String prefix); - - @Message(level = MessageLevel.INFO, text = "Monitoring advanced service discovery configuration changes is disabled.") - void disableMonitoring(); - - @Message(level = MessageLevel.ERROR, text = "Error while monitoring CM advanced configuration: {0}") - void failedToMonitorClouderaManagerAdvancedConfiguration(String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e); - - @Message(level = MessageLevel.INFO, text = "Notifying listeners due to advanced service discovery configuration changes in {0}...") - void notifyListeners(String path); - -} diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java deleted file mode 100644 index c463bea689..0000000000 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfig.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.knox.gateway.topology.discovery.advanced; - -import static java.util.stream.Collectors.toSet; - -import java.util.Locale; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; - -/** - * Wrapper class providing useful methods on properties coming from - * $KNOX_CONF_DIR/auto-discovery-advanced-configuration.properties - */ -public class AdvancedServiceDiscoveryConfig { - - public static final String PARAMETER_NAME_PREFIX_ENABLED_SERVICE = "gateway.auto.discovery."; - public static final String PARAMETER_NAME_POSTFIX_ENABLED_SERVICE = ".enabled."; - public static final String PARAMETER_NAME_TOPOLOGY_NAME = "gateway.auto.discovery.topology.name"; - public static final String PARAMETER_NAME_DISCOVERY_ADDRESS = "gateway.auto.discovery.address"; - public static final String PARAMETER_NAME_DISCOVERY_CLUSTER = "gateway.auto.discovery.cluster"; - - private final Properties properties; - - public AdvancedServiceDiscoveryConfig() { - this(null); - } - - public AdvancedServiceDiscoveryConfig(Properties properties) { - this.properties = properties == null ? new Properties() : properties; - } - - public boolean isServiceEnabled(String serviceName) { - final String propertyName = PARAMETER_NAME_PREFIX_ENABLED_SERVICE + getTopologyName() + PARAMETER_NAME_POSTFIX_ENABLED_SERVICE + serviceName; - return Boolean.valueOf(getPropertyIgnoreCase(propertyName, "true")); - } - - public Set getEnabledServiceNames() { - return properties.entrySet().stream().filter(keyValuePair -> Boolean.valueOf((String) keyValuePair.getValue())) - .map(keyValuePair -> ((String) keyValuePair.getKey()).substring(((String) keyValuePair.getKey()).lastIndexOf('.') + 1).toUpperCase(Locale.getDefault())).collect(toSet()); - } - - public String getTopologyName() { - return getPropertyIgnoreCase(PARAMETER_NAME_TOPOLOGY_NAME, ""); - } - - public String getDiscoveryAddress() { - return getPropertyIgnoreCase(PARAMETER_NAME_DISCOVERY_ADDRESS, ""); - } - - public String getDiscoveryCluster() { - return getPropertyIgnoreCase(PARAMETER_NAME_DISCOVERY_CLUSTER, ""); - } - - private String getPropertyIgnoreCase(String propertyName, String defaultValue) { - final String property = properties.getProperty(propertyName); - if (property != null) { - return property; - } else { - for (Entry entry : properties.entrySet()) { - if (propertyName.equalsIgnoreCase((String) entry.getKey())) { - return (String) entry.getValue(); - } - } - return defaultValue; - } - } - - @Override - public String toString() { - return this.properties.toString(); - } -} diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java deleted file mode 100644 index 21061ecd24..0000000000 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigChangeListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.knox.gateway.topology.discovery.advanced; - -import java.util.Properties; - -/** - * The listener interface for receiving service discovery configuration events. - */ -public interface AdvancedServiceDiscoveryConfigChangeListener { - - void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration); - -} diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java deleted file mode 100644 index c314a4d525..0000000000 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/discovery/advanced/AdvancedServiceDiscoveryConfigurationMonitor.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.knox.gateway.topology.discovery.advanced; - -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.io.filefilter.PrefixFileFilter; -import org.apache.commons.lang3.concurrent.BasicThreadFactory; -import org.apache.knox.gateway.config.GatewayConfig; -import org.apache.knox.gateway.i18n.messages.MessagesFactory; - -/** - * Monitoring $KNOX_CONF_DIR/auto-discovery-advanced-configuration.properties (if exists) and notifies any - * {@link AdvancedServiceDiscoveryConfigChangeListener} if the file is changed since the last time it was loaded - * - */ -public class AdvancedServiceDiscoveryConfigurationMonitor { - - private static final String ADVANCED_CONFIGURATION_FILE_NAME_PREFIX = "auto-discovery-advanced-configuration-"; - private static final AdvanceServiceDiscoveryConfigurationMessages LOG = MessagesFactory.get(AdvanceServiceDiscoveryConfigurationMessages.class); - - private final List listeners; - private final String gatewayConfigurationDir; - private final long monitoringInterval; - private final Map lastReloadTimes; - - public AdvancedServiceDiscoveryConfigurationMonitor(GatewayConfig gatewayConfig) { - this.gatewayConfigurationDir = gatewayConfig.getGatewayConfDir(); - this.monitoringInterval = gatewayConfig.getClouderaManagerAdvancedServiceDiscoveryConfigurationMonitoringInterval(); - this.listeners = new ArrayList<>(); - this.lastReloadTimes = new ConcurrentHashMap<>(); - } - - public void init() { - monitorAdvancedServiceConfigurations(); - setupMonitor(); - } - - private void setupMonitor() { - if (monitoringInterval > 0) { - final ScheduledExecutorService executorService = newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("AdvancedServiceDiscoveryConfigurationMonitor-%d").build()); - executorService.scheduleAtFixedRate(() -> monitorAdvancedServiceConfigurations(), 0, monitoringInterval, TimeUnit.MILLISECONDS); - LOG.monitorStarted(gatewayConfigurationDir, ADVANCED_CONFIGURATION_FILE_NAME_PREFIX); - } else { - LOG.disableMonitoring(); - } - } - - public void registerListener(AdvancedServiceDiscoveryConfigChangeListener listener) { - listeners.add(listener); - } - - private void monitorAdvancedServiceConfigurations() { - final File[] advancedConfigurationFiles = new File(gatewayConfigurationDir).listFiles((FileFilter) new PrefixFileFilter(ADVANCED_CONFIGURATION_FILE_NAME_PREFIX)); - if (advancedConfigurationFiles != null) { - for (File advancedConfigurationFile : advancedConfigurationFiles) { - monitorAdvancedServiceConfiguration(Paths.get(advancedConfigurationFile.getAbsolutePath())); - } - } - } - - private void monitorAdvancedServiceConfiguration(Path resourcePath) { - try { - if (Files.exists(resourcePath) && Files.isReadable(resourcePath)) { - FileTime lastModifiedTime = Files.getLastModifiedTime(resourcePath); - FileTime lastReloadTime = lastReloadTimes.get(resourcePath); - if (lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) { - lastReloadTimes.put(resourcePath, lastModifiedTime); - try (InputStream advanceconfigurationFileInputStream = Files.newInputStream(resourcePath)) { - Properties properties = new Properties(); - properties.load(advanceconfigurationFileInputStream); - notifyListeners(resourcePath.toString(), properties); - } - } - } - } catch (IOException e) { - LOG.failedToMonitorClouderaManagerAdvancedConfiguration(e.getMessage(), e); - } - } - - private void notifyListeners(String path, Properties properties) { - LOG.notifyListeners(path); - listeners.forEach(listener -> listener.onAdvancedServiceDiscoveryConfigurationChange(properties)); - } - -} diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java index 063fbad4b2..67ea33c10d 100644 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java +++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMessages.java @@ -56,11 +56,11 @@ public interface HadoopXmlResourceMessages { @Message(level = MessageLevel.ERROR, text = "Parsing XML configuration {0} failed: {1}") void failedToParseXmlConfiguration(String path, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e); - @Message(level = MessageLevel.DEBUG, text = "Processing Hadoop XML resource {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})") - void processHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime); + @Message(level = MessageLevel.DEBUG, text = "Processing Hadoop XML resource {0} (lastReloadTime = {1}; lastModified = {2})") + void processHadoopXmlResource(String descriptorPath, FileTime lastReloadTime, FileTime lastModifiedTime); - @Message(level = MessageLevel.DEBUG, text = "Skipping Hadoop XML resource monitoring of {0} (force = {1}; lastReloadTime = {2}; lastModified = {3})") - void skipMonitorHadoopXmlResource(String descriptorPath, boolean force, FileTime lastReloadTime, FileTime lastModifiedTime); + @Message(level = MessageLevel.DEBUG, text = "Skipping Hadoop XML resource monitoring of {0} (lastReloadTime = {1}; lastModified = {2})") + void skipMonitorHadoopXmlResource(String descriptorPath, FileTime lastReloadTime, FileTime lastModifiedTime); @Message(level = MessageLevel.ERROR, text = "Error while monitoring Hadoop style XML configuration {0}: {1}") void failedToMonitorHadoopXmlResource(String descriptorPath, String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e); @@ -71,9 +71,6 @@ public interface HadoopXmlResourceMessages { @Message(level = MessageLevel.ERROR, text = "Error while producing Knox provider: {0}") void failedToProduceKnoxProvider(String errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e); - @Message(level = MessageLevel.WARN, text = "Service {0} is disabled. It will NOT be added in {1}") - void serviceDisabled(String serviceName, String descriptorName); - @Message(level = MessageLevel.INFO, text = "Updated advanced service discovery configuration for {0}.") void updatedAdvanceServiceDiscoverytConfiguration(String topologyName); diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java index f77d419f4f..8ed729b606 100644 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java +++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceMonitor.java @@ -25,9 +25,8 @@ import java.nio.file.Paths; import java.nio.file.attribute.FileTime; import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -36,19 +35,16 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.SuffixFileFilter; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigChangeListener; import org.apache.knox.gateway.util.JsonUtils; /** * Monitoring KNOX_DESCRIPTOR_DIR for *.hxr files - which is a Hadoop XML configuration - and processing those files if they were modified * since the last time it they were processed */ -public class HadoopXmlResourceMonitor implements AdvancedServiceDiscoveryConfigChangeListener { +public class HadoopXmlResourceMonitor { private static final String HADOOP_XML_RESOURCE_FILE_EXTENSION = ".hxr"; private static final HadoopXmlResourceMessages LOG = MessagesFactory.get(HadoopXmlResourceMessages.class); @@ -72,31 +68,31 @@ public HadoopXmlResourceMonitor(GatewayConfig gatewayConfig, HadoopXmlResourcePa public void setupMonitor() { if (monitoringInterval > 0) { final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("ClouderaManagerDescriptorMonitor-%d").build()); - monitorClouderaManagerDescriptors(false); // call it explicitly first to generate descriptors up front, so that the health checker can pick them up - executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(false), monitoringInterval, monitoringInterval, TimeUnit.MILLISECONDS); + monitorClouderaManagerDescriptors(); // call it explicitly first to generate descriptors up front, so that the health checker can pick them up + executorService.scheduleAtFixedRate(() -> monitorClouderaManagerDescriptors(), monitoringInterval, monitoringInterval, TimeUnit.MILLISECONDS); LOG.monitoringHadoopXmlResources(descriptorsDir); } } - private void monitorClouderaManagerDescriptors(boolean force) { + private void monitorClouderaManagerDescriptors() { final File[] clouderaManagerDescriptorFiles = new File(descriptorsDir).listFiles((FileFilter) new SuffixFileFilter(HADOOP_XML_RESOURCE_FILE_EXTENSION)); for (File clouderaManagerDescriptorFile : clouderaManagerDescriptorFiles) { - monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath()), force); + monitorClouderaManagerDescriptor(Paths.get(clouderaManagerDescriptorFile.getAbsolutePath())); } } - private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile, boolean force) { + private void monitorClouderaManagerDescriptor(Path clouderaManagerDescriptorFile) { monitorLock.lock(); try { if (Files.isReadable(clouderaManagerDescriptorFile)) { final FileTime lastModifiedTime = Files.getLastModifiedTime(clouderaManagerDescriptorFile); FileTime lastReloadTime = lastReloadTimes.get(clouderaManagerDescriptorFile); - if (force || lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) { + if (lastReloadTime == null || lastReloadTime.compareTo(lastModifiedTime) < 0) { lastReloadTimes.put(clouderaManagerDescriptorFile, lastModifiedTime); - LOG.processHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime); + LOG.processHadoopXmlResource(clouderaManagerDescriptorFile.toString(), lastReloadTime, lastModifiedTime); processClouderaManagerDescriptor(clouderaManagerDescriptorFile.toString()); } else { - LOG.skipMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), force, lastReloadTime, lastModifiedTime); + LOG.skipMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), lastReloadTime, lastModifiedTime); } } else { LOG.failedToMonitorHadoopXmlResource(clouderaManagerDescriptorFile.toString(), "File is not readable!", null); @@ -169,12 +165,4 @@ private boolean isResourceChangedOrNew(File knoxDescriptorFile, String simpleDes return true; } - @Override - public void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration) { - final String topologyName = new AdvancedServiceDiscoveryConfig(newConfiguration).getTopologyName(); - if (StringUtils.isBlank(topologyName)) { - throw new IllegalArgumentException("Invalid advanced service discovery configuration: topology name is missing!"); - } - monitorClouderaManagerDescriptors(true); - } } diff --git a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java index a66c36361b..75f00fe626 100644 --- a/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java +++ b/gateway-topology-hadoop-xml/src/main/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParser.java @@ -23,20 +23,16 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfigChangeListener; import org.apache.knox.gateway.topology.simple.JSONProviderConfiguration; import org.apache.knox.gateway.topology.simple.JSONProviderConfiguration.JSONProvider; import org.apache.knox.gateway.topology.simple.ProviderConfiguration; @@ -50,7 +46,7 @@ * Parses Knox descriptors and provider configurations from Hadoop style XML config * */ -public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigChangeListener { +public class HadoopXmlResourceParser { private static final HadoopXmlResourceMessages log = MessagesFactory.get(HadoopXmlResourceMessages.class); //shared provider related constants @@ -71,15 +67,14 @@ public class HadoopXmlResourceParser implements AdvancedServiceDiscoveryConfigCh private static final String CONFIG_NAME_APPLICATION_PREFIX = "app"; private static final String CONFIG_NAME_SERVICE_URL = "url"; private static final String CONFIG_NAME_SERVICE_VERSION = "version"; + private static final String CONFIG_NAME_SERVICES = "services"; private static final String REMOVE = "remove"; - private final Map advancedServiceDiscoveryConfigMap; private final String sharedProvidersDir; private final GatewayConfig gatewayConfig; public HadoopXmlResourceParser(GatewayConfig gatewayConfig) { this.gatewayConfig = gatewayConfig; - this.advancedServiceDiscoveryConfigMap = new ConcurrentHashMap<>(); this.sharedProvidersDir = gatewayConfig.getGatewayProvidersConfigDir(); } @@ -270,6 +265,9 @@ private SimpleDescriptor parseXmlDescriptor(String name, String xmlValue) { case CONFIG_NAME_PROVISION_ENCRYPT_QUERY_STRING_CREDENTIAL: descriptor.setProvisionEncryptQueryStringCredential(Boolean.valueOf(parameterPairParts[1].trim())); break; + case CONFIG_NAME_SERVICES: + Arrays.asList(parameterPairParts[1].trim().split(",", -1)).forEach(serviceName -> descriptor.addService(emptyService(serviceName))); + break; default: if (parameterName.startsWith(CONFIG_NAME_APPLICATION_PREFIX)) { parseApplication(descriptor, configurationPair.trim()); @@ -280,11 +278,6 @@ private SimpleDescriptor parseXmlDescriptor(String name, String xmlValue) { } } - final AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig = advancedServiceDiscoveryConfigMap.get(name); - if (advancedServiceDiscoveryConfig != null) { - setDiscoveryDetails(descriptor, advancedServiceDiscoveryConfig); - addEnabledServices(descriptor, advancedServiceDiscoveryConfig); - } return descriptor; } catch (Exception e) { log.failedToParseDescriptor(name, e.getMessage(), e); @@ -292,31 +285,10 @@ private SimpleDescriptor parseXmlDescriptor(String name, String xmlValue) { } } - private void setDiscoveryDetails(SimpleDescriptorImpl descriptor, AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig) { - if (StringUtils.isBlank(descriptor.getDiscoveryAddress())) { - descriptor.setDiscoveryAddress(advancedServiceDiscoveryConfig.getDiscoveryAddress()); - } - - if (StringUtils.isBlank(descriptor.getCluster())) { - descriptor.setCluster(advancedServiceDiscoveryConfig.getDiscoveryCluster()); - } - - if (StringUtils.isBlank(descriptor.getDiscoveryType())) { - descriptor.setDiscoveryType("ClouderaManager"); - } - } - - /* - * Adds any enabled service which is not listed in the CM descriptor - */ - private void addEnabledServices(SimpleDescriptorImpl descriptor, AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig) { - advancedServiceDiscoveryConfig.getEnabledServiceNames().forEach(enabledServiceName -> { - if (descriptor.getService(enabledServiceName) == null) { - ServiceImpl service = new ServiceImpl(); - service.setName(enabledServiceName); - descriptor.addService(service); - } - }); + private ServiceImpl emptyService(String serviceName) { + final ServiceImpl service = new ServiceImpl(); + service.setName(serviceName.trim().toUpperCase(Locale.ROOT)); + return service; } /** @@ -354,7 +326,7 @@ private void parseApplication(SimpleDescriptorImpl descriptor, String configurat *
  • $SERVICE_NAME:version=$VERSION (optional)
    • *
  • $SERVICE_NAME[:$PARAMETER_NAME=$PARAMETER_VALUE] (optional)
    • * - * Sample application configurations: + * Sample service configurations: *
      *
    • HIVE:url=http://localhost:123
      • *
    • HIVE:version=1.0
      • @@ -364,50 +336,31 @@ private void parseApplication(SimpleDescriptorImpl descriptor, String configurat private void parseService(SimpleDescriptorImpl descriptor, String configurationPair) { final String[] serviceParts = configurationPair.split(":"); final String serviceName = serviceParts[0].trim(); - if (isServiceEnabled(descriptor.getName(), serviceName)) { - ServiceImpl service = (ServiceImpl) descriptor.getService(serviceName); - if (service == null) { - service = new ServiceImpl(); - service.setName(serviceName); - descriptor.addService(service); - } - if (serviceParts.length > 1) { - // configuration value may contain ":" (for instance http://host:port) -> considering a configuration name/value pair everything after '$SERVICE_NAME:' - final String serviceConfiguration = configurationPair.substring(serviceName.length() + 1).trim(); - final String[] serviceConfigurationParts = serviceConfiguration.split("=", 2); - final String serviceConfigurationName = serviceConfigurationParts[0].trim(); - final String serviceConfigurationValue = serviceConfigurationParts[1].trim(); - switch (serviceConfigurationName) { - case CONFIG_NAME_SERVICE_URL: - service.addUrl(serviceConfigurationValue); - break; - case CONFIG_NAME_SERVICE_VERSION: - service.setVersion(serviceConfigurationValue); - break; - default: - service.addParam(serviceConfigurationName, serviceConfigurationValue); - break; - } - } - } else { - log.serviceDisabled(serviceName, descriptor.getName()); + ServiceImpl service = (ServiceImpl) descriptor.getService(serviceName); + if (service == null) { + service = emptyService(serviceName); + descriptor.addService(service); } - } - private boolean isServiceEnabled(String descriptorName, String serviceName) { - return advancedServiceDiscoveryConfigMap.containsKey(descriptorName) ? advancedServiceDiscoveryConfigMap.get(descriptorName).isServiceEnabled(serviceName) : true; - } - - @Override - public void onAdvancedServiceDiscoveryConfigurationChange(Properties newConfiguration) { - final AdvancedServiceDiscoveryConfig advancedServiceDiscoveryConfig = new AdvancedServiceDiscoveryConfig(newConfiguration); - final String topologyName = advancedServiceDiscoveryConfig.getTopologyName(); - if (StringUtils.isBlank(topologyName)) { - throw new IllegalArgumentException("Invalid advanced service discovery configuration: topology name is missing!"); + if (serviceParts.length > 1) { + // configuration value may contain ":" (for instance http://host:port) -> considering a configuration name/value pair everything after '$SERVICE_NAME:' + final String serviceConfiguration = configurationPair.substring(serviceName.length() + 1).trim(); + final String[] serviceConfigurationParts = serviceConfiguration.split("=", 2); + final String serviceConfigurationName = serviceConfigurationParts[0].trim(); + final String serviceConfigurationValue = serviceConfigurationParts[1].trim(); + switch (serviceConfigurationName) { + case CONFIG_NAME_SERVICE_URL: + service.addUrl(serviceConfigurationValue); + break; + case CONFIG_NAME_SERVICE_VERSION: + service.setVersion(serviceConfigurationValue); + break; + default: + service.addParam(serviceConfigurationName, serviceConfigurationValue); + break; + } } - advancedServiceDiscoveryConfigMap.put(topologyName, advancedServiceDiscoveryConfig); - log.updatedAdvanceServiceDiscoverytConfiguration(topologyName); } } diff --git a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java index a0e78f6194..3f80d91c4c 100644 --- a/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java +++ b/gateway-topology-hadoop-xml/src/test/java/org/apache/knox/gateway/topology/hadoop/xml/HadoopXmlResourceParserTest.java @@ -33,14 +33,12 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.apache.knox.gateway.config.GatewayConfig; -import org.apache.knox.gateway.topology.discovery.advanced.AdvancedServiceDiscoveryConfig; import org.apache.knox.gateway.topology.simple.ProviderConfiguration; import org.apache.knox.gateway.topology.simple.SimpleDescriptor; import org.apache.knox.gateway.topology.simple.SimpleDescriptor.Application; @@ -129,50 +127,26 @@ public void testCMDescriptorParserWrongXMLContent() throws Exception { } @Test - public void testCMDescriptorParserWithNotEnabledServices() throws Exception { - final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath(); - - final Properties advancedConfigurationTopology1 = new Properties(); - advancedConfigurationTopology1.put(buildEnabledParameter("topology1", "HIVE"), "false"); - advancedConfigurationTopology1.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1"); - hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfigurationTopology1); - - final Properties advancedConfigurationTopology2 = new Properties(); - advancedConfigurationTopology2.put(buildEnabledParameter("topology2", "NIFI"), "false"); - advancedConfigurationTopology2.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology2"); - hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfigurationTopology2); + public void testDescriptorParserWithServices() throws Exception { + final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorWithServiceList.xml").getPath(); final Set descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors(); - assertEquals(2, descriptors.size()); + assertEquals(1, descriptors.size()); final Iterator descriptorsIterator = descriptors.iterator(); - SimpleDescriptor topology1 = descriptorsIterator.next(); + final SimpleDescriptor topology1 = descriptorsIterator.next(); assertNotNull(topology1); - // topology1 comes with HIVE which is disabled - assertTrue(topology1.getServices().isEmpty()); - - SimpleDescriptor topology2 = descriptorsIterator.next(); - assertNotNull(topology2); - // topology1 comes with ATLAS and NIFI but the latter one is disabled - validateTopology2Descriptors(topology2, false); - } - - @Test - public void testCMDescriptorParserWithEnabledNotListedServiceInTopology1() throws Exception { - final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptor.xml").getPath(); - final Properties advancedConfiguration = new Properties(); - advancedConfiguration.put(buildEnabledParameter("topology1", "oozie"), "true"); //it should not matter if service name is lowercase advanced configuration - advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1"); - hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfiguration); - final Set descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors(); - final Iterator descriptorsIterator = descriptors.iterator(); - SimpleDescriptor descriptor = descriptorsIterator.next(); - assertNotNull(descriptor); - // topology1 comes without OOZIE but it's enabled in topology1 -> OOZIE should be added without any url/version/parameter - assertService(descriptor, "OOZIE", null, null, null); - - descriptor = descriptorsIterator.next(); - validateTopology2Descriptors(descriptor, true); - assertNull(descriptor.getService("OOZIE")); + // services= NIFI,ATLAS ,hive, hue ,IMPALA# + // + RANGER and FLINK service declarations in separate lines + assertEquals(7, topology1.getServices().size()); + assertNotNull(topology1.getService("NIFI")); + assertNotNull(topology1.getService("ATLAS")); + assertNotNull(topology1.getService("HIVE")); //should be uppercase + assertNotNull(topology1.getService("HUE")); //should be uppercase + assertNotNull(topology1.getService("IMPALA")); + assertNotNull(topology1.getService("RANGER")); + assertNotNull(topology1.getService("FLINK")); + assertEquals("3.0", topology1.getService("FLINK").getVersion()); + assertNull(topology1.getService("AMBARI")); } @Test @@ -219,28 +193,6 @@ public void testCMDescriptorParserRemovingProviderParams() { validateTestDescriptorProviderConfigs(parserResult.getProviders(), "ldap://localhost:33389", true, false); } - private String buildEnabledParameter(String topologyName, String serviceName) { - return AdvancedServiceDiscoveryConfig.PARAMETER_NAME_PREFIX_ENABLED_SERVICE + topologyName + AdvancedServiceDiscoveryConfig.PARAMETER_NAME_POSTFIX_ENABLED_SERVICE + serviceName; - } - - @Test - public void testSettingDiscoveryDetails() throws Exception { - final String address = "http://myCmHost:7180"; - final String cluster = "My Test Cluster"; - final String testConfigPath = this.getClass().getClassLoader().getResource("testDescriptorWithoutDiscoveryDetails.xml").getPath(); - final Properties advancedConfiguration = new Properties(); - advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_TOPOLOGY_NAME, "topology1"); - advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_DISCOVERY_ADDRESS, address); - advancedConfiguration.put(AdvancedServiceDiscoveryConfig.PARAMETER_NAME_DISCOVERY_CLUSTER, cluster); - hadoopXmlResourceParser.onAdvancedServiceDiscoveryConfigurationChange(advancedConfiguration); - final Set descriptors = hadoopXmlResourceParser.parse(testConfigPath).getDescriptors(); - final Iterator descriptorsIterator = descriptors.iterator(); - SimpleDescriptor descriptor = descriptorsIterator.next(); - assertEquals(address, descriptor.getDiscoveryAddress()); - assertEquals(cluster, descriptor.getCluster()); - assertEquals("ClouderaManager", descriptor.getDiscoveryType()); - } - @Test public void testDelete() throws Exception { String testConfigPath = this.getClass().getClassLoader().getResource("testDelete.xml").getPath(); diff --git a/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml b/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml new file mode 100644 index 0000000000..978133631e --- /dev/null +++ b/gateway-topology-hadoop-xml/src/test/resources/testDescriptorWithServiceList.xml @@ -0,0 +1,34 @@ + + + + + topology1 + + discoveryType=ClouderaManager# + discoveryAddress=http://host:123# + discoveryUser=user# + discoveryPasswordAlias=alias# + cluster=Cluster 1# + providerConfigRef=topology1-provider# + app:knoxauth:param1.name=param1.value# + app:admin-ui# + services=NIFI,ATLAS,hive,hue,IMPALA# + RANGER# + FLINK:version=3.0 + + + \ No newline at end of file