diff --git a/dubbo-common/src/main/java/org/apache/dubbo/config/context/AbstractConfigManager.java b/dubbo-common/src/main/java/org/apache/dubbo/config/context/AbstractConfigManager.java
index ebd903ba332..04e9b0bdeb3 100644
--- a/dubbo-common/src/main/java/org/apache/dubbo/config/context/AbstractConfigManager.java
+++ b/dubbo-common/src/main/java/org/apache/dubbo/config/context/AbstractConfigManager.java
@@ -223,6 +223,13 @@ private <C extends AbstractConfig> C addIfAbsent(C config, Map<String, C> config
         return config;
     }
 
+    protected <C extends AbstractConfig> boolean removeIfAbsent(C config, Map<String, C> configsMap) {
+        if(config.getId() != null) {
+            return configsMap.remove(config.getId(), config);
+        }
+        return configsMap.values().removeIf(c -> config == c);
+    }
+
     protected boolean isUniqueConfig(AbstractConfig config) {
         if (uniqueConfigTypes.contains(config.getClass())) {
             return true;
@@ -651,7 +658,10 @@ public boolean removeConfig(AbstractConfig config) {
 
         Map<String, AbstractConfig> configs = configsCache.get(getTagName(config.getClass()));
         if (CollectionUtils.isNotEmptyMap(configs)) {
-            return configs.values().removeIf(c -> config == c);
+            // lock by config type
+            synchronized (configs) {
+                return removeIfAbsent(config, configs);
+            }
         }
         return false;
     }
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/config/context/ModuleConfigManager.java b/dubbo-common/src/main/java/org/apache/dubbo/config/context/ModuleConfigManager.java
index a935a881f56..bc1ddc38da8 100644
--- a/dubbo-common/src/main/java/org/apache/dubbo/config/context/ModuleConfigManager.java
+++ b/dubbo-common/src/main/java/org/apache/dubbo/config/context/ModuleConfigManager.java
@@ -199,6 +199,17 @@ protected <C extends AbstractConfig> Optional<C> findDuplicatedConfig(Map<String
         return Optional.empty();
     }
 
+    @Override
+    protected <C extends AbstractConfig> boolean removeIfAbsent(C config, Map<String, C> configsMap) {
+        if(super.removeIfAbsent(config, configsMap)) {
+            if (config instanceof ReferenceConfigBase || config instanceof ServiceConfigBase) {
+                removeInterfaceConfig((AbstractInterfaceConfig) config);
+            }
+            return true;
+        }
+        return false;
+    }
+
     /**
      * check duplicated ReferenceConfig/ServiceConfig
      *
@@ -247,6 +258,21 @@ private AbstractInterfaceConfig findDuplicatedInterfaceConfig(AbstractInterfaceC
         return prevConfig;
     }
 
+    private void removeInterfaceConfig(AbstractInterfaceConfig config) {
+        String uniqueServiceName;
+        Map<String, AbstractInterfaceConfig> configCache;
+        if (config instanceof ReferenceConfigBase) {
+            return;
+        } else if (config instanceof ServiceConfigBase) {
+            ServiceConfigBase serviceConfig = (ServiceConfigBase) config;
+            uniqueServiceName = serviceConfig.getUniqueServiceName();
+            configCache = serviceConfigCache;
+        } else {
+            throw new IllegalArgumentException("Illegal type of parameter 'config' : " + config.getClass().getName());
+        }
+        configCache.remove(uniqueServiceName, config);
+    }
+
     @Override
     public void loadConfigs() {
         // load dubbo.providers.xxx
diff --git a/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/ServiceBean.java b/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/ServiceBean.java
index 0d209a14f1e..7e0c4ee6408 100644
--- a/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/ServiceBean.java
+++ b/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/ServiceBean.java
@@ -104,6 +104,13 @@ public void afterPropertiesSet() throws Exception {
         moduleModel.getDeployer().setPending();
     }
 
+    @Override
+    public void unexport() {
+        ModuleModel moduleModel = DubboBeanUtils.getModuleModel(applicationContext);
+        moduleModel.getConfigManager().removeConfig(this);
+        super.unexport();
+    }
+
     /**
      * Get the name of {@link ServiceBean}
      *