diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py
index 4b7c48f5649..7eddb24f935 100644
--- a/readthedocs/config/config.py
+++ b/readthedocs/config/config.py
@@ -405,12 +405,6 @@ def validate_build(self):
             self.env_config.update(
                 DOCKER_IMAGE_SETTINGS[build['image']]
             )
-        # Update docker settings from user config
-        if 'DOCKER_IMAGE_SETTINGS' in self.env_config and \
-                build['image'] in self.env_config['DOCKER_IMAGE_SETTINGS']:
-            self.env_config.update(
-                self.env_config['DOCKER_IMAGE_SETTINGS'][build['image']]
-            )
 
         # Allow to override specific project
         config_image = self.defaults.get('build_image')
diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py
index 00a0095bcc6..4f755f65f8f 100644
--- a/readthedocs/doc_builder/config.py
+++ b/readthedocs/doc_builder/config.py
@@ -59,7 +59,6 @@ def load_yaml_config(version):
     img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None)
     if img_settings:
         env_config.update(img_settings)
-        env_config['DOCKER_IMAGE_SETTINGS'] = img_settings
 
     try:
         config = load_config(