diff --git a/packages/block-library/src/gallery/editor.scss b/packages/block-library/src/gallery/editor.scss
index 419fc365f468af..ffd0f7847a09cd 100644
--- a/packages/block-library/src/gallery/editor.scss
+++ b/packages/block-library/src/gallery/editor.scss
@@ -59,6 +59,19 @@ ul.wp-block-gallery {
 		}
 	}
 
+	.is-selected .block-library-gallery-item__inline-menu {
+		background-color: theme(primary);
+
+		.components-button {
+			color: $white;
+		}
+
+		.components-button:focus {
+			color: inherit;
+		}
+
+	}
+
 	.block-editor-rich-text figcaption {
 		a {
 			color: $white;
@@ -71,25 +84,16 @@ ul.wp-block-gallery {
 	position: absolute;
 	top: -2px;
 	right: -2px;
-	background-color: theme(primary);
 	display: inline-flex;
 	z-index: z-index(".block-library-gallery-item__inline-menu");
 
 	.components-button {
-		color: $white;
-		&:hover,
-		&:focus {
-			color: $white;
-		}
+		color: transparent;
 	}
 }
 
 .blocks-gallery-item__remove {
 	padding: 0;
-
-	&.components-button:focus {
-		color: inherit;
-	}
 }
 
 .blocks-gallery-item .components-spinner {
diff --git a/packages/block-library/src/gallery/gallery-image.js b/packages/block-library/src/gallery/gallery-image.js
index e69b8385fe7e52..2c91f13df2c6ab 100644
--- a/packages/block-library/src/gallery/gallery-image.js
+++ b/packages/block-library/src/gallery/gallery-image.js
@@ -127,28 +127,26 @@ class GalleryImage extends Component {
 
 		return (
 			<figure className={ className }>
-				{ isSelected &&
-					<div className="block-library-gallery-item__inline-menu">
-						<IconButton
-							icon="no-alt"
-							onClick={ onRemove }
-							className="blocks-gallery-item__remove"
-							label={ __( 'Remove Image' ) }
-						/>
-					</div>
-				}
 				{ href ? <a href={ href }>{ img }</a> : img }
-				{ ( ! RichText.isEmpty( caption ) || isSelected ) ? (
-					<RichText
-						tagName="figcaption"
-						placeholder={ __( 'Write caption…' ) }
-						value={ caption }
-						isSelected={ this.state.captionSelected }
-						onChange={ ( newCaption ) => setAttributes( { caption: newCaption } ) }
-						unstableOnFocus={ this.onSelectCaption }
-						inlineToolbar
+				<div className="block-library-gallery-item__inline-menu">
+					<IconButton
+						icon="no-alt"
+						onClick={ onRemove }
+						onFocus={ this.onSelectImage }
+						className="blocks-gallery-item__remove"
+						label={ __( 'Remove Image' ) }
+						disabled={ ! isSelected }
 					/>
-				) : null }
+				</div>
+				<RichText
+					tagName="figcaption"
+					placeholder={ isSelected ? __( 'Write caption…' ) : null }
+					value={ caption }
+					isSelected={ this.state.captionSelected }
+					onChange={ ( newCaption ) => setAttributes( { caption: newCaption } ) }
+					unstableOnFocus={ this.onSelectCaption }
+					inlineToolbar
+				/>
 			</figure>
 		);
 	}