diff --git a/packages/components/src/popover/index.js b/packages/components/src/popover/index.js
index 8df361ad9c7975..3a0625a13c5130 100644
--- a/packages/components/src/popover/index.js
+++ b/packages/components/src/popover/index.js
@@ -215,6 +215,7 @@ const Popover = (
 
 	const arrowRef = useRef( null );
 
+	const [ referenceElement, setReferenceElement ] = useState();
 	const [ fallbackReferenceElement, setFallbackReferenceElement ] =
 		useState();
 	const [ referenceOwnerDocument, setReferenceOwnerDocument ] = useState();
@@ -377,6 +378,7 @@ const Popover = (
 
 		referenceCallbackRef( resultingReferenceElement );
 
+		setReferenceElement( resultingReferenceElement );
 		setReferenceOwnerDocument( resultingReferenceOwnerDoc );
 	}, [
 		anchor,
@@ -494,7 +496,7 @@ const Popover = (
 		content = <Fill name={ slotName }>{ content }</Fill>;
 	}
 
-	if ( anchorRef || anchorRect ) {
+	if ( referenceElement && referenceElement !== fallbackReferenceElement ) {
 		return content;
 	}