diff --git a/kube/patch.sh b/kube/patch.sh
index f6f8d6c8..0402fff1 100755
--- a/kube/patch.sh
+++ b/kube/patch.sh
@@ -3,8 +3,9 @@
 set -e
 set -u
 
+CLEAN_UP_ORPHANED_REPLICA_SETS='--clean-up-orphaned-replica-sets'
 usage() {
-  echo -e "Usage: $0 <deployment|statefulset> <name>\n"
+  echo -e "Usage: $0 <deployment|statefulset> <name> [${CLEAN_UP_ORPHANED_REPLICA_SETS}]\n"
 }
 
 if [  $# -le 1 ]; then
@@ -12,6 +13,8 @@ if [  $# -le 1 ]; then
   exit 1
 fi
 
+SHOULD_CLEAN_UP=${3:-}
+
 # Override to use a different Docker image name for the sidecar.
 export SIDECAR_IMAGE_NAME=${SIDECAR_IMAGE_NAME:-'gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar'}
 
@@ -37,3 +40,10 @@ spec:
         - name: ${DATA_VOLUME}
           mountPath: ${DATA_DIR}
 "
+if [[ "${SHOULD_CLEAN_UP}" == "${CLEAN_UP_ORPHANED_REPLICA_SETS}" ]]; then
+  # Delete the replica sets from the old deployment. If the Prometheus Server is
+  # a deployment that does not have `revisionHistoryLimit` set to 0, this is
+  # useful to avoid PVC conflicts between the old replica set and the new one
+  # that prevents the pod from entering a RUNNING state.
+  kubectl -n "${KUBE_NAMESPACE}" get rs | grep "$2-" | awk '{print $1}' | xargs kubectl delete -n "${KUBE_NAMESPACE}" rs
+fi