diff --git a/scripts/v1alpha1/undeploy.sh b/scripts/v1alpha1/undeploy.sh
new file mode 100755
index 00000000000..70206b1459d
--- /dev/null
+++ b/scripts/v1alpha1/undeploy.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Delete CR first
+studyjobs=`kubectl get studyjobs --all-namespaces | awk 'NR>1' | awk '{print $1"/"$2}'`
+for s in $studyjobs
+do
+  kubectl delete studyjobs $s --grace-period=0 --force;
+done
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../..
+
+cd ${SCRIPT_ROOT}
+kubectl delete -f manifests/v1alpha1/pv
+kubectl delete -f manifests/v1alpha1/vizier/db
+kubectl delete -f manifests/v1alpha1/vizier/core
+kubectl delete -f manifests/v1alpha1/vizier/core-rest
+kubectl delete -f manifests/v1alpha1/vizier/ui
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/random
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/grid
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/hyperband
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/bayesianoptimization
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/nasrl
+kubectl delete -f manifests/v1alpha1/vizier/suggestion/nasenvelopenet
+kubectl delete -f manifests/v1alpha1/vizier/earlystopping/medianstopping
+kubectl delete -f manifests/v1alpha1/studyjobcontroller
+kubectl delete -f manifests/v1alpha1
+cd - > /dev/null
diff --git a/scripts/v1alpha2/undeploy.sh b/scripts/v1alpha2/undeploy.sh
new file mode 100755
index 00000000000..baa9f3cc03a
--- /dev/null
+++ b/scripts/v1alpha2/undeploy.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Delete CR first
+experiments=`kubectl get experiments --all-namespaces | awk 'NR>1' | awk '{print $1"/"$2}'`
+for s in $experiments
+do
+  kubectl delete experiments $s --grace-period=0 --force;
+done
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../..
+
+cd ${SCRIPT_ROOT}
+kubectl delete -f manifests/v1alpha2/katib-controller
+kubectl delete -f manifests/v1alpha2/katib/manager
+kubectl delete -f manifests/v1alpha2/katib/manager-rest
+kubectl delete -f manifests/v1alpha2/katib/pv
+kubectl delete -f manifests/v1alpha2/katib/db
+kubectl delete -f manifests/v1alpha2/katib/suggestion/random
+kubectl delete -f manifests/v1alpha2
+cd - > /dev/null