diff --git a/pkg/debounce/refresher_test.go b/pkg/debounce/refresher_test.go
index 50ad896ea..08708dbce 100644
--- a/pkg/debounce/refresher_test.go
+++ b/pkg/debounce/refresher_test.go
@@ -26,7 +26,7 @@ func TestRefreshAfter(t *testing.T) {
 	}
 	debounce.RefreshAfter(time.Millisecond * 2)
 	debounce.RefreshAfter(time.Microsecond * 2)
-	time.Sleep(time.Millisecond * 1)
+	time.Sleep(time.Millisecond * 2)
 	// test that the second refresh call overrode the first - Micro < Milli so this should have ran
 	require.True(t, ref.wasRefreshed.Load())
 	ref.wasRefreshed.Store(false)
@@ -42,6 +42,6 @@ func TestRefreshAfter(t *testing.T) {
 	}
 	debounce.RefreshAfter(time.Microsecond * 2)
 	// test the error case
-	time.Sleep(time.Millisecond * 1)
+	time.Sleep(time.Millisecond * 2)
 	require.True(t, ref.wasRefreshed.Load())
 }
diff --git a/pkg/schema/definitions/refresh_test.go b/pkg/schema/definitions/refresh_test.go
index 627aecc8c..f3d9a5c50 100644
--- a/pkg/schema/definitions/refresh_test.go
+++ b/pkg/schema/definitions/refresh_test.go
@@ -29,7 +29,7 @@ func Test_onChangeCRD(t *testing.T) {
 	}
 	refreshHandler := refreshHandler{
 		debounceRef:      &refresher,
-		debounceDuration: time.Microsecond * 5,
+		debounceDuration: time.Microsecond * 2,
 	}
 	input := apiextv1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{
@@ -51,7 +51,7 @@ func Test_onChangeAPIService(t *testing.T) {
 	}
 	refreshHandler := refreshHandler{
 		debounceRef:      &refresher,
-		debounceDuration: time.Microsecond * 5,
+		debounceDuration: time.Microsecond * 2,
 	}
 	input := apiregv1.APIService{
 		ObjectMeta: metav1.ObjectMeta{
@@ -74,11 +74,11 @@ func Test_startBackgroundRefresh(t *testing.T) {
 	}
 	refreshHandler := refreshHandler{
 		debounceRef:      &refresher,
-		debounceDuration: time.Microsecond * 5,
+		debounceDuration: time.Microsecond * 2,
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-	refreshHandler.startBackgroundRefresh(ctx, time.Microsecond*10)
-	time.Sleep(time.Millisecond * 2)
+	refreshHandler.startBackgroundRefresh(ctx, time.Microsecond*2)
+	time.Sleep(time.Millisecond * 4)
 	require.True(t, internalRefresh.wasRefreshed.Load())
 	cancel()
 }