fix flaky test TestPeriodicQueueWithMultipleWorkers/queue_with_multip…

…le_workers_should_requeue_errors

The default queue adds a 5ms delay before requeuing the item, we should
account for that before checking the number of requeueus.

Tested it works fine now:

go test -timeout 30s -run ^TestPeriodicQueueWithMultipleWorkers/queue_with_multiple_workers_should_requeue_errors$ -v k8s.io/ingress-gce/pkg/utils -c -race

stress ./utils.test -test.run ^TestPeriodicQueueWithMultipleWorkers/queue_with_multiple_workers_should_requeue_errors$
5s: 24 runs so far, 0 failures
10s: 66 runs so far, 0 failures
15s: 99 runs so far, 0 failures
20s: 133 runs so far, 0 failures
25s: 173 runs so far, 0 failures
30s: 206 runs so far, 0 failures
35s: 243 runs so far, 0 failures
40s: 284 runs so far, 0 failures

Before

20s: 143 runs so far, 5 failures (3.50%)

pkg/utils/taskqueue_test.go

@@ -20,10 +20,11 @@ import (
 	"errors"
 	"reflect"
 	"testing"
+	"time"
 
-	"k8s.io/client-go/tools/cache"
 	"sync"
-	"time"
+
+	"k8s.io/client-go/tools/cache"
 )
 
 func TestPeriodicTaskQueue(t *testing.T) {
@@ -112,12 +113,13 @@ func TestPeriodicQueueWithMultipleWorkers(t *testing.T) {
 			for _, obj := range tc.inputObjs {
 				tq.Enqueue(cache.ExplicitKey(obj))
 			}
-
 			for tq.Len() > 0 {
 				time.Sleep(1 * time.Second)
 			}
 
 			if tc.expectRequeueForKey != "" {
+				// the item is requeued with a delay, by default is 5 ms in DefaultControllerRateLimiter()
+				time.Sleep(10 * time.Millisecond)
 				if tq.queue.NumRequeues(tc.expectRequeueForKey) == 0 {
 					t.Errorf("Got 0 requeues for %q, expected non-zero requeue on error.", tc.expectRequeueForKey)
 				}