diff --git a/test/load/allocation/README.md b/test/load/allocation/README.md index 5d71c19387..a20b080e27 100644 --- a/test/load/allocation/README.md +++ b/test/load/allocation/README.md @@ -169,45 +169,17 @@ last for 1 minute, representing a short game. You can use the provided runScenario.sh script by providing one parameter (a scenario file). The scenario file is a simple text file where each line represents a "scenario" that the program will execute before moving to the next -scenario. A scenario is a duration and the number of concurrent clients to use, +scenario. A scenario is a duration, the number of concurrent clients to use, and +the interval of the allocation requests submitted by each client in milliseconds separated by a comma. The program will create the desired number of clients and those clients send allocation requests to the allocator service for the scenario -duration. At the end of each scenario the program will print out some statistics -for the scenario. +duration in the defined cadence. At the end of each scenario the program will print +out some statistics for the scenario. Two sample scenario files are included in this directory, one which sends a constant rate of allocations for the duration of the test and another that sends a variable number of allocations. -Upon concluding, the program will print out the overall statistics from the test. - -``` -./runScenario.sh variable.txt -... -2022-02-24 10:57:44.985216321 +0000 UTC m=+13814.879251454 :Running Scenario 24 with 15 clients for 10m0s -=================== - -Finished Scenario 24 -Count: 100 Error: ObjectHasBeenModified -Count: 113 Error: TooManyConcurrentRequests -Count: 0 Error: NoAvailableGameServer -Count: 0 Error: Unknown - -Scenario Failure Count: 213, Allocation Count: 15497 - -Total Failure Count: 6841, Total Allocation Count: 523204 - -Final Error Totals -Count: 0 Error: NoAvailableGameServer -Count: 0 Error: Unknown -Count: 3950 Error: ObjectHasBeenModified -Count: 2891 Error: TooManyConcurrentRequests - - -2022-02-24 11:07:45.677220867 +0000 UTC m=+14415.571255996 -Final Total Failure Count: 6841, Total Allocation Count: 523204 -``` - Since error counts are gathered per scenario, it's recommended to keep each scenario short (e.g. 10 minutes) to narrow down the window when errors occurred even if the allocation rate stays at the same level for longer than diff --git a/test/load/allocation/fixed.txt b/test/load/allocation/fixed.txt index 9da82579ed..9c61f470f9 100644 --- a/test/load/allocation/fixed.txt +++ b/test/load/allocation/fixed.txt @@ -13,11 +13,11 @@ # limitations under the License. ### Fixed rate of allocations -#Duration,Number_of_clients/allocations -10m,10 -10m,10 -10m,10 -10m,10 -10m,10 -10m,10 -10m,10 +#Duration,Number_of_clients/allocations,interval +10m,10,500 +10m,10,500 +10m,10,500 +10m,10,500 +10m,10,500 +10m,10,500 +10m,10,500 diff --git a/test/load/allocation/runscenario/runscenario.go b/test/load/allocation/runscenario/runscenario.go index b3fad70aec..fddd7b6e0b 100644 --- a/test/load/allocation/runscenario/runscenario.go +++ b/test/load/allocation/runscenario/runscenario.go @@ -42,6 +42,11 @@ const ( objectHasBeenModified allocErrorCode = "ObjectHasBeenModified" tooManyConcurrentRequests allocErrorCode = "TooManyConcurrentRequests" noAvailableGameServer allocErrorCode = "NoAvailableGameServer" + storageError allocErrorCode = "StorageError" + deadLineExceeded allocErrorCode = "DeadLineExceeded" + connectionTimedOut allocErrorCode = "ConnectionTimedOut" + connectionRefused allocErrorCode = "ConnectionRefused" + errReadingFromServer allocErrorCode = "ErrReadingFromServer" ) var ( @@ -50,6 +55,11 @@ var ( objectHasBeenModified: "the object has been modified", tooManyConcurrentRequests: "too many concurrent requests", noAvailableGameServer: "no available GameServer to allocate", + storageError: "storage error", + deadLineExceeded: "context deadline exceeded", + connectionTimedOut: "connection timed out", + connectionRefused: "connection refused", + errReadingFromServer: "allocator seems crashed and restarted", } ) @@ -67,8 +77,9 @@ var ( ) type scenario struct { - duration time.Duration - numOfClients int + duration time.Duration + numOfClients int + intervalMillisecond int } func main() { @@ -83,10 +94,11 @@ func main() { scenarios := readScenarios(*scenariosFile) var totalAllocCnt uint64 var totalFailureCnt uint64 + var totalDuration float64 totalFailureDtls := allocErrorCodeCntMap() for i, sc := range *scenarios { - logger.Printf("\n\n%v :Running Scenario %v with %v clients for %v\n===================\n", time.Now(), i+1, sc.numOfClients, sc.duration) + logger.Printf("\n\n%v :Running Scenario %v with %v clients submitting requests every %vms for %v\n===================\n", time.Now(), i+1, sc.numOfClients, sc.intervalMillisecond, sc.duration) var wg sync.WaitGroup failureCnts := make([]uint64, sc.numOfClients) @@ -105,13 +117,20 @@ func main() { return } client := pb.NewAllocationServiceClient(conn) + var wgc sync.WaitGroup for durCtx.Err() == nil { - if err := allocate(client); err != noerror { - failureDtls[clientID][err]++ - failureCnts[clientID]++ - } + wgc.Add(1) + go func() { + defer wgc.Done() + if err := allocate(client); err != noerror { + failureDtls[clientID][err]++ + failureCnts[clientID]++ + } + }() allocCnts[clientID]++ + time.Sleep(time.Duration(sc.intervalMillisecond) * time.Millisecond) } + wgc.Wait() _ = conn.Close() // Ignore error handling because the connection will be closed when the main func exits anyway. }(k) } @@ -131,13 +150,13 @@ func main() { } totalAllocCnt += scnAllocCnt totalFailureCnt += scnFailureCnt + totalDuration += sc.duration.Seconds() for k, v := range scnErrDtls { if k != noerror { logger.Printf("Count: %v\t\tError: %v", v, k) } } - logger.Printf("\nScenario Failure Count: %v, Allocation Count: %v", scnFailureCnt, scnAllocCnt) - logger.Printf("\nTotal Failure Count: %v, Total Allocation Count: %v", totalFailureCnt, totalAllocCnt) + logger.Printf("\n\n%v\nnScenario Failure Count: %v, Allocation Count: %v, Failure rate: %v, allocation rate: %v", time.Now(), scnFailureCnt, scnAllocCnt, float64(scnFailureCnt)/float64(scnAllocCnt), float64(scnAllocCnt-scnFailureCnt)/sc.duration.Seconds()) } logger.Print("\nFinal Error Totals\n") @@ -146,7 +165,7 @@ func main() { logger.Printf("Count: %v\t\tError: %v", v, k) } } - logger.Printf("\n\n%v\nFinal Total Failure Count: %v, Total Allocation Count: %v", time.Now(), totalFailureCnt, totalAllocCnt) + logger.Printf("\n\n%v\nFinal Total Failure Count: %v, Total Allocation Count: %v, Failure rate: %v, allocation rate: %v", time.Now(), totalFailureCnt, totalAllocCnt, float64(totalFailureCnt)/float64(totalAllocCnt), float64(totalAllocCnt-totalFailureCnt)/totalDuration) } func dialOptions(certFile, keyFile, cacertFile string) (grpc.DialOption, error) { @@ -224,8 +243,8 @@ func readScenarios(file string) *[]scenario { continue } lineParts := strings.Split(line, ",") - if len(lineParts) != 2 { - logger.Fatalf("There should be 2 parts for each scenario but there is %d for %q", len(lineParts), line) + if len(lineParts) != 3 { + logger.Fatalf("There should be 3 parts for each scenario but there is %d for %q", len(lineParts), line) } duration, err := time.ParseDuration(lineParts[0]) if err != nil { @@ -235,7 +254,11 @@ func readScenarios(file string) *[]scenario { if err != nil { logger.Fatalf("Failed parsing number of clients %s: %v", lineParts[1], err) } - scenarios = append(scenarios, scenario{duration, numClients}) + intervalMillisecond, err := strconv.Atoi(lineParts[2]) + if err != nil { + logger.Fatalf("Failed parsing intervalMillisecond %s: %v", lineParts[2], err) + } + scenarios = append(scenarios, scenario{duration, numClients, intervalMillisecond}) } return &scenarios @@ -257,5 +280,10 @@ func allocErrorCodeCntMap() map[allocErrorCode]uint64 { objectHasBeenModified: 0, tooManyConcurrentRequests: 0, noAvailableGameServer: 0, + storageError: 0, + deadLineExceeded: 0, + connectionTimedOut: 0, + connectionRefused: 0, + errReadingFromServer: 0, } } diff --git a/test/load/allocation/variable.txt b/test/load/allocation/variable.txt index b860f89ee5..3754a3a0d7 100644 --- a/test/load/allocation/variable.txt +++ b/test/load/allocation/variable.txt @@ -14,35 +14,35 @@ # ### Varying allocations #Duration,Number_of_clients/allocations -# run for 20 mins with 10 clients -10m,10 -10m,10 -# run for 30 mins with 30 clients -10m,30 -10m,30 -10m,30 -# run for 30 mins with 20 clients -10m,20 -10m,20 -10m,20 -# run for 30 mins with 30 clients -10m,30 -10m,30 -10m,30 -# run for 30 mins with 20 clients -10m,20 -10m,20 -10m,20 -# run for 30 mins with 5 clients -10m,5 -10m,5 -10m,5 -# run for 40 mins with 30 clients -10m,30 -10m,30 -10m,30 -10m,30 -# run for 30 mins with 15 clients -10m,15 -10m,15 -10m,15 +# run for 20 mins with 10 clients and 500ms interval +10m,10,500 +10m,10,500 +# run for 30 mins with 30 clients and 500ms interval +10m,30,500 +10m,30,500 +10m,30,500 +# run for 30 mins with 20 clients and 500ms interval +10m,20,500 +10m,20,500 +10m,20,500 +# run for 30 mins with 30 clients and 500ms interval +10m,30,500 +10m,30,500 +10m,30,500 +# run for 30 mins with 20 clients and 500ms interval +10m,20,500 +10m,20,500 +10m,20,500 +# run for 30 mins with 5 clients and 500ms interval +10m,5,500 +10m,5,500 +10m,5,500 +# run for 40 mins with 30 clients and 500ms interval +10m,30,500 +10m,30,500 +10m,30,500 +10m,30,500 +# run for 30 mins with 15 clients and 500ms interval +10m,15,500 +10m,15,500 +10m,15,500